ronin-web 0.2.1 → 0.3.0.pre1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (110) hide show
  1. data/.document +4 -0
  2. data/.gemtest +0 -0
  3. data/.rspec +1 -0
  4. data/.yardopts +1 -0
  5. data/COPYING.txt +674 -0
  6. data/{History.txt → ChangeLog.md} +39 -8
  7. data/Gemfile +49 -0
  8. data/{README.txt → README.md} +56 -40
  9. data/Rakefile +25 -24
  10. data/bin/ronin-web +18 -5
  11. data/gemspec.yml +30 -0
  12. data/lib/ronin/network/{helpers → mixins}/web.rb +74 -39
  13. data/lib/ronin/web.rb +7 -6
  14. data/lib/ronin/web/extensions.rb +7 -6
  15. data/lib/ronin/web/extensions/nokogiri.rb +22 -0
  16. data/lib/ronin/web/extensions/nokogiri/xml.rb +22 -0
  17. data/lib/ronin/web/extensions/nokogiri/xml/attr.rb +35 -1
  18. data/lib/ronin/web/extensions/nokogiri/xml/document.rb +34 -4
  19. data/lib/ronin/web/extensions/nokogiri/xml/element.rb +36 -2
  20. data/lib/ronin/web/extensions/nokogiri/xml/node.rb +59 -20
  21. data/lib/ronin/web/extensions/nokogiri/xml/text.rb +36 -2
  22. data/lib/ronin/web/middleware.rb +27 -0
  23. data/lib/ronin/web/middleware/base.rb +144 -0
  24. data/lib/ronin/web/middleware/directories.rb +177 -0
  25. data/lib/ronin/web/middleware/files.rb +142 -0
  26. data/lib/ronin/web/middleware/filters.rb +28 -0
  27. data/lib/ronin/web/middleware/filters/campaign_filter.rb +77 -0
  28. data/lib/ronin/web/middleware/filters/ip_filter.rb +73 -0
  29. data/lib/ronin/web/middleware/filters/path_filter.rb +73 -0
  30. data/lib/ronin/web/middleware/filters/referer_filter.rb +71 -0
  31. data/lib/ronin/web/middleware/filters/user_agent_filter.rb +71 -0
  32. data/lib/ronin/web/middleware/filters/vhost_filter.rb +71 -0
  33. data/lib/ronin/web/middleware/helpers.rb +145 -0
  34. data/lib/ronin/web/middleware/proxy.rb +265 -0
  35. data/lib/ronin/web/middleware/proxy_request.rb +262 -0
  36. data/lib/ronin/web/middleware/request.rb +58 -0
  37. data/lib/ronin/web/middleware/response.rb +33 -0
  38. data/lib/ronin/web/middleware/router.rb +167 -0
  39. data/lib/ronin/web/middleware/rule.rb +103 -0
  40. data/lib/ronin/web/proxy.rb +7 -6
  41. data/lib/ronin/web/proxy/app.rb +7 -6
  42. data/lib/ronin/web/proxy/base.rb +14 -9
  43. data/lib/ronin/web/proxy/web.rb +9 -6
  44. data/lib/ronin/web/server.rb +7 -6
  45. data/lib/ronin/web/server/app.rb +7 -6
  46. data/lib/ronin/web/server/base.rb +209 -82
  47. data/lib/ronin/web/server/web.rb +10 -6
  48. data/lib/ronin/web/spider.rb +38 -26
  49. data/lib/ronin/web/version.rb +8 -7
  50. data/lib/ronin/web/web.rb +122 -80
  51. data/ronin-web.gemspec +15 -0
  52. data/spec/helpers/output.rb +3 -0
  53. data/spec/spec_helper.rb +2 -3
  54. data/spec/web/extensions/nokogiri_spec.rb +7 -7
  55. data/spec/web/{server/helpers/server.rb → helpers/rack_app.rb} +3 -14
  56. data/spec/web/helpers/root.rb +15 -0
  57. data/spec/web/helpers/root/test1.txt +1 -0
  58. data/spec/web/helpers/root/test1/index.html +1 -0
  59. data/spec/web/helpers/root/test1/test1.txt +1 -0
  60. data/spec/web/helpers/root/test2.txt +1 -0
  61. data/spec/web/helpers/root/test2/test2.txt +1 -0
  62. data/spec/web/helpers/root/test3.txt +1 -0
  63. data/spec/web/helpers/root/test3/test3.txt +1 -0
  64. data/spec/web/middleware/directories_spec.rb +86 -0
  65. data/spec/web/middleware/files_spec.rb +57 -0
  66. data/spec/web/middleware/filters/campaign_filter_spec.rb +30 -0
  67. data/spec/web/middleware/filters/ip_filter_spec.rb +25 -0
  68. data/spec/web/middleware/filters/path_filter_spec.rb +29 -0
  69. data/spec/web/middleware/filters/referer_filter_spec.rb +25 -0
  70. data/spec/web/middleware/filters/user_agent_filter_spec.rb +25 -0
  71. data/spec/web/middleware/filters/vhost_filter_spec.rb +23 -0
  72. data/spec/web/middleware/proxy_spec.rb +67 -0
  73. data/spec/web/middleware/response_spec.rb +20 -0
  74. data/spec/web/middleware/router_spec.rb +65 -0
  75. data/spec/web/middleware/rule_spec.rb +37 -0
  76. data/spec/web/proxy/base_spec.rb +1 -2
  77. data/spec/web/server/base_spec.rb +4 -13
  78. data/spec/web/server/classes/sub_app.rb +2 -2
  79. data/spec/web/server/classes/test_app.rb +1 -1
  80. data/spec/web/web_spec.rb +57 -67
  81. metadata +171 -146
  82. data.tar.gz.sig +0 -0
  83. data/Manifest.txt +0 -64
  84. data/lib/ronin/scanners/web.rb +0 -52
  85. data/lib/ronin/web/server/files.rb +0 -92
  86. data/lib/ronin/web/server/helpers.rb +0 -25
  87. data/lib/ronin/web/server/helpers/files.rb +0 -126
  88. data/lib/ronin/web/server/helpers/hosts.rb +0 -72
  89. data/lib/ronin/web/server/helpers/proxy.rb +0 -153
  90. data/lib/ronin/web/server/helpers/rendering.rb +0 -36
  91. data/lib/ronin/web/server/hosts.rb +0 -86
  92. data/lib/ronin/web/server/proxy.rb +0 -116
  93. data/spec/scanners/web_spec.rb +0 -24
  94. data/spec/web/helpers/root/index.html +0 -1
  95. data/spec/web/helpers/root/test.txt +0 -1
  96. data/spec/web/helpers/server.rb +0 -10
  97. data/spec/web/server/classes/files/dir/file.txt +0 -1
  98. data/spec/web/server/classes/files/dir/index.html +0 -1
  99. data/spec/web/server/classes/files/dir2/file2.txt +0 -1
  100. data/spec/web/server/classes/files/dir3/page.xml +0 -4
  101. data/spec/web/server/classes/files/file.txt +0 -1
  102. data/spec/web/server/classes/files_app.rb +0 -27
  103. data/spec/web/server/classes/hosts_app.rb +0 -40
  104. data/spec/web/server/classes/proxy_app.rb +0 -45
  105. data/spec/web/server/files_spec.rb +0 -74
  106. data/spec/web/server/hosts_spec.rb +0 -55
  107. data/spec/web/server/proxy_spec.rb +0 -49
  108. data/tasks/spec.rb +0 -10
  109. data/tasks/yard.rb +0 -13
  110. metadata.gz.sig +0 -3
@@ -0,0 +1,67 @@
1
+ require 'spec_helper'
2
+ require 'ronin/web/middleware/proxy'
3
+
4
+ require 'web/helpers/rack_app'
5
+
6
+ describe Web::Middleware::Proxy do
7
+ include Helpers::Web::RackApp
8
+
9
+ before(:all) do
10
+ self.app = Class.new(Sinatra::Base) do
11
+ use Ronin::Web::Middleware::Proxy, :path => '/login' do |proxy|
12
+ proxy.every_request do |request|
13
+ request.scheme = 'https'
14
+ request.host = 'github.com'
15
+ request.port = 443
16
+ request.referer = 'http://github.com/login'
17
+ end
18
+
19
+ proxy.every_response do |response|
20
+ response.body.each { |chunk| chunk.gsub!('https:','http:') }
21
+ end
22
+ end
23
+
24
+ get '/' do
25
+ 'unproxied'
26
+ end
27
+
28
+ get '/login' do
29
+ 'unproxied login'
30
+ end
31
+ end
32
+ end
33
+
34
+ it "should proxy requests that match the proxies filters" do
35
+ pending "http://github.com/brynary/rack-test/issues#issue/16" do
36
+ get '/login'
37
+
38
+ last_response.should be_ok
39
+ last_response.body.should_not == 'unproxied login'
40
+ end
41
+ end
42
+
43
+ it "should allow rewriting proxied requests" do
44
+ pending "http://github.com/brynary/rack-test/issues#issue/16" do
45
+ get '/login'
46
+
47
+ last_response.should be_ok
48
+ last_response.body.should include('Log in')
49
+ end
50
+ end
51
+
52
+ it "should allow rewriting proxied responses" do
53
+ pending "http://github.com/brynary/rack-test/issues#issue/16" do
54
+ get '/login'
55
+
56
+ last_response.should be_ok
57
+ last_response.body.should_not include('https:')
58
+ end
59
+ end
60
+
61
+ it "should still route un-matched requests to the app" do
62
+ get '/'
63
+
64
+ last_response.should be_ok
65
+ last_response.body.should == 'unproxied'
66
+ end
67
+ end
@@ -0,0 +1,20 @@
1
+ require 'spec_helper'
2
+ require 'ronin/web/middleware/response'
3
+
4
+ describe Web::Middleware::Response do
5
+ subject do
6
+ Web::Middleware::Response.new(
7
+ ['Hello'],
8
+ 200,
9
+ {'Content-Type' => 'text/html'}
10
+ )
11
+ end
12
+
13
+ it "should allow implicit splatting" do
14
+ status, headers, body = subject
15
+
16
+ status.should == 200
17
+ headers['Content-Type'].should == 'text/html'
18
+ body.should == subject
19
+ end
20
+ end
@@ -0,0 +1,65 @@
1
+ require 'spec_helper'
2
+ require 'ronin/web/middleware/router'
3
+
4
+ require 'web/helpers/rack_app'
5
+
6
+ describe Web::Middleware::Router do
7
+ include Helpers::Web::RackApp
8
+
9
+ before(:all) do
10
+ self.app = Class.new(Sinatra::Base) do
11
+ FakeApp = Class.new(Sinatra::Base) do
12
+
13
+ get '/test/1' do
14
+ 'fake'
15
+ end
16
+
17
+ get '/test/2' do
18
+ 'fake'
19
+ end
20
+
21
+ end
22
+
23
+ use Ronin::Web::Middleware::Router do |router|
24
+ router.draw :referer => /google\.com/, :to => FakeApp
25
+
26
+ router.draw :user_agent => /MSIE/,
27
+ :referer => /myspace\.com/,
28
+ :to => FakeApp
29
+ end
30
+
31
+ get '/test/1' do
32
+ 'real'
33
+ end
34
+
35
+ get '/test/2' do
36
+ 'real'
37
+ end
38
+
39
+ end
40
+ end
41
+
42
+ it "should route matched requests to other apps" do
43
+ get '/test/1', {}, {'HTTP_REFERER' => 'http://www.google.com/'}
44
+
45
+ last_response.should be_ok
46
+ last_response.body.should == 'fake'
47
+ end
48
+
49
+ it "should not route requests that do not match all rules" do
50
+ get '/test/2', {}, {
51
+ 'HTTP_REFERER' => 'http://www.myspace.com/',
52
+ 'HTTP_USER_AGENT' => 'Curl'
53
+ }
54
+
55
+ last_response.should be_ok
56
+ last_response.body.should_not == 'fake'
57
+ end
58
+
59
+ it "should still route un-matched requests to the app" do
60
+ get '/test/1'
61
+
62
+ last_response.should be_ok
63
+ last_response.body.should == 'real'
64
+ end
65
+ end
@@ -0,0 +1,37 @@
1
+ require 'spec_helper'
2
+ require 'ronin/web/middleware/rule'
3
+
4
+ describe Web::Middleware::Rule do
5
+ subject { Web::Middleware::Rule }
6
+
7
+ before(:each) do
8
+ @request = mock('request')
9
+
10
+ @request.stub!(:host).and_return('www.example.com')
11
+ @request.stub!(:path).and_return('/path/sub/dir')
12
+ end
13
+
14
+ it "should match requests by default" do
15
+ rule = subject.new()
16
+
17
+ rule.match?(@request).should == true
18
+ end
19
+
20
+ it "should match requests against all filters" do
21
+ rule = subject.new(
22
+ :vhost => 'www.example.com',
23
+ :path => '/path/sub/dir'
24
+ )
25
+
26
+ rule.match?(@request).should == true
27
+ end
28
+
29
+ it "should match requests against against custom logic" do
30
+ rule = subject.new(
31
+ :path => '/path/sub/dir',
32
+ :when => lambda { |request| request.host =~ /example/ }
33
+ )
34
+
35
+ rule.match?(@request).should == true
36
+ end
37
+ end
@@ -1,6 +1,5 @@
1
- require 'ronin/web/proxy/base'
2
-
3
1
  require 'spec_helper'
2
+ require 'ronin/web/proxy/base'
4
3
 
5
4
  describe Web::Proxy::Base do
6
5
  it "should run on a different port than Web::Server::Base" do
@@ -1,25 +1,16 @@
1
+ require 'spec_helper'
1
2
  require 'ronin/web/server/base'
2
3
 
3
- require 'spec_helper'
4
4
  require 'web/server/classes/test_app'
5
- require 'web/server/helpers/server'
5
+ require 'web/helpers/rack_app'
6
6
 
7
7
  describe Web::Server::Base do
8
- include Helpers::Web::Server
8
+ include Helpers::Web::RackApp
9
9
 
10
10
  before(:all) do
11
11
  self.app = TestApp
12
12
  end
13
13
 
14
- it "should define a set of index file-names to search for" do
15
- TestApp.indices.should == TestApp::DEFAULT_INDICES.to_set
16
- end
17
-
18
- it "should allow for defining new index file-names to search for" do
19
- TestApp.index 'index.xml'
20
- TestApp.indices.include?('index.xml').should == true
21
- end
22
-
23
14
  it "should find a suitable Rack::Handler for the web server" do
24
15
  TestApp.handler_class.should_not be_nil
25
16
  end
@@ -63,7 +54,7 @@ describe Web::Server::Base do
63
54
  last_response.body.should == 'SubApp'
64
55
  end
65
56
 
66
- it "should modify the path_info as it maps paths to sub-apps" do
57
+ it "should not modify the path_info as it maps paths to sub-apps" do
67
58
  get '/tests/subapp/hello'
68
59
 
69
60
  last_response.should be_ok
@@ -2,11 +2,11 @@ require 'ronin/web/server/base'
2
2
 
3
3
  class SubApp < Ronin::Web::Server::Base
4
4
 
5
- get '/hello' do
5
+ get '/tests/subapp/hello' do
6
6
  'SubApp greets you'
7
7
  end
8
8
 
9
- get '/' do
9
+ get '/tests/subapp/' do
10
10
  'SubApp'
11
11
  end
12
12
 
@@ -12,7 +12,7 @@ class TestApp < Ronin::Web::Server::Base
12
12
  'any tested'
13
13
  end
14
14
 
15
- map '/tests/subapp', SubApp
15
+ map '/tests/subapp/', SubApp
16
16
 
17
17
  public_dir File.join(File.dirname(__FILE__),'public1')
18
18
  public_dir File.join(File.dirname(__FILE__),'public2')
@@ -1,8 +1,10 @@
1
- require 'ronin/web/web'
2
-
3
1
  require 'spec_helper'
2
+ require 'ronin/web/web'
4
3
 
5
4
  describe Web do
5
+ let(:url) { 'http://ronin-ruby.github.com/' }
6
+ let(:title) { 'Ronin' }
7
+
6
8
  it "should have a VERSION constant" do
7
9
  Web.const_defined?('VERSION').should == true
8
10
  end
@@ -26,7 +28,7 @@ describe Web do
26
28
  }
27
29
  end
28
30
 
29
- doc.to_html.include?("<html><body><div>hello</div></body></html>").should == true
31
+ doc.to_html.should include("<html><body><div>hello</div></body></html>")
30
32
  end
31
33
 
32
34
  it "should be able to parse XML" do
@@ -47,7 +49,7 @@ describe Web do
47
49
  }
48
50
  end
49
51
 
50
- doc.to_xml.include?("<root>\n <stuff name=\"bla\">hello</stuff>\n</root>").should == true
52
+ doc.to_xml.should include("<root>\n <stuff name=\"bla\">hello</stuff>\n</root>")
51
53
  end
52
54
 
53
55
  it "should have a default proxy" do
@@ -58,30 +60,6 @@ describe Web do
58
60
  Web.proxy.should_not be_enabled
59
61
  end
60
62
 
61
- describe "proxy_url" do
62
- before(:all) do
63
- @uri = URI('http://www.example.com:9001')
64
- end
65
-
66
- it "should convert Network::HTTP::Proxy objects into a URI" do
67
- Web.proxy_url(Network::HTTP::Proxy.new(
68
- :host => 'www.example.com',
69
- :port => 9001
70
- )).should == @uri
71
- end
72
-
73
- it "should convert a Hash into a URI" do
74
- Web.proxy_url(
75
- :host => 'www.example.com',
76
- :port => 9001
77
- ).should == @uri
78
- end
79
-
80
- it "should convert a String into a valid URI" do
81
- Web.proxy_url("www.example.com:9001").should == @uri
82
- end
83
- end
84
-
85
63
  it "should provide User-Agent aliases" do
86
64
  Web.user_agent_aliases.should_not be_empty
87
65
  end
@@ -93,75 +71,87 @@ describe Web do
93
71
  it "should allow setting of the User-Agent string using an alias" do
94
72
  Web.user_agent_alias = 'Mac FireFox'
95
73
 
96
- Web.user_agent.should == "Mozilla/5.0 (Macintosh; U; PPC Mac OS X Mach-O; en-US; rv:1.8.0.3) Gecko/20060426 Firefox/1.5.0.3"
74
+ Web.user_agent.should == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6"
97
75
  end
98
76
 
99
77
  it "should open URLs as temporary files" do
100
- file = Web.open('http://www.example.com/')
78
+ file = Web.open(url)
101
79
 
102
- file.read.should =~ /Example Web Page/
80
+ file.read.should include(title)
103
81
  end
104
82
 
105
83
  describe "agent" do
106
- after(:each) do
107
- Web.user_agent = nil
84
+ it "should provide Mechanize agents" do
85
+ Web.agent.class.should == Mechanize
108
86
  end
109
87
 
110
- it "should provide WWW::Mechanize agents" do
111
- Web.agent.class.should == WWW::Mechanize
112
- end
88
+ describe ":user_agent" do
89
+ before(:all) do
90
+ Web.user_agent = 'test'
91
+ end
113
92
 
114
- it "should use the Ronin User-Agent string" do
115
- Web.user_agent = 'test'
116
- Web.agent.user_agent.should == 'test'
117
- end
93
+ it "should default to Web.user_agent" do
94
+ Web.agent.user_agent.should == 'test'
95
+ end
118
96
 
119
- it "should support using a custom User-Agent string" do
120
- agent = Web.agent(:user_agent => 'test2')
97
+ it "should support using a custom User-Agent string" do
98
+ agent = Web.agent(:user_agent => 'test2')
121
99
 
122
- agent.user_agent.should == 'test2'
123
- end
100
+ agent.user_agent.should == 'test2'
101
+ end
124
102
 
125
- it "should support using a custom User-Agent alias" do
126
- agent = Web.agent(:user_agent_alias => 'iPhone')
127
-
128
- agent.user_agent.should == 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3'
129
- end
103
+ it "should support using a custom User-Agent alias" do
104
+ agent = Web.agent(:user_agent_alias => 'iPhone')
130
105
 
131
- describe ":proxy" do
132
- it "should accept Proxy values" do
133
- pending "WWW::Mechanize needs reader methods for the proxy settings"
106
+ agent.user_agent.should == "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3"
134
107
  end
135
108
 
136
- it "should accept Hash values" do
109
+ after(:all) do
110
+ Web.user_agent = nil
137
111
  end
112
+ end
113
+
114
+ describe ":proxy" do
115
+ let(:host) { '127.0.0.1' }
116
+ let(:port) { 8080 }
138
117
 
139
- it "should accept String values" do
140
- pending "WWW::Mechanize needs reader methods for the proxy settings"
118
+ before(:all) do
119
+ Web.proxy = {:host => 'www.example.com', :port => port}
141
120
  end
142
121
 
143
122
  it "should default to Web.proxy" do
144
- pending "WWW::Mechanize needs reader methods for the proxy settings"
123
+ agent = Web.agent
124
+
125
+ agent.proxy_addr.should == Web.proxy.host
126
+ agent.proxy_port.should == Web.proxy.port
127
+ end
128
+
129
+ it "should support using custom proxies" do
130
+ agent = Web.agent(:proxy => Network::HTTP::Proxy.new(
131
+ :host => host,
132
+ :port => port
133
+ ))
134
+
135
+ agent.proxy_addr.should == host
136
+ agent.proxy_port.should == port
145
137
  end
146
138
 
147
- it "should raise a RuntimeError exception for bad :proxy options" do
148
- lambda {
149
- Web.agent(:proxy => 42)
150
- }.should raise_error(RuntimeError)
139
+ after(:all) do
140
+ Web.proxy = nil
151
141
  end
152
142
  end
153
143
  end
154
144
 
155
- it "should be able to get WWW::Mechanize pages" do
156
- page = Web.get('http://www.example.com/')
145
+ it "should be able to get Mechanize pages" do
146
+ page = Web.get(url)
157
147
 
158
- page.class.should == WWW::Mechanize::Page
159
- page.at('title').inner_text.should == 'Example Web Page'
148
+ page.class.should == Mechanize::Page
149
+ page.at('title').inner_text.should include(title)
160
150
  end
161
151
 
162
- it "should be able to get the bodies of WWW::Mechanize pages" do
163
- body = Web.get_body('http://www.example.com/')
152
+ it "should be able to get the bodies of Mechanize pages" do
153
+ body = Web.get_body(url)
164
154
 
165
- body.should =~ /Example Web Page/
155
+ body.should include(title)
166
156
  end
167
157
  end