ronin-web 0.2.1 → 0.3.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +4 -0
- data/.gemtest +0 -0
- data/.rspec +1 -0
- data/.yardopts +1 -0
- data/COPYING.txt +674 -0
- data/{History.txt → ChangeLog.md} +39 -8
- data/Gemfile +49 -0
- data/{README.txt → README.md} +56 -40
- data/Rakefile +25 -24
- data/bin/ronin-web +18 -5
- data/gemspec.yml +30 -0
- data/lib/ronin/network/{helpers → mixins}/web.rb +74 -39
- data/lib/ronin/web.rb +7 -6
- data/lib/ronin/web/extensions.rb +7 -6
- data/lib/ronin/web/extensions/nokogiri.rb +22 -0
- data/lib/ronin/web/extensions/nokogiri/xml.rb +22 -0
- data/lib/ronin/web/extensions/nokogiri/xml/attr.rb +35 -1
- data/lib/ronin/web/extensions/nokogiri/xml/document.rb +34 -4
- data/lib/ronin/web/extensions/nokogiri/xml/element.rb +36 -2
- data/lib/ronin/web/extensions/nokogiri/xml/node.rb +59 -20
- data/lib/ronin/web/extensions/nokogiri/xml/text.rb +36 -2
- data/lib/ronin/web/middleware.rb +27 -0
- data/lib/ronin/web/middleware/base.rb +144 -0
- data/lib/ronin/web/middleware/directories.rb +177 -0
- data/lib/ronin/web/middleware/files.rb +142 -0
- data/lib/ronin/web/middleware/filters.rb +28 -0
- data/lib/ronin/web/middleware/filters/campaign_filter.rb +77 -0
- data/lib/ronin/web/middleware/filters/ip_filter.rb +73 -0
- data/lib/ronin/web/middleware/filters/path_filter.rb +73 -0
- data/lib/ronin/web/middleware/filters/referer_filter.rb +71 -0
- data/lib/ronin/web/middleware/filters/user_agent_filter.rb +71 -0
- data/lib/ronin/web/middleware/filters/vhost_filter.rb +71 -0
- data/lib/ronin/web/middleware/helpers.rb +145 -0
- data/lib/ronin/web/middleware/proxy.rb +265 -0
- data/lib/ronin/web/middleware/proxy_request.rb +262 -0
- data/lib/ronin/web/middleware/request.rb +58 -0
- data/lib/ronin/web/middleware/response.rb +33 -0
- data/lib/ronin/web/middleware/router.rb +167 -0
- data/lib/ronin/web/middleware/rule.rb +103 -0
- data/lib/ronin/web/proxy.rb +7 -6
- data/lib/ronin/web/proxy/app.rb +7 -6
- data/lib/ronin/web/proxy/base.rb +14 -9
- data/lib/ronin/web/proxy/web.rb +9 -6
- data/lib/ronin/web/server.rb +7 -6
- data/lib/ronin/web/server/app.rb +7 -6
- data/lib/ronin/web/server/base.rb +209 -82
- data/lib/ronin/web/server/web.rb +10 -6
- data/lib/ronin/web/spider.rb +38 -26
- data/lib/ronin/web/version.rb +8 -7
- data/lib/ronin/web/web.rb +122 -80
- data/ronin-web.gemspec +15 -0
- data/spec/helpers/output.rb +3 -0
- data/spec/spec_helper.rb +2 -3
- data/spec/web/extensions/nokogiri_spec.rb +7 -7
- data/spec/web/{server/helpers/server.rb → helpers/rack_app.rb} +3 -14
- data/spec/web/helpers/root.rb +15 -0
- data/spec/web/helpers/root/test1.txt +1 -0
- data/spec/web/helpers/root/test1/index.html +1 -0
- data/spec/web/helpers/root/test1/test1.txt +1 -0
- data/spec/web/helpers/root/test2.txt +1 -0
- data/spec/web/helpers/root/test2/test2.txt +1 -0
- data/spec/web/helpers/root/test3.txt +1 -0
- data/spec/web/helpers/root/test3/test3.txt +1 -0
- data/spec/web/middleware/directories_spec.rb +86 -0
- data/spec/web/middleware/files_spec.rb +57 -0
- data/spec/web/middleware/filters/campaign_filter_spec.rb +30 -0
- data/spec/web/middleware/filters/ip_filter_spec.rb +25 -0
- data/spec/web/middleware/filters/path_filter_spec.rb +29 -0
- data/spec/web/middleware/filters/referer_filter_spec.rb +25 -0
- data/spec/web/middleware/filters/user_agent_filter_spec.rb +25 -0
- data/spec/web/middleware/filters/vhost_filter_spec.rb +23 -0
- data/spec/web/middleware/proxy_spec.rb +67 -0
- data/spec/web/middleware/response_spec.rb +20 -0
- data/spec/web/middleware/router_spec.rb +65 -0
- data/spec/web/middleware/rule_spec.rb +37 -0
- data/spec/web/proxy/base_spec.rb +1 -2
- data/spec/web/server/base_spec.rb +4 -13
- data/spec/web/server/classes/sub_app.rb +2 -2
- data/spec/web/server/classes/test_app.rb +1 -1
- data/spec/web/web_spec.rb +57 -67
- metadata +171 -146
- data.tar.gz.sig +0 -0
- data/Manifest.txt +0 -64
- data/lib/ronin/scanners/web.rb +0 -52
- data/lib/ronin/web/server/files.rb +0 -92
- data/lib/ronin/web/server/helpers.rb +0 -25
- data/lib/ronin/web/server/helpers/files.rb +0 -126
- data/lib/ronin/web/server/helpers/hosts.rb +0 -72
- data/lib/ronin/web/server/helpers/proxy.rb +0 -153
- data/lib/ronin/web/server/helpers/rendering.rb +0 -36
- data/lib/ronin/web/server/hosts.rb +0 -86
- data/lib/ronin/web/server/proxy.rb +0 -116
- data/spec/scanners/web_spec.rb +0 -24
- data/spec/web/helpers/root/index.html +0 -1
- data/spec/web/helpers/root/test.txt +0 -1
- data/spec/web/helpers/server.rb +0 -10
- data/spec/web/server/classes/files/dir/file.txt +0 -1
- data/spec/web/server/classes/files/dir/index.html +0 -1
- data/spec/web/server/classes/files/dir2/file2.txt +0 -1
- data/spec/web/server/classes/files/dir3/page.xml +0 -4
- data/spec/web/server/classes/files/file.txt +0 -1
- data/spec/web/server/classes/files_app.rb +0 -27
- data/spec/web/server/classes/hosts_app.rb +0 -40
- data/spec/web/server/classes/proxy_app.rb +0 -45
- data/spec/web/server/files_spec.rb +0 -74
- data/spec/web/server/hosts_spec.rb +0 -55
- data/spec/web/server/proxy_spec.rb +0 -49
- data/tasks/spec.rb +0 -10
- data/tasks/yard.rb +0 -13
- metadata.gz.sig +0 -3
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'ronin/web/middleware/proxy'
|
3
|
+
|
4
|
+
require 'web/helpers/rack_app'
|
5
|
+
|
6
|
+
describe Web::Middleware::Proxy do
|
7
|
+
include Helpers::Web::RackApp
|
8
|
+
|
9
|
+
before(:all) do
|
10
|
+
self.app = Class.new(Sinatra::Base) do
|
11
|
+
use Ronin::Web::Middleware::Proxy, :path => '/login' do |proxy|
|
12
|
+
proxy.every_request do |request|
|
13
|
+
request.scheme = 'https'
|
14
|
+
request.host = 'github.com'
|
15
|
+
request.port = 443
|
16
|
+
request.referer = 'http://github.com/login'
|
17
|
+
end
|
18
|
+
|
19
|
+
proxy.every_response do |response|
|
20
|
+
response.body.each { |chunk| chunk.gsub!('https:','http:') }
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
get '/' do
|
25
|
+
'unproxied'
|
26
|
+
end
|
27
|
+
|
28
|
+
get '/login' do
|
29
|
+
'unproxied login'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
it "should proxy requests that match the proxies filters" do
|
35
|
+
pending "http://github.com/brynary/rack-test/issues#issue/16" do
|
36
|
+
get '/login'
|
37
|
+
|
38
|
+
last_response.should be_ok
|
39
|
+
last_response.body.should_not == 'unproxied login'
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should allow rewriting proxied requests" do
|
44
|
+
pending "http://github.com/brynary/rack-test/issues#issue/16" do
|
45
|
+
get '/login'
|
46
|
+
|
47
|
+
last_response.should be_ok
|
48
|
+
last_response.body.should include('Log in')
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
it "should allow rewriting proxied responses" do
|
53
|
+
pending "http://github.com/brynary/rack-test/issues#issue/16" do
|
54
|
+
get '/login'
|
55
|
+
|
56
|
+
last_response.should be_ok
|
57
|
+
last_response.body.should_not include('https:')
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
it "should still route un-matched requests to the app" do
|
62
|
+
get '/'
|
63
|
+
|
64
|
+
last_response.should be_ok
|
65
|
+
last_response.body.should == 'unproxied'
|
66
|
+
end
|
67
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'ronin/web/middleware/response'
|
3
|
+
|
4
|
+
describe Web::Middleware::Response do
|
5
|
+
subject do
|
6
|
+
Web::Middleware::Response.new(
|
7
|
+
['Hello'],
|
8
|
+
200,
|
9
|
+
{'Content-Type' => 'text/html'}
|
10
|
+
)
|
11
|
+
end
|
12
|
+
|
13
|
+
it "should allow implicit splatting" do
|
14
|
+
status, headers, body = subject
|
15
|
+
|
16
|
+
status.should == 200
|
17
|
+
headers['Content-Type'].should == 'text/html'
|
18
|
+
body.should == subject
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'ronin/web/middleware/router'
|
3
|
+
|
4
|
+
require 'web/helpers/rack_app'
|
5
|
+
|
6
|
+
describe Web::Middleware::Router do
|
7
|
+
include Helpers::Web::RackApp
|
8
|
+
|
9
|
+
before(:all) do
|
10
|
+
self.app = Class.new(Sinatra::Base) do
|
11
|
+
FakeApp = Class.new(Sinatra::Base) do
|
12
|
+
|
13
|
+
get '/test/1' do
|
14
|
+
'fake'
|
15
|
+
end
|
16
|
+
|
17
|
+
get '/test/2' do
|
18
|
+
'fake'
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
use Ronin::Web::Middleware::Router do |router|
|
24
|
+
router.draw :referer => /google\.com/, :to => FakeApp
|
25
|
+
|
26
|
+
router.draw :user_agent => /MSIE/,
|
27
|
+
:referer => /myspace\.com/,
|
28
|
+
:to => FakeApp
|
29
|
+
end
|
30
|
+
|
31
|
+
get '/test/1' do
|
32
|
+
'real'
|
33
|
+
end
|
34
|
+
|
35
|
+
get '/test/2' do
|
36
|
+
'real'
|
37
|
+
end
|
38
|
+
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should route matched requests to other apps" do
|
43
|
+
get '/test/1', {}, {'HTTP_REFERER' => 'http://www.google.com/'}
|
44
|
+
|
45
|
+
last_response.should be_ok
|
46
|
+
last_response.body.should == 'fake'
|
47
|
+
end
|
48
|
+
|
49
|
+
it "should not route requests that do not match all rules" do
|
50
|
+
get '/test/2', {}, {
|
51
|
+
'HTTP_REFERER' => 'http://www.myspace.com/',
|
52
|
+
'HTTP_USER_AGENT' => 'Curl'
|
53
|
+
}
|
54
|
+
|
55
|
+
last_response.should be_ok
|
56
|
+
last_response.body.should_not == 'fake'
|
57
|
+
end
|
58
|
+
|
59
|
+
it "should still route un-matched requests to the app" do
|
60
|
+
get '/test/1'
|
61
|
+
|
62
|
+
last_response.should be_ok
|
63
|
+
last_response.body.should == 'real'
|
64
|
+
end
|
65
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'ronin/web/middleware/rule'
|
3
|
+
|
4
|
+
describe Web::Middleware::Rule do
|
5
|
+
subject { Web::Middleware::Rule }
|
6
|
+
|
7
|
+
before(:each) do
|
8
|
+
@request = mock('request')
|
9
|
+
|
10
|
+
@request.stub!(:host).and_return('www.example.com')
|
11
|
+
@request.stub!(:path).and_return('/path/sub/dir')
|
12
|
+
end
|
13
|
+
|
14
|
+
it "should match requests by default" do
|
15
|
+
rule = subject.new()
|
16
|
+
|
17
|
+
rule.match?(@request).should == true
|
18
|
+
end
|
19
|
+
|
20
|
+
it "should match requests against all filters" do
|
21
|
+
rule = subject.new(
|
22
|
+
:vhost => 'www.example.com',
|
23
|
+
:path => '/path/sub/dir'
|
24
|
+
)
|
25
|
+
|
26
|
+
rule.match?(@request).should == true
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should match requests against against custom logic" do
|
30
|
+
rule = subject.new(
|
31
|
+
:path => '/path/sub/dir',
|
32
|
+
:when => lambda { |request| request.host =~ /example/ }
|
33
|
+
)
|
34
|
+
|
35
|
+
rule.match?(@request).should == true
|
36
|
+
end
|
37
|
+
end
|
data/spec/web/proxy/base_spec.rb
CHANGED
@@ -1,25 +1,16 @@
|
|
1
|
+
require 'spec_helper'
|
1
2
|
require 'ronin/web/server/base'
|
2
3
|
|
3
|
-
require 'spec_helper'
|
4
4
|
require 'web/server/classes/test_app'
|
5
|
-
require 'web/
|
5
|
+
require 'web/helpers/rack_app'
|
6
6
|
|
7
7
|
describe Web::Server::Base do
|
8
|
-
include Helpers::Web::
|
8
|
+
include Helpers::Web::RackApp
|
9
9
|
|
10
10
|
before(:all) do
|
11
11
|
self.app = TestApp
|
12
12
|
end
|
13
13
|
|
14
|
-
it "should define a set of index file-names to search for" do
|
15
|
-
TestApp.indices.should == TestApp::DEFAULT_INDICES.to_set
|
16
|
-
end
|
17
|
-
|
18
|
-
it "should allow for defining new index file-names to search for" do
|
19
|
-
TestApp.index 'index.xml'
|
20
|
-
TestApp.indices.include?('index.xml').should == true
|
21
|
-
end
|
22
|
-
|
23
14
|
it "should find a suitable Rack::Handler for the web server" do
|
24
15
|
TestApp.handler_class.should_not be_nil
|
25
16
|
end
|
@@ -63,7 +54,7 @@ describe Web::Server::Base do
|
|
63
54
|
last_response.body.should == 'SubApp'
|
64
55
|
end
|
65
56
|
|
66
|
-
it "should modify the path_info as it maps paths to sub-apps" do
|
57
|
+
it "should not modify the path_info as it maps paths to sub-apps" do
|
67
58
|
get '/tests/subapp/hello'
|
68
59
|
|
69
60
|
last_response.should be_ok
|
data/spec/web/web_spec.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
|
-
require 'ronin/web/web'
|
2
|
-
|
3
1
|
require 'spec_helper'
|
2
|
+
require 'ronin/web/web'
|
4
3
|
|
5
4
|
describe Web do
|
5
|
+
let(:url) { 'http://ronin-ruby.github.com/' }
|
6
|
+
let(:title) { 'Ronin' }
|
7
|
+
|
6
8
|
it "should have a VERSION constant" do
|
7
9
|
Web.const_defined?('VERSION').should == true
|
8
10
|
end
|
@@ -26,7 +28,7 @@ describe Web do
|
|
26
28
|
}
|
27
29
|
end
|
28
30
|
|
29
|
-
doc.to_html.include
|
31
|
+
doc.to_html.should include("<html><body><div>hello</div></body></html>")
|
30
32
|
end
|
31
33
|
|
32
34
|
it "should be able to parse XML" do
|
@@ -47,7 +49,7 @@ describe Web do
|
|
47
49
|
}
|
48
50
|
end
|
49
51
|
|
50
|
-
doc.to_xml.include
|
52
|
+
doc.to_xml.should include("<root>\n <stuff name=\"bla\">hello</stuff>\n</root>")
|
51
53
|
end
|
52
54
|
|
53
55
|
it "should have a default proxy" do
|
@@ -58,30 +60,6 @@ describe Web do
|
|
58
60
|
Web.proxy.should_not be_enabled
|
59
61
|
end
|
60
62
|
|
61
|
-
describe "proxy_url" do
|
62
|
-
before(:all) do
|
63
|
-
@uri = URI('http://www.example.com:9001')
|
64
|
-
end
|
65
|
-
|
66
|
-
it "should convert Network::HTTP::Proxy objects into a URI" do
|
67
|
-
Web.proxy_url(Network::HTTP::Proxy.new(
|
68
|
-
:host => 'www.example.com',
|
69
|
-
:port => 9001
|
70
|
-
)).should == @uri
|
71
|
-
end
|
72
|
-
|
73
|
-
it "should convert a Hash into a URI" do
|
74
|
-
Web.proxy_url(
|
75
|
-
:host => 'www.example.com',
|
76
|
-
:port => 9001
|
77
|
-
).should == @uri
|
78
|
-
end
|
79
|
-
|
80
|
-
it "should convert a String into a valid URI" do
|
81
|
-
Web.proxy_url("www.example.com:9001").should == @uri
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
63
|
it "should provide User-Agent aliases" do
|
86
64
|
Web.user_agent_aliases.should_not be_empty
|
87
65
|
end
|
@@ -93,75 +71,87 @@ describe Web do
|
|
93
71
|
it "should allow setting of the User-Agent string using an alias" do
|
94
72
|
Web.user_agent_alias = 'Mac FireFox'
|
95
73
|
|
96
|
-
Web.user_agent.should == "Mozilla/5.0 (Macintosh; U;
|
74
|
+
Web.user_agent.should == "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6"
|
97
75
|
end
|
98
76
|
|
99
77
|
it "should open URLs as temporary files" do
|
100
|
-
file = Web.open(
|
78
|
+
file = Web.open(url)
|
101
79
|
|
102
|
-
file.read.should
|
80
|
+
file.read.should include(title)
|
103
81
|
end
|
104
82
|
|
105
83
|
describe "agent" do
|
106
|
-
|
107
|
-
Web.
|
84
|
+
it "should provide Mechanize agents" do
|
85
|
+
Web.agent.class.should == Mechanize
|
108
86
|
end
|
109
87
|
|
110
|
-
|
111
|
-
|
112
|
-
|
88
|
+
describe ":user_agent" do
|
89
|
+
before(:all) do
|
90
|
+
Web.user_agent = 'test'
|
91
|
+
end
|
113
92
|
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
end
|
93
|
+
it "should default to Web.user_agent" do
|
94
|
+
Web.agent.user_agent.should == 'test'
|
95
|
+
end
|
118
96
|
|
119
|
-
|
120
|
-
|
97
|
+
it "should support using a custom User-Agent string" do
|
98
|
+
agent = Web.agent(:user_agent => 'test2')
|
121
99
|
|
122
|
-
|
123
|
-
|
100
|
+
agent.user_agent.should == 'test2'
|
101
|
+
end
|
124
102
|
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
agent.user_agent.should == 'Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3'
|
129
|
-
end
|
103
|
+
it "should support using a custom User-Agent alias" do
|
104
|
+
agent = Web.agent(:user_agent_alias => 'iPhone')
|
130
105
|
|
131
|
-
|
132
|
-
it "should accept Proxy values" do
|
133
|
-
pending "WWW::Mechanize needs reader methods for the proxy settings"
|
106
|
+
agent.user_agent.should == "Mozilla/5.0 (iPhone; U; CPU like Mac OS X; en) AppleWebKit/420+ (KHTML, like Gecko) Version/3.0 Mobile/1C28 Safari/419.3"
|
134
107
|
end
|
135
108
|
|
136
|
-
|
109
|
+
after(:all) do
|
110
|
+
Web.user_agent = nil
|
137
111
|
end
|
112
|
+
end
|
113
|
+
|
114
|
+
describe ":proxy" do
|
115
|
+
let(:host) { '127.0.0.1' }
|
116
|
+
let(:port) { 8080 }
|
138
117
|
|
139
|
-
|
140
|
-
|
118
|
+
before(:all) do
|
119
|
+
Web.proxy = {:host => 'www.example.com', :port => port}
|
141
120
|
end
|
142
121
|
|
143
122
|
it "should default to Web.proxy" do
|
144
|
-
|
123
|
+
agent = Web.agent
|
124
|
+
|
125
|
+
agent.proxy_addr.should == Web.proxy.host
|
126
|
+
agent.proxy_port.should == Web.proxy.port
|
127
|
+
end
|
128
|
+
|
129
|
+
it "should support using custom proxies" do
|
130
|
+
agent = Web.agent(:proxy => Network::HTTP::Proxy.new(
|
131
|
+
:host => host,
|
132
|
+
:port => port
|
133
|
+
))
|
134
|
+
|
135
|
+
agent.proxy_addr.should == host
|
136
|
+
agent.proxy_port.should == port
|
145
137
|
end
|
146
138
|
|
147
|
-
|
148
|
-
|
149
|
-
Web.agent(:proxy => 42)
|
150
|
-
}.should raise_error(RuntimeError)
|
139
|
+
after(:all) do
|
140
|
+
Web.proxy = nil
|
151
141
|
end
|
152
142
|
end
|
153
143
|
end
|
154
144
|
|
155
|
-
it "should be able to get
|
156
|
-
page = Web.get(
|
145
|
+
it "should be able to get Mechanize pages" do
|
146
|
+
page = Web.get(url)
|
157
147
|
|
158
|
-
page.class.should ==
|
159
|
-
page.at('title').inner_text.should
|
148
|
+
page.class.should == Mechanize::Page
|
149
|
+
page.at('title').inner_text.should include(title)
|
160
150
|
end
|
161
151
|
|
162
|
-
it "should be able to get the bodies of
|
163
|
-
body = Web.get_body(
|
152
|
+
it "should be able to get the bodies of Mechanize pages" do
|
153
|
+
body = Web.get_body(url)
|
164
154
|
|
165
|
-
body.should
|
155
|
+
body.should include(title)
|
166
156
|
end
|
167
157
|
end
|