ronin-web 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- data.tar.gz.sig +0 -0
- data/History.txt +18 -0
- data/Manifest.txt +2 -0
- data/README.txt +94 -1
- data/Rakefile +9 -10
- data/TODO.txt +7 -0
- data/lib/ronin/web.rb +6 -0
- data/lib/ronin/web/fingerprint.rb +76 -0
- data/lib/ronin/web/server.rb +129 -83
- data/lib/ronin/web/spider.rb +8 -11
- data/lib/ronin/web/version.rb +1 -1
- data/lib/ronin/web/web.rb +43 -6
- data/spec/web/server_spec.rb +22 -22
- metadata +42 -20
- metadata.gz.sig +0 -0
data.tar.gz.sig
ADDED
Binary file
|
data/History.txt
CHANGED
@@ -1,3 +1,21 @@
|
|
1
|
+
=== 0.1.3 / 2009-07-02
|
2
|
+
|
3
|
+
* Use Hoe >= 2.0.0.
|
4
|
+
* Require spidr >= 0.1.9.
|
5
|
+
* Require rack >= 1.0.0.
|
6
|
+
* Require ronin >= 0.2.4.
|
7
|
+
* Added Ronin::Web::Fingerprint.
|
8
|
+
* Added Web.build_html.
|
9
|
+
* Added Web.build_xml.
|
10
|
+
* Allow Web.html to accept a block.
|
11
|
+
* Allow Web.xml to accept a block.
|
12
|
+
* Allow Server#return_file to accept a content_type option.
|
13
|
+
* Make sure Ronin::Web::Server is thread safe.
|
14
|
+
* Blocks passed to Server.new will not be instance_evaled.
|
15
|
+
* Removed Spider#default_options.
|
16
|
+
* Removed Server.run.
|
17
|
+
* Removed Server#config.
|
18
|
+
|
1
19
|
=== 0.1.2 / 2009-03-28
|
2
20
|
|
3
21
|
* Added Ronin::Web::Proxy.
|
data/Manifest.txt
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
History.txt
|
2
2
|
Manifest.txt
|
3
3
|
README.txt
|
4
|
+
TODO.txt
|
4
5
|
Rakefile
|
5
6
|
bin/ronin-web
|
6
7
|
lib/ronin/sessions/web.rb
|
@@ -13,6 +14,7 @@ lib/ronin/web/extensions/nokogiri/xml/text.rb
|
|
13
14
|
lib/ronin/web/extensions/nokogiri/xml/attr.rb
|
14
15
|
lib/ronin/web/extensions/nokogiri/xml/element.rb
|
15
16
|
lib/ronin/web/extensions/nokogiri/xml/document.rb
|
17
|
+
lib/ronin/web/fingerprint.rb
|
16
18
|
lib/ronin/web/server.rb
|
17
19
|
lib/ronin/web/proxy.rb
|
18
20
|
lib/ronin/web/spider.rb
|
data/README.txt
CHANGED
@@ -61,7 +61,100 @@ of Ronin.
|
|
61
61
|
|
62
62
|
* Start the Ronin console with Ronin Web preloaded:
|
63
63
|
|
64
|
-
|
64
|
+
$ ronin-web
|
65
|
+
|
66
|
+
== EXAMPLES:
|
67
|
+
|
68
|
+
* Get a web-page:
|
69
|
+
|
70
|
+
Web.get('http://www.example.com/')
|
71
|
+
|
72
|
+
* Get only the body of the web-page:
|
73
|
+
|
74
|
+
Web.get_body('http://www.example.com/')
|
75
|
+
|
76
|
+
* Get a WWW::Mechanize agent:
|
77
|
+
|
78
|
+
agent = Web.agent
|
79
|
+
|
80
|
+
* Parse HTML:
|
81
|
+
|
82
|
+
Web.html(open('some_file.html'))
|
83
|
+
# => <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
|
84
|
+
<html>
|
85
|
+
<head>
|
86
|
+
<script type="text/javascript" src="redirect.js"></script>
|
87
|
+
</head>
|
88
|
+
</html>
|
89
|
+
|
90
|
+
* Build a HTML document:
|
91
|
+
|
92
|
+
doc = Web.build_html do
|
93
|
+
html {
|
94
|
+
head {
|
95
|
+
script(:type => 'text/javascript', :src => 'redirect.js')
|
96
|
+
}
|
97
|
+
}
|
98
|
+
end
|
99
|
+
|
100
|
+
puts doc.to_html
|
101
|
+
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
|
102
|
+
<html><head><script src="redirect.js" type="text/javascript"></script></head></html>
|
103
|
+
|
104
|
+
* Parse XML:
|
105
|
+
|
106
|
+
Web.xml(some_text)
|
107
|
+
=> <?xml version="1.0"?>
|
108
|
+
<users>
|
109
|
+
<user>
|
110
|
+
<name>admin</name>
|
111
|
+
<password>0mni</password>
|
112
|
+
</user>
|
113
|
+
</users>
|
114
|
+
|
115
|
+
|
116
|
+
* Build a XML document:
|
117
|
+
|
118
|
+
doc = Web.build_xml do
|
119
|
+
playlist {
|
120
|
+
mp3 {
|
121
|
+
file { text('02 THE WAIT.mp3') }
|
122
|
+
artist { text('Evil Nine') }
|
123
|
+
track { text('The Wait feat David Autokratz') }
|
124
|
+
duration { text('1000000000') }
|
125
|
+
}
|
126
|
+
}
|
127
|
+
end
|
128
|
+
|
129
|
+
puts doc.to_xml
|
130
|
+
<?xml version="1.0"?>
|
131
|
+
<playlist>
|
132
|
+
<mp3>
|
133
|
+
<file>02 THE WAIT.mp3</file>
|
134
|
+
<artist>Evil Nine</artist>
|
135
|
+
<track>The Wait feat David Autokratz</track>
|
136
|
+
<duration>1000000000</duration>
|
137
|
+
</mp3>
|
138
|
+
</playlist>
|
139
|
+
|
140
|
+
* Spider a web site:
|
141
|
+
|
142
|
+
Web::Spider.host('www.example.com') do |spider|
|
143
|
+
spider.every_url do |url|
|
144
|
+
...
|
145
|
+
end
|
146
|
+
|
147
|
+
spider.every_page do |page|
|
148
|
+
...
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
* Serve files via a Web Server:
|
153
|
+
|
154
|
+
Web::Server.start do |server|
|
155
|
+
server.file '/opensearch.xml', '/tmp/test.xml'
|
156
|
+
server.directory '/download/', '/tmp/download/'
|
157
|
+
end
|
65
158
|
|
66
159
|
== LICENSE:
|
67
160
|
|
data/Rakefile
CHANGED
@@ -2,19 +2,18 @@
|
|
2
2
|
|
3
3
|
require 'rubygems'
|
4
4
|
require 'hoe'
|
5
|
+
require 'hoe/signing'
|
5
6
|
require './tasks/spec.rb'
|
6
|
-
require './lib/ronin/web/version.rb'
|
7
7
|
|
8
|
-
Hoe.
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
['nokogiri', '>=1.2.0'],
|
8
|
+
Hoe.spec('ronin-web') do
|
9
|
+
self.rubyforge_name = 'ronin'
|
10
|
+
self.developer('Postmodern', 'postmodern.mod3@gmail.com')
|
11
|
+
self.remote_rdoc_dir = 'docs/ronin-web'
|
12
|
+
self.extra_deps = [
|
14
13
|
['mechanize', '>=0.9.0'],
|
15
|
-
['spidr', '>=0.1.
|
16
|
-
['rack', '>=0.
|
17
|
-
['ronin', '>=0.2.
|
14
|
+
['spidr', '>=0.1.9'],
|
15
|
+
['rack', '>=1.0.0'],
|
16
|
+
['ronin', '>=0.2.4']
|
18
17
|
]
|
19
18
|
end
|
20
19
|
|
data/TODO.txt
ADDED
data/lib/ronin/web.rb
CHANGED
@@ -0,0 +1,76 @@
|
|
1
|
+
#
|
2
|
+
#--
|
3
|
+
# Ronin Web - A Ruby library for Ronin that provides support for web
|
4
|
+
# scraping and spidering functionality.
|
5
|
+
#
|
6
|
+
# Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
|
7
|
+
#
|
8
|
+
# This program is free software; you can redistribute it and/or modify
|
9
|
+
# it under the terms of the GNU General Public License as published by
|
10
|
+
# the Free Software Foundation; either version 2 of the License, or
|
11
|
+
# (at your option) any later version.
|
12
|
+
#
|
13
|
+
# This program is distributed in the hope that it will be useful,
|
14
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
15
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
16
|
+
# GNU General Public License for more details.
|
17
|
+
#
|
18
|
+
# You should have received a copy of the GNU General Public License
|
19
|
+
# along with this program; if not, write to the Free Software
|
20
|
+
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
|
21
|
+
#++
|
22
|
+
#
|
23
|
+
|
24
|
+
module Ronin
|
25
|
+
module Web
|
26
|
+
module Fingerprint
|
27
|
+
#
|
28
|
+
# The Hash of web application identities and their associated
|
29
|
+
# fingerprint tests.
|
30
|
+
#
|
31
|
+
def Fingerprint.tests
|
32
|
+
@@ronin_web_fingerprints ||= Hash.new do |hash,key|
|
33
|
+
hash[key] ||= []
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
#
|
38
|
+
# Adds a test for the web application identity with the specified
|
39
|
+
# _name_ and _block_. When the _block_ is called, it will be passed
|
40
|
+
# the URL of the unknown web application.
|
41
|
+
#
|
42
|
+
# Fingerprint.test_for('app') do |url|
|
43
|
+
# url.path.include?('/app/')
|
44
|
+
# end
|
45
|
+
#
|
46
|
+
def Fingerprint.test_for(name,&block)
|
47
|
+
Fingerprint.tests[name.to_sym] << block
|
48
|
+
return nil
|
49
|
+
end
|
50
|
+
|
51
|
+
#
|
52
|
+
# Identifies the web application represented by the specified _url_,
|
53
|
+
# returning the name of identified web application. If the
|
54
|
+
# web application cannot be identified, +nil+ will be returned.
|
55
|
+
#
|
56
|
+
def Fingerprint.identify(url)
|
57
|
+
unless url.kind_of?(URI)
|
58
|
+
url = URI(url.to_s)
|
59
|
+
end
|
60
|
+
|
61
|
+
matches = []
|
62
|
+
|
63
|
+
Fingerprint.tests.each do |name,blocks|
|
64
|
+
blocks.each do |block|
|
65
|
+
if block.call(url)
|
66
|
+
matches << name
|
67
|
+
break
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
return matches
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
data/lib/ronin/web/server.rb
CHANGED
@@ -23,9 +23,12 @@
|
|
23
23
|
|
24
24
|
require 'uri'
|
25
25
|
require 'cgi'
|
26
|
+
require 'thread'
|
26
27
|
|
27
28
|
begin
|
28
29
|
require 'mongrel'
|
30
|
+
rescue Gem::LoadError => e
|
31
|
+
raise(e)
|
29
32
|
rescue LoadError
|
30
33
|
require 'webrick'
|
31
34
|
end
|
@@ -51,8 +54,8 @@ module Ronin
|
|
51
54
|
# The port to listen on
|
52
55
|
attr_accessor :port
|
53
56
|
|
54
|
-
# The
|
55
|
-
|
57
|
+
# The handler to run the server under
|
58
|
+
attr_accessor :handler
|
56
59
|
|
57
60
|
#
|
58
61
|
# Creates a new Web Server using the given configuration _block_.
|
@@ -60,28 +63,30 @@ module Ronin
|
|
60
63
|
# _options_ may contain the following keys:
|
61
64
|
# <tt>:host</tt>:: The host to bind to.
|
62
65
|
# <tt>:port</tt>:: The port to listen on.
|
63
|
-
# <tt>:
|
64
|
-
# in responses.
|
66
|
+
# <tt>:handler</tt>:: The handler to run the server under.
|
65
67
|
#
|
66
68
|
def initialize(options={},&block)
|
67
69
|
@host = options[:host]
|
68
70
|
@port = options[:port]
|
69
|
-
@
|
70
|
-
|
71
|
-
if options.has_key?(:config)
|
72
|
-
@config.merge!(options[:config])
|
73
|
-
end
|
71
|
+
@handler = options[:handler]
|
74
72
|
|
75
73
|
@default = method(:not_found)
|
76
74
|
|
77
|
-
@
|
78
|
-
@
|
75
|
+
@vhost_patterns = {}
|
76
|
+
@vhosts = {}
|
79
77
|
|
80
|
-
@
|
78
|
+
@patterns = {}
|
81
79
|
@paths = {}
|
82
80
|
@directories = {}
|
83
81
|
|
84
|
-
|
82
|
+
@default_mutex = Mutex.new
|
83
|
+
@vhost_patterns_mutex = Mutex.new
|
84
|
+
@vhosts_mutex = Mutex.new
|
85
|
+
@patterns_mutex = Mutex.new
|
86
|
+
@paths_mutex = Mutex.new
|
87
|
+
@directories_mutex = Mutex.new
|
88
|
+
|
89
|
+
block.call(self) if block
|
85
90
|
end
|
86
91
|
|
87
92
|
#
|
@@ -127,36 +132,12 @@ module Ronin
|
|
127
132
|
#
|
128
133
|
# Server.content_type 'text/xml', ['xml', 'xsl']
|
129
134
|
#
|
130
|
-
def
|
135
|
+
def Server.content_type(type,extensions)
|
131
136
|
extensions.each { |ext| Server.content_types[ext] = type }
|
132
137
|
|
133
138
|
return self
|
134
139
|
end
|
135
140
|
|
136
|
-
#
|
137
|
-
# Runs the specified _server_ with the given _options_. Server.run
|
138
|
-
# will use Mongrel to run the _server_, if it is installed. Otherwise
|
139
|
-
# WEBrick will be used to run the _server_.
|
140
|
-
#
|
141
|
-
# _options_ can contain the following keys:
|
142
|
-
# <tt>:host</tt>:: The host the server will bind to, defaults to
|
143
|
-
# Server.default_host.
|
144
|
-
# <tt>:port</tt>:: The port the server will listen on, defaults to
|
145
|
-
# Server.default_port.
|
146
|
-
#
|
147
|
-
def Server.run(server,options={})
|
148
|
-
rack_options = {}
|
149
|
-
|
150
|
-
rack_options[:Host] = (options[:host] || Server.default_host)
|
151
|
-
rack_options[:Port] = (options[:port] || Server.default_port)
|
152
|
-
|
153
|
-
if Object.const_defined?('Mongrel')
|
154
|
-
Rack::Handler::Mongrel.run(server,rack_options)
|
155
|
-
else
|
156
|
-
Rack::Handler::WEBrick.run(server,rack_options)
|
157
|
-
end
|
158
|
-
end
|
159
|
-
|
160
141
|
#
|
161
142
|
# Creates a new Web Server object with the given _block_ and starts
|
162
143
|
# it using the given _options_.
|
@@ -168,17 +149,18 @@ module Ronin
|
|
168
149
|
#
|
169
150
|
# Returns the HTTP Content-Type for the specified file _extension_.
|
170
151
|
#
|
171
|
-
# content_type('html')
|
152
|
+
# server.content_type('html')
|
172
153
|
# # => "text/html"
|
173
154
|
#
|
174
155
|
def content_type(extension)
|
175
|
-
Server.content_types[extension] ||
|
156
|
+
Server.content_types[extension] ||
|
157
|
+
'application/x-unknown-content-type'
|
176
158
|
end
|
177
159
|
|
178
160
|
#
|
179
161
|
# Returns the HTTP Content-Type for the specified _file_.
|
180
162
|
#
|
181
|
-
#
|
163
|
+
# server.content_type_for('file.html')
|
182
164
|
# # => "text/html"
|
183
165
|
#
|
184
166
|
def content_type_for(file)
|
@@ -228,7 +210,9 @@ module Ronin
|
|
228
210
|
# an index file. If no index file can be found or _path_ points to a
|
229
211
|
# non-existant file, a "404 Not Found" response will be returned.
|
230
212
|
#
|
231
|
-
def return_file(path,request)
|
213
|
+
def return_file(path,request,content_type=nil)
|
214
|
+
content_type ||= content_type_for(path)
|
215
|
+
|
232
216
|
if !(File.exists?(path))
|
233
217
|
return not_found(request)
|
234
218
|
end
|
@@ -239,7 +223,7 @@ module Ronin
|
|
239
223
|
end
|
240
224
|
end
|
241
225
|
|
242
|
-
return response(File.new(path), :content_type =>
|
226
|
+
return response(File.new(path), :content_type => content_type)
|
243
227
|
end
|
244
228
|
|
245
229
|
#
|
@@ -249,10 +233,14 @@ module Ronin
|
|
249
233
|
# _options_ may include the following keys:
|
250
234
|
# <tt>:status</tt>:: The HTTP Response status code, defaults to 200.
|
251
235
|
#
|
252
|
-
# response("<data>lol</data>", :content_type => 'text/xml')
|
236
|
+
# server.response("<data>lol</data>", :content_type => 'text/xml')
|
253
237
|
#
|
254
238
|
def response(body='',options={},&block)
|
255
|
-
status = (
|
239
|
+
status = (
|
240
|
+
options.delete(:status) ||
|
241
|
+
options.delete('Status') ||
|
242
|
+
200
|
243
|
+
)
|
256
244
|
headers = {}
|
257
245
|
|
258
246
|
options.each do |name,value|
|
@@ -270,15 +258,19 @@ module Ronin
|
|
270
258
|
# Returns the server that handles requests for the specified host
|
271
259
|
# _name_.
|
272
260
|
#
|
273
|
-
def
|
261
|
+
def vhost(name)
|
274
262
|
name = name.to_s
|
275
263
|
|
276
|
-
|
277
|
-
|
264
|
+
@vhosts_mutex.synchronize do
|
265
|
+
if @vhosts.has_key?(name)
|
266
|
+
return @vhosts[name]
|
267
|
+
end
|
278
268
|
end
|
279
269
|
|
280
|
-
@
|
281
|
-
|
270
|
+
@vhost_patterns_mutex.synchronize do
|
271
|
+
@vhost_patterns.each do |pattern,server|
|
272
|
+
return server if name.match(pattern)
|
273
|
+
end
|
282
274
|
end
|
283
275
|
|
284
276
|
return nil
|
@@ -288,12 +280,15 @@ module Ronin
|
|
288
280
|
# Use the given _server_ or _block_ as the default route for all
|
289
281
|
# other requests.
|
290
282
|
#
|
291
|
-
# default do |request|
|
283
|
+
# server.default do |request|
|
292
284
|
# [200, {'Content-Type' => 'text/html'}, 'lol train']
|
293
285
|
# end
|
294
286
|
#
|
295
287
|
def default(server=nil,&block)
|
296
|
-
@
|
288
|
+
@default_mutex.synchronize do
|
289
|
+
@default = (server || block)
|
290
|
+
end
|
291
|
+
|
297
292
|
return self
|
298
293
|
end
|
299
294
|
|
@@ -301,26 +296,35 @@ module Ronin
|
|
301
296
|
# Registers the given _server_ or _block_ to be called when receiving
|
302
297
|
# requests to host names which match the specified _pattern_.
|
303
298
|
#
|
304
|
-
# hosts_like(/^a[0-9]\./) do
|
305
|
-
# map('/download/') do |request|
|
299
|
+
# server.hosts_like(/^a[0-9]\./) do |vhost|
|
300
|
+
# vhost.map('/download/') do |request|
|
306
301
|
# ...
|
307
302
|
# end
|
308
303
|
# end
|
309
304
|
#
|
310
305
|
def hosts_like(pattern,server=nil,&block)
|
311
|
-
|
306
|
+
server ||= self.class.new(&block)
|
307
|
+
|
308
|
+
@vhost_patterns_mutex.synchronize do
|
309
|
+
@vhost_patterns[pattern] = server
|
310
|
+
end
|
311
|
+
|
312
|
+
return server
|
312
313
|
end
|
313
314
|
|
314
315
|
#
|
315
316
|
# Registers the given _server_ or _block_ to be called when receiving
|
316
317
|
# requests for paths which match the specified _pattern_.
|
317
318
|
#
|
318
|
-
# paths_like(/\.xml$/) do |request|
|
319
|
+
# server.paths_like(/\.xml$/) do |request|
|
319
320
|
# ...
|
320
321
|
# end
|
321
322
|
#
|
322
323
|
def paths_like(pattern,server=nil,&block)
|
323
|
-
@
|
324
|
+
@patterns_mutex.synchronize do
|
325
|
+
@patterns[pattern] = (server || block)
|
326
|
+
end
|
327
|
+
|
324
328
|
return self
|
325
329
|
end
|
326
330
|
|
@@ -329,23 +333,32 @@ module Ronin
|
|
329
333
|
# connects it as a virtual host representing the specified host
|
330
334
|
# _name_.
|
331
335
|
#
|
332
|
-
# host('cdn.evil.com') do
|
336
|
+
# server.host('cdn.evil.com') do |server|
|
333
337
|
# ...
|
334
338
|
# end
|
335
339
|
#
|
336
340
|
def host(name,server=nil,&block)
|
337
|
-
|
341
|
+
server ||= self.class.new(&block)
|
342
|
+
|
343
|
+
@vhosts_mutex.synchronize do
|
344
|
+
@vhosts[name.to_s] = server
|
345
|
+
end
|
346
|
+
|
347
|
+
return server
|
338
348
|
end
|
339
349
|
|
340
350
|
#
|
341
351
|
# Binds the specified URL _path_ to the given _server_ or _block_.
|
342
352
|
#
|
343
|
-
# bind '/secrets.xml' do |request|
|
353
|
+
# server.bind '/secrets.xml' do |request|
|
344
354
|
# [200, {'Content-Type' => 'text/xml'}, "Made you look."]
|
345
355
|
# end
|
346
356
|
#
|
347
357
|
def bind(path,server=nil,&block)
|
348
|
-
@
|
358
|
+
@paths_mutex.synchronize do
|
359
|
+
@paths[path] = (server || block)
|
360
|
+
end
|
361
|
+
|
349
362
|
return self
|
350
363
|
end
|
351
364
|
|
@@ -353,15 +366,18 @@ module Ronin
|
|
353
366
|
# Binds the specified URL directory _path_ to the given
|
354
367
|
# _server_ or _block_.
|
355
368
|
#
|
356
|
-
# map '/downloads' do |request|
|
357
|
-
# response(
|
369
|
+
# server.map '/downloads' do |request|
|
370
|
+
# server.response(
|
358
371
|
# "Your somewhere inside the downloads directory",
|
359
372
|
# :content_type' => 'text/xml'
|
360
373
|
# )
|
361
374
|
# end
|
362
375
|
#
|
363
376
|
def map(path,server=nil,&block)
|
364
|
-
@
|
377
|
+
@directories_mutex.synchronize do
|
378
|
+
@directories[path] = (server || block)
|
379
|
+
end
|
380
|
+
|
365
381
|
return self
|
366
382
|
end
|
367
383
|
|
@@ -369,15 +385,19 @@ module Ronin
|
|
369
385
|
# Binds the contents of the specified _file_ to the specified URL
|
370
386
|
# _path_, using the given _options_.
|
371
387
|
#
|
372
|
-
#
|
388
|
+
# _options_ may contain the following keys:
|
389
|
+
# <tt>content_type</tt>:: The content-type to use when serving
|
390
|
+
# the file at the specified _path_.
|
391
|
+
#
|
392
|
+
# server.file '/robots.txt', '/path/to/my_robots.txt'
|
373
393
|
#
|
374
394
|
def file(path,file,options={})
|
375
395
|
file = File.expand_path(file)
|
376
|
-
content_type =
|
396
|
+
content_type = options[:content_type]
|
377
397
|
|
378
398
|
bind(path) do |request|
|
379
399
|
if File.file?(file)
|
380
|
-
return_file(file,request)
|
400
|
+
return_file(file,request,content_type)
|
381
401
|
else
|
382
402
|
not_found(request)
|
383
403
|
end
|
@@ -388,7 +408,7 @@ module Ronin
|
|
388
408
|
# Binds the contents of the specified _directory_ to the given
|
389
409
|
# prefix _path_.
|
390
410
|
#
|
391
|
-
# directory '/download/', '/tmp/files/'
|
411
|
+
# server.directory '/download/', '/tmp/files/'
|
392
412
|
#
|
393
413
|
def directory(path,directory)
|
394
414
|
sub_dirs = path.split('/')
|
@@ -406,10 +426,24 @@ module Ronin
|
|
406
426
|
end
|
407
427
|
|
408
428
|
#
|
409
|
-
# Starts the server.
|
429
|
+
# Starts the server. Mongrel will be used to run the server, if it
|
430
|
+
# is installed, otherwise WEBrick will be used.
|
410
431
|
#
|
411
432
|
def start
|
412
|
-
|
433
|
+
rack_handler = [@handler, 'Mongrel', 'WEBrick'].find do |name|
|
434
|
+
name && Rack::Handler.const_defined?(name)
|
435
|
+
end
|
436
|
+
|
437
|
+
unless rack_handler
|
438
|
+
raise(StandardError,"unable to find any Rack handlers",caller)
|
439
|
+
end
|
440
|
+
|
441
|
+
rack_options = {
|
442
|
+
'Host' => (@host || Server.default_host),
|
443
|
+
'Port' => (@port || Server.default_port)
|
444
|
+
}
|
445
|
+
|
446
|
+
Rack::Handler.const_get(rack_handler).run(self,rack_options)
|
413
447
|
return self
|
414
448
|
end
|
415
449
|
|
@@ -422,36 +456,48 @@ module Ronin
|
|
422
456
|
request = Rack::Request.new(env)
|
423
457
|
|
424
458
|
if http_host
|
425
|
-
if (server =
|
459
|
+
if (server = vhost(http_host))
|
426
460
|
return server.call(env)
|
427
461
|
end
|
428
462
|
end
|
429
463
|
|
430
464
|
if http_path
|
431
|
-
|
432
|
-
|
465
|
+
@paths_mutex.synchronize do
|
466
|
+
if (block = @paths[http_path])
|
467
|
+
return block.call(request)
|
468
|
+
end
|
433
469
|
end
|
434
470
|
|
435
|
-
@
|
436
|
-
|
437
|
-
|
471
|
+
@patterns_mutex.synchronize do
|
472
|
+
@patterns.each do |pattern,block|
|
473
|
+
if http_path.match(pattern)
|
474
|
+
return block.call(request)
|
475
|
+
end
|
438
476
|
end
|
439
477
|
end
|
440
478
|
|
441
479
|
http_dirs = http_path.split('/')
|
442
480
|
|
443
|
-
|
444
|
-
|
481
|
+
@directories_mutex.synchronize do
|
482
|
+
sub_dir = @directories.keys.select { |path|
|
483
|
+
dirs = path.split('/')
|
445
484
|
|
446
|
-
|
447
|
-
|
485
|
+
http_dirs[0...dirs.length] == dirs
|
486
|
+
}.sort.last
|
448
487
|
|
449
|
-
|
450
|
-
|
488
|
+
if (sub_dir && (block = @directories[sub_dir]))
|
489
|
+
return block.call(request)
|
490
|
+
end
|
451
491
|
end
|
452
492
|
end
|
453
493
|
|
454
|
-
|
494
|
+
resp = nil
|
495
|
+
|
496
|
+
@default_mutex.synchronize do
|
497
|
+
resp = @default.call(request)
|
498
|
+
end
|
499
|
+
|
500
|
+
return resp
|
455
501
|
end
|
456
502
|
|
457
503
|
#
|
data/lib/ronin/web/spider.rb
CHANGED
@@ -43,8 +43,8 @@ module Ronin
|
|
43
43
|
# <tt>:user_agent</tt>:: The User-Agent string to send. Defaults to
|
44
44
|
# Web.user_agent.
|
45
45
|
# <tt>:referer</tt>:: The referer URL to send.
|
46
|
-
# <tt>:delay</tt>:: Duration in seconds to pause between spidering
|
47
|
-
# link. Defaults to 0.
|
46
|
+
# <tt>:delay</tt>:: Duration in seconds to pause between spidering
|
47
|
+
# each link. Defaults to 0.
|
48
48
|
# <tt>:host</tt>:: The host-name to visit.
|
49
49
|
# <tt>:hosts</tt>:: An +Array+ of host patterns to visit.
|
50
50
|
# <tt>:ignore_hosts</tt>:: An +Array+ of host patterns to not visit.
|
@@ -57,7 +57,12 @@ module Ronin
|
|
57
57
|
# visit.
|
58
58
|
#
|
59
59
|
def initialize(options={},&block)
|
60
|
-
|
60
|
+
options = {
|
61
|
+
:proxy => Web.proxy,
|
62
|
+
:user_agent => Web.user_agent
|
63
|
+
}.merge(options)
|
64
|
+
|
65
|
+
super(options)
|
61
66
|
|
62
67
|
every_url do |url|
|
63
68
|
print_info("Spidering #{url}")
|
@@ -66,14 +71,6 @@ module Ronin
|
|
66
71
|
block.call(self) if block
|
67
72
|
end
|
68
73
|
|
69
|
-
protected
|
70
|
-
|
71
|
-
#
|
72
|
-
# Returns the default options for Spider.
|
73
|
-
#
|
74
|
-
def default_options
|
75
|
-
{:proxy => Web.proxy, :user_agent => Web.user_agent}
|
76
|
-
end
|
77
74
|
end
|
78
75
|
end
|
79
76
|
end
|
data/lib/ronin/web/version.rb
CHANGED
data/lib/ronin/web/web.rb
CHANGED
@@ -34,16 +34,51 @@ module Ronin
|
|
34
34
|
# Returns a Nokogiri::HTML::Document object for the specified _body_
|
35
35
|
# of html.
|
36
36
|
#
|
37
|
-
def Web.html(body)
|
38
|
-
Nokogiri::HTML(body)
|
37
|
+
def Web.html(body,&block)
|
38
|
+
doc = Nokogiri::HTML(body)
|
39
|
+
|
40
|
+
block.call(doc) if block
|
41
|
+
return doc
|
42
|
+
end
|
43
|
+
|
44
|
+
#
|
45
|
+
# Creates a new Nokogiri::HTML::Builder with the given _block_.
|
46
|
+
#
|
47
|
+
# Web.build_html do
|
48
|
+
# body {
|
49
|
+
# div(:style => 'display:none;') {
|
50
|
+
# object(:classid => 'blabla')
|
51
|
+
# }
|
52
|
+
# }
|
53
|
+
# end
|
54
|
+
#
|
55
|
+
def Web.build_html(&block)
|
56
|
+
Nokogiri::HTML::Builder.new(&block)
|
39
57
|
end
|
40
58
|
|
41
59
|
#
|
42
60
|
# Returns a Nokogiri::XML::Document object for the specified _body_
|
43
|
-
# of xml.
|
61
|
+
# of xml and the given _block_.
|
44
62
|
#
|
45
|
-
def Web.xml(body)
|
46
|
-
Nokogiri::XML(body)
|
63
|
+
def Web.xml(body,&block)
|
64
|
+
doc = Nokogiri::XML(body)
|
65
|
+
|
66
|
+
block.call(doc) if block
|
67
|
+
return doc
|
68
|
+
end
|
69
|
+
|
70
|
+
#
|
71
|
+
# Creates a new Nokogiri::XML::Builder with the given _block_.
|
72
|
+
#
|
73
|
+
# Web.build_xml do
|
74
|
+
# post(:id => 2) {
|
75
|
+
# title { text("some example) }
|
76
|
+
# body { text("this is one contrived example.") }
|
77
|
+
# }
|
78
|
+
# end
|
79
|
+
#
|
80
|
+
def Web.build_xml(&block)
|
81
|
+
Nokogiri::XML::Builder.new(&block)
|
47
82
|
end
|
48
83
|
|
49
84
|
#
|
@@ -272,7 +307,9 @@ module Ronin
|
|
272
307
|
# Web.post('http://www.rubyinside.com') # => WWW::Mechanize::Page
|
273
308
|
#
|
274
309
|
def Web.post(url,options={},&block)
|
275
|
-
query =
|
310
|
+
query = {}
|
311
|
+
query.merge!(options[:query]) if options[:query]
|
312
|
+
|
276
313
|
page = Web.agent(options).post(url,query)
|
277
314
|
|
278
315
|
block.call(page) if block
|
data/spec/web/server_spec.rb
CHANGED
@@ -5,43 +5,43 @@ require 'web/helpers/server'
|
|
5
5
|
|
6
6
|
describe Web::Server do
|
7
7
|
before(:all) do
|
8
|
-
@server = Web::Server.new do
|
9
|
-
default do |env|
|
10
|
-
response('This is default.')
|
8
|
+
@server = Web::Server.new do |server|
|
9
|
+
server.default do |env|
|
10
|
+
server.response('This is default.')
|
11
11
|
end
|
12
12
|
|
13
|
-
bind('/test/bind.xml') do |env|
|
14
|
-
response('<secret/>', :content_type => 'text/xml')
|
13
|
+
server.bind('/test/bind.xml') do |env|
|
14
|
+
server.response('<secret/>', :content_type => 'text/xml')
|
15
15
|
end
|
16
16
|
|
17
|
-
paths_like(/path_patterns\/secret\./) do |env|
|
18
|
-
response('No secrets here.')
|
17
|
+
server.paths_like(/path_patterns\/secret\./) do |env|
|
18
|
+
server.response('No secrets here.')
|
19
19
|
end
|
20
20
|
|
21
|
-
map('/test/map') do |env|
|
22
|
-
response('mapped')
|
21
|
+
server.map('/test/map') do |env|
|
22
|
+
server.response('mapped')
|
23
23
|
end
|
24
24
|
|
25
|
-
file('/test/file.txt',File.join(WEB_SERVER_ROOT,'test.txt'))
|
25
|
+
server.file('/test/file.txt',File.join(WEB_SERVER_ROOT,'test.txt'))
|
26
26
|
|
27
|
-
directory('/test/directory/',WEB_SERVER_ROOT)
|
27
|
+
server.directory('/test/directory/',WEB_SERVER_ROOT)
|
28
28
|
end
|
29
29
|
|
30
|
-
@
|
31
|
-
bind('/test/virtual_host.xml') do |env|
|
32
|
-
response('<virtual/>', :content_type => 'text/xml')
|
30
|
+
@vhost = Web::Server.new do |vhost|
|
31
|
+
vhost.bind('/test/virtual_host.xml') do |env|
|
32
|
+
vhost.response('<virtual/>', :content_type => 'text/xml')
|
33
33
|
end
|
34
34
|
end
|
35
35
|
|
36
|
-
@server.host('virtual.host.com') do
|
37
|
-
bind('/test/virtual_host.xml') do |env|
|
38
|
-
response('<virtual/>', :content_type => 'text/xml')
|
36
|
+
@server.host('virtual.host.com') do |vhost|
|
37
|
+
vhost.bind('/test/virtual_host.xml') do |env|
|
38
|
+
vhost.response('<virtual/>', :content_type => 'text/xml')
|
39
39
|
end
|
40
40
|
end
|
41
41
|
|
42
|
-
@server.hosts_like(/^virtual[0-9]\./) do
|
43
|
-
bind('/test/virtual_host_patterns.xml') do |env|
|
44
|
-
response('<virtual-patterns/>', :content_type => 'text/xml')
|
42
|
+
@server.hosts_like(/^virtual[0-9]\./) do |vhost|
|
43
|
+
vhost.bind('/test/virtual_host_patterns.xml') do |env|
|
44
|
+
vhost.response('<virtual-patterns/>', :content_type => 'text/xml')
|
45
45
|
end
|
46
46
|
end
|
47
47
|
end
|
@@ -127,14 +127,14 @@ describe Web::Server do
|
|
127
127
|
end
|
128
128
|
|
129
129
|
it "should provide access to servers via their host-names" do
|
130
|
-
virtual_host = @server.
|
130
|
+
virtual_host = @server.vhost('virtual.host.com')
|
131
131
|
url = 'http://virtual.host.com/test/virtual_host.xml'
|
132
132
|
|
133
133
|
get_url(virtual_host,url).body.should == ['<virtual/>']
|
134
134
|
end
|
135
135
|
|
136
136
|
it "should provide access to servers via their host-names that match virtual host patterns" do
|
137
|
-
virtual_host = @server.
|
137
|
+
virtual_host = @server.vhost('virtual1.host.com')
|
138
138
|
url = 'http://virtual0.host.com/test/virtual_host_patterns.xml'
|
139
139
|
|
140
140
|
get_url(virtual_host,url).body.should == ['<virtual-patterns/>']
|
metadata
CHANGED
@@ -1,27 +1,38 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ronin-web
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Postmodern
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
|
-
cert_chain:
|
10
|
+
cert_chain:
|
11
|
+
- |
|
12
|
+
-----BEGIN CERTIFICATE-----
|
13
|
+
MIIDQDCCAiigAwIBAgIBADANBgkqhkiG9w0BAQUFADBGMRgwFgYDVQQDDA9wb3N0
|
14
|
+
bW9kZXJuLm1vZDMxFTATBgoJkiaJk/IsZAEZFgVnbWFpbDETMBEGCgmSJomT8ixk
|
15
|
+
ARkWA2NvbTAeFw0wOTA2MDMwNDU5MDNaFw0xMDA2MDMwNDU5MDNaMEYxGDAWBgNV
|
16
|
+
BAMMD3Bvc3Rtb2Rlcm4ubW9kMzEVMBMGCgmSJomT8ixkARkWBWdtYWlsMRMwEQYK
|
17
|
+
CZImiZPyLGQBGRYDY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
|
18
|
+
1wvANkTDHFgVih5XLjuTwTZjgBq1lBGybXJiH6Id1lY2JOMqM5FB1DDHVvvij94i
|
19
|
+
mJabN0zkzu6VKWC70y0IwOxY7CPokr0eFdK/D0y7mCq1P8QITv76i2YqAl0eYqIt
|
20
|
+
W+IhIkANQ7E6uMZIZcdnfadC6lPAtlKkqtd9crvRbFgr6e3kyflmohbRnTEJHoRd
|
21
|
+
7SHHsybE6DSn7oTDs6XBTNrNIn5VfZA0z01eeos/+zBm1zKJOK2+/7xtLLDuDU9G
|
22
|
+
+Rd+ltUBbvxUrMNZmDG29pnmN2xTRH+Q8HxD2AxlvM5SRpK6OeZaHV7PaCCAVZ4L
|
23
|
+
T9BFl1sfMvRlABeGEkSyuQIDAQABozkwNzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIE
|
24
|
+
sDAdBgNVHQ4EFgQUKwsd+PqEYmBvyaTyoL+uRuk+PhEwDQYJKoZIhvcNAQEFBQAD
|
25
|
+
ggEBAB4TvHsrlbcXcKg6gX5BIb9tI+zGkpzo0Z7jnxMEcNO7NGGwmzafDBI/xZYv
|
26
|
+
xkRH3/HXbGGYDOi6Q6gWt5GujSx0bOImDtYTJTH8jnzN92HzEK5WdScm1QpZKF1e
|
27
|
+
cezArMbxbSPaosxTCtG6LQTkE28lFQsmFZ5xzouugS4h5+LVJiVMmiP+l3EfkjFa
|
28
|
+
GOURU+rNEMPWo8MCWivGW7jes6BMzWHcW7DQ0scNVmIcCIgdyMmpscuAEOSeghy9
|
29
|
+
/fFs57Ey2OXBL55nDOyvN/ZQ2Vab05UH4t+GCxjAPeirzL/29FBtePT6VD44c38j
|
30
|
+
pDj+ws7QjtH/Qcrr1l9jfN0ehDs=
|
31
|
+
-----END CERTIFICATE-----
|
11
32
|
|
12
|
-
date: 2009-
|
33
|
+
date: 2009-07-08 00:00:00 -07:00
|
13
34
|
default_executable:
|
14
35
|
dependencies:
|
15
|
-
- !ruby/object:Gem::Dependency
|
16
|
-
name: nokogiri
|
17
|
-
type: :runtime
|
18
|
-
version_requirement:
|
19
|
-
version_requirements: !ruby/object:Gem::Requirement
|
20
|
-
requirements:
|
21
|
-
- - ">="
|
22
|
-
- !ruby/object:Gem::Version
|
23
|
-
version: 1.2.0
|
24
|
-
version:
|
25
36
|
- !ruby/object:Gem::Dependency
|
26
37
|
name: mechanize
|
27
38
|
type: :runtime
|
@@ -40,7 +51,7 @@ dependencies:
|
|
40
51
|
requirements:
|
41
52
|
- - ">="
|
42
53
|
- !ruby/object:Gem::Version
|
43
|
-
version: 0.1.
|
54
|
+
version: 0.1.9
|
44
55
|
version:
|
45
56
|
- !ruby/object:Gem::Dependency
|
46
57
|
name: rack
|
@@ -50,7 +61,7 @@ dependencies:
|
|
50
61
|
requirements:
|
51
62
|
- - ">="
|
52
63
|
- !ruby/object:Gem::Version
|
53
|
-
version: 0.
|
64
|
+
version: 1.0.0
|
54
65
|
version:
|
55
66
|
- !ruby/object:Gem::Dependency
|
56
67
|
name: ronin
|
@@ -60,7 +71,7 @@ dependencies:
|
|
60
71
|
requirements:
|
61
72
|
- - ">="
|
62
73
|
- !ruby/object:Gem::Version
|
63
|
-
version: 0.2.
|
74
|
+
version: 0.2.4
|
64
75
|
version:
|
65
76
|
- !ruby/object:Gem::Dependency
|
66
77
|
name: hoe
|
@@ -70,9 +81,15 @@ dependencies:
|
|
70
81
|
requirements:
|
71
82
|
- - ">="
|
72
83
|
- !ruby/object:Gem::Version
|
73
|
-
version:
|
84
|
+
version: 2.3.2
|
74
85
|
version:
|
75
|
-
description:
|
86
|
+
description: |-
|
87
|
+
Ronin Web is a Ruby library for Ronin that provides support for web
|
88
|
+
scraping and spidering functionality.
|
89
|
+
|
90
|
+
Ronin is a Ruby platform designed for information security and data
|
91
|
+
exploration tasks. Ronin allows for the rapid development and distribution
|
92
|
+
of code over many of the common Source-Code-Management (SCM) systems.
|
76
93
|
email:
|
77
94
|
- postmodern.mod3@gmail.com
|
78
95
|
executables:
|
@@ -83,10 +100,12 @@ extra_rdoc_files:
|
|
83
100
|
- History.txt
|
84
101
|
- Manifest.txt
|
85
102
|
- README.txt
|
103
|
+
- TODO.txt
|
86
104
|
files:
|
87
105
|
- History.txt
|
88
106
|
- Manifest.txt
|
89
107
|
- README.txt
|
108
|
+
- TODO.txt
|
90
109
|
- Rakefile
|
91
110
|
- bin/ronin-web
|
92
111
|
- lib/ronin/sessions/web.rb
|
@@ -99,6 +118,7 @@ files:
|
|
99
118
|
- lib/ronin/web/extensions/nokogiri/xml/attr.rb
|
100
119
|
- lib/ronin/web/extensions/nokogiri/xml/element.rb
|
101
120
|
- lib/ronin/web/extensions/nokogiri/xml/document.rb
|
121
|
+
- lib/ronin/web/fingerprint.rb
|
102
122
|
- lib/ronin/web/server.rb
|
103
123
|
- lib/ronin/web/proxy.rb
|
104
124
|
- lib/ronin/web/spider.rb
|
@@ -113,6 +133,8 @@ files:
|
|
113
133
|
- spec/web/server_spec.rb
|
114
134
|
has_rdoc: true
|
115
135
|
homepage: http://ronin.rubyforge.org/web/
|
136
|
+
licenses: []
|
137
|
+
|
116
138
|
post_install_message:
|
117
139
|
rdoc_options:
|
118
140
|
- --main
|
@@ -134,9 +156,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
134
156
|
requirements: []
|
135
157
|
|
136
158
|
rubyforge_project: ronin
|
137
|
-
rubygems_version: 1.3.
|
159
|
+
rubygems_version: 1.3.4
|
138
160
|
signing_key:
|
139
|
-
specification_version:
|
161
|
+
specification_version: 3
|
140
162
|
summary: Ronin Web is a Ruby library for Ronin that provides support for web scraping and spidering functionality
|
141
163
|
test_files: []
|
142
164
|
|
metadata.gz.sig
ADDED
Binary file
|