ronin-web 0.1.2 → 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
data.tar.gz.sig ADDED
Binary file
data/History.txt CHANGED
@@ -1,3 +1,21 @@
1
+ === 0.1.3 / 2009-07-02
2
+
3
+ * Use Hoe >= 2.0.0.
4
+ * Require spidr >= 0.1.9.
5
+ * Require rack >= 1.0.0.
6
+ * Require ronin >= 0.2.4.
7
+ * Added Ronin::Web::Fingerprint.
8
+ * Added Web.build_html.
9
+ * Added Web.build_xml.
10
+ * Allow Web.html to accept a block.
11
+ * Allow Web.xml to accept a block.
12
+ * Allow Server#return_file to accept a content_type option.
13
+ * Make sure Ronin::Web::Server is thread safe.
14
+ * Blocks passed to Server.new will not be instance_evaled.
15
+ * Removed Spider#default_options.
16
+ * Removed Server.run.
17
+ * Removed Server#config.
18
+
1
19
  === 0.1.2 / 2009-03-28
2
20
 
3
21
  * Added Ronin::Web::Proxy.
data/Manifest.txt CHANGED
@@ -1,6 +1,7 @@
1
1
  History.txt
2
2
  Manifest.txt
3
3
  README.txt
4
+ TODO.txt
4
5
  Rakefile
5
6
  bin/ronin-web
6
7
  lib/ronin/sessions/web.rb
@@ -13,6 +14,7 @@ lib/ronin/web/extensions/nokogiri/xml/text.rb
13
14
  lib/ronin/web/extensions/nokogiri/xml/attr.rb
14
15
  lib/ronin/web/extensions/nokogiri/xml/element.rb
15
16
  lib/ronin/web/extensions/nokogiri/xml/document.rb
17
+ lib/ronin/web/fingerprint.rb
16
18
  lib/ronin/web/server.rb
17
19
  lib/ronin/web/proxy.rb
18
20
  lib/ronin/web/spider.rb
data/README.txt CHANGED
@@ -61,7 +61,100 @@ of Ronin.
61
61
 
62
62
  * Start the Ronin console with Ronin Web preloaded:
63
63
 
64
- $ ronin-web
64
+ $ ronin-web
65
+
66
+ == EXAMPLES:
67
+
68
+ * Get a web-page:
69
+
70
+ Web.get('http://www.example.com/')
71
+
72
+ * Get only the body of the web-page:
73
+
74
+ Web.get_body('http://www.example.com/')
75
+
76
+ * Get a WWW::Mechanize agent:
77
+
78
+ agent = Web.agent
79
+
80
+ * Parse HTML:
81
+
82
+ Web.html(open('some_file.html'))
83
+ # => <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
84
+ <html>
85
+ <head>
86
+ <script type="text/javascript" src="redirect.js"></script>
87
+ </head>
88
+ </html>
89
+
90
+ * Build a HTML document:
91
+
92
+ doc = Web.build_html do
93
+ html {
94
+ head {
95
+ script(:type => 'text/javascript', :src => 'redirect.js')
96
+ }
97
+ }
98
+ end
99
+
100
+ puts doc.to_html
101
+ <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
102
+ <html><head><script src="redirect.js" type="text/javascript"></script></head></html>
103
+
104
+ * Parse XML:
105
+
106
+ Web.xml(some_text)
107
+ => <?xml version="1.0"?>
108
+ <users>
109
+ <user>
110
+ <name>admin</name>
111
+ <password>0mni</password>
112
+ </user>
113
+ </users>
114
+
115
+
116
+ * Build a XML document:
117
+
118
+ doc = Web.build_xml do
119
+ playlist {
120
+ mp3 {
121
+ file { text('02 THE WAIT.mp3') }
122
+ artist { text('Evil Nine') }
123
+ track { text('The Wait feat David Autokratz') }
124
+ duration { text('1000000000') }
125
+ }
126
+ }
127
+ end
128
+
129
+ puts doc.to_xml
130
+ <?xml version="1.0"?>
131
+ <playlist>
132
+ <mp3>
133
+ <file>02 THE WAIT.mp3</file>
134
+ <artist>Evil Nine</artist>
135
+ <track>The Wait feat David Autokratz</track>
136
+ <duration>1000000000</duration>
137
+ </mp3>
138
+ </playlist>
139
+
140
+ * Spider a web site:
141
+
142
+ Web::Spider.host('www.example.com') do |spider|
143
+ spider.every_url do |url|
144
+ ...
145
+ end
146
+
147
+ spider.every_page do |page|
148
+ ...
149
+ end
150
+ end
151
+
152
+ * Serve files via a Web Server:
153
+
154
+ Web::Server.start do |server|
155
+ server.file '/opensearch.xml', '/tmp/test.xml'
156
+ server.directory '/download/', '/tmp/download/'
157
+ end
65
158
 
66
159
  == LICENSE:
67
160
 
data/Rakefile CHANGED
@@ -2,19 +2,18 @@
2
2
 
3
3
  require 'rubygems'
4
4
  require 'hoe'
5
+ require 'hoe/signing'
5
6
  require './tasks/spec.rb'
6
- require './lib/ronin/web/version.rb'
7
7
 
8
- Hoe.new('ronin-web', Ronin::Web::VERSION) do |p|
9
- p.rubyforge_name = 'ronin'
10
- p.developer('Postmodern', 'postmodern.mod3@gmail.com')
11
- p.remote_rdoc_dir = 'docs/ronin-web'
12
- p.extra_deps = [
13
- ['nokogiri', '>=1.2.0'],
8
+ Hoe.spec('ronin-web') do
9
+ self.rubyforge_name = 'ronin'
10
+ self.developer('Postmodern', 'postmodern.mod3@gmail.com')
11
+ self.remote_rdoc_dir = 'docs/ronin-web'
12
+ self.extra_deps = [
14
13
  ['mechanize', '>=0.9.0'],
15
- ['spidr', '>=0.1.3'],
16
- ['rack', '>=0.9.1'],
17
- ['ronin', '>=0.2.2']
14
+ ['spidr', '>=0.1.9'],
15
+ ['rack', '>=1.0.0'],
16
+ ['ronin', '>=0.2.4']
18
17
  ]
19
18
  end
20
19
 
data/TODO.txt ADDED
@@ -0,0 +1,7 @@
1
+ == TODO:
2
+
3
+ === Ronin Web 0.1.3:
4
+
5
+ * Add configurable hooks to Web::Spider:
6
+ * Leverage for SQL Injection, LFI and RFI scanning.
7
+
data/lib/ronin/web.rb CHANGED
@@ -25,3 +25,9 @@ require 'ronin/web/extensions'
25
25
  require 'ronin/web/web'
26
26
  require 'ronin/web/spider'
27
27
  require 'ronin/web/server'
28
+ require 'ronin/web/version'
29
+ require 'ronin/config'
30
+
31
+ module Ronin
32
+ Config.load :web
33
+ end
@@ -0,0 +1,76 @@
1
+ #
2
+ #--
3
+ # Ronin Web - A Ruby library for Ronin that provides support for web
4
+ # scraping and spidering functionality.
5
+ #
6
+ # Copyright (c) 2006-2009 Hal Brodigan (postmodern.mod3 at gmail.com)
7
+ #
8
+ # This program is free software; you can redistribute it and/or modify
9
+ # it under the terms of the GNU General Public License as published by
10
+ # the Free Software Foundation; either version 2 of the License, or
11
+ # (at your option) any later version.
12
+ #
13
+ # This program is distributed in the hope that it will be useful,
14
+ # but WITHOUT ANY WARRANTY; without even the implied warranty of
15
+ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16
+ # GNU General Public License for more details.
17
+ #
18
+ # You should have received a copy of the GNU General Public License
19
+ # along with this program; if not, write to the Free Software
20
+ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
21
+ #++
22
+ #
23
+
24
+ module Ronin
25
+ module Web
26
+ module Fingerprint
27
+ #
28
+ # The Hash of web application identities and their associated
29
+ # fingerprint tests.
30
+ #
31
+ def Fingerprint.tests
32
+ @@ronin_web_fingerprints ||= Hash.new do |hash,key|
33
+ hash[key] ||= []
34
+ end
35
+ end
36
+
37
+ #
38
+ # Adds a test for the web application identity with the specified
39
+ # _name_ and _block_. When the _block_ is called, it will be passed
40
+ # the URL of the unknown web application.
41
+ #
42
+ # Fingerprint.test_for('app') do |url|
43
+ # url.path.include?('/app/')
44
+ # end
45
+ #
46
+ def Fingerprint.test_for(name,&block)
47
+ Fingerprint.tests[name.to_sym] << block
48
+ return nil
49
+ end
50
+
51
+ #
52
+ # Identifies the web application represented by the specified _url_,
53
+ # returning the name of identified web application. If the
54
+ # web application cannot be identified, +nil+ will be returned.
55
+ #
56
+ def Fingerprint.identify(url)
57
+ unless url.kind_of?(URI)
58
+ url = URI(url.to_s)
59
+ end
60
+
61
+ matches = []
62
+
63
+ Fingerprint.tests.each do |name,blocks|
64
+ blocks.each do |block|
65
+ if block.call(url)
66
+ matches << name
67
+ break
68
+ end
69
+ end
70
+ end
71
+
72
+ return matches
73
+ end
74
+ end
75
+ end
76
+ end
@@ -23,9 +23,12 @@
23
23
 
24
24
  require 'uri'
25
25
  require 'cgi'
26
+ require 'thread'
26
27
 
27
28
  begin
28
29
  require 'mongrel'
30
+ rescue Gem::LoadError => e
31
+ raise(e)
29
32
  rescue LoadError
30
33
  require 'webrick'
31
34
  end
@@ -51,8 +54,8 @@ module Ronin
51
54
  # The port to listen on
52
55
  attr_accessor :port
53
56
 
54
- # The Hash of configurable variables for the server
55
- attr_reader :config
57
+ # The handler to run the server under
58
+ attr_accessor :handler
56
59
 
57
60
  #
58
61
  # Creates a new Web Server using the given configuration _block_.
@@ -60,28 +63,30 @@ module Ronin
60
63
  # _options_ may contain the following keys:
61
64
  # <tt>:host</tt>:: The host to bind to.
62
65
  # <tt>:port</tt>:: The port to listen on.
63
- # <tt>:config</tt>:: A +Hash+ of configurable variables to be used
64
- # in responses.
66
+ # <tt>:handler</tt>:: The handler to run the server under.
65
67
  #
66
68
  def initialize(options={},&block)
67
69
  @host = options[:host]
68
70
  @port = options[:port]
69
- @config = {}
70
-
71
- if options.has_key?(:config)
72
- @config.merge!(options[:config])
73
- end
71
+ @handler = options[:handler]
74
72
 
75
73
  @default = method(:not_found)
76
74
 
77
- @virtual_host_patterns = {}
78
- @virtual_hosts = {}
75
+ @vhost_patterns = {}
76
+ @vhosts = {}
79
77
 
80
- @path_patterns = {}
78
+ @patterns = {}
81
79
  @paths = {}
82
80
  @directories = {}
83
81
 
84
- instance_eval(&block) if block
82
+ @default_mutex = Mutex.new
83
+ @vhost_patterns_mutex = Mutex.new
84
+ @vhosts_mutex = Mutex.new
85
+ @patterns_mutex = Mutex.new
86
+ @paths_mutex = Mutex.new
87
+ @directories_mutex = Mutex.new
88
+
89
+ block.call(self) if block
85
90
  end
86
91
 
87
92
  #
@@ -127,36 +132,12 @@ module Ronin
127
132
  #
128
133
  # Server.content_type 'text/xml', ['xml', 'xsl']
129
134
  #
130
- def self.content_type(type,extensions)
135
+ def Server.content_type(type,extensions)
131
136
  extensions.each { |ext| Server.content_types[ext] = type }
132
137
 
133
138
  return self
134
139
  end
135
140
 
136
- #
137
- # Runs the specified _server_ with the given _options_. Server.run
138
- # will use Mongrel to run the _server_, if it is installed. Otherwise
139
- # WEBrick will be used to run the _server_.
140
- #
141
- # _options_ can contain the following keys:
142
- # <tt>:host</tt>:: The host the server will bind to, defaults to
143
- # Server.default_host.
144
- # <tt>:port</tt>:: The port the server will listen on, defaults to
145
- # Server.default_port.
146
- #
147
- def Server.run(server,options={})
148
- rack_options = {}
149
-
150
- rack_options[:Host] = (options[:host] || Server.default_host)
151
- rack_options[:Port] = (options[:port] || Server.default_port)
152
-
153
- if Object.const_defined?('Mongrel')
154
- Rack::Handler::Mongrel.run(server,rack_options)
155
- else
156
- Rack::Handler::WEBrick.run(server,rack_options)
157
- end
158
- end
159
-
160
141
  #
161
142
  # Creates a new Web Server object with the given _block_ and starts
162
143
  # it using the given _options_.
@@ -168,17 +149,18 @@ module Ronin
168
149
  #
169
150
  # Returns the HTTP Content-Type for the specified file _extension_.
170
151
  #
171
- # content_type('html')
152
+ # server.content_type('html')
172
153
  # # => "text/html"
173
154
  #
174
155
  def content_type(extension)
175
- Server.content_types[extension] || 'application/x-unknown-content-type'
156
+ Server.content_types[extension] ||
157
+ 'application/x-unknown-content-type'
176
158
  end
177
159
 
178
160
  #
179
161
  # Returns the HTTP Content-Type for the specified _file_.
180
162
  #
181
- # srv.content_type_for('file.html')
163
+ # server.content_type_for('file.html')
182
164
  # # => "text/html"
183
165
  #
184
166
  def content_type_for(file)
@@ -228,7 +210,9 @@ module Ronin
228
210
  # an index file. If no index file can be found or _path_ points to a
229
211
  # non-existant file, a "404 Not Found" response will be returned.
230
212
  #
231
- def return_file(path,request)
213
+ def return_file(path,request,content_type=nil)
214
+ content_type ||= content_type_for(path)
215
+
232
216
  if !(File.exists?(path))
233
217
  return not_found(request)
234
218
  end
@@ -239,7 +223,7 @@ module Ronin
239
223
  end
240
224
  end
241
225
 
242
- return response(File.new(path), :content_type => content_type_for(path))
226
+ return response(File.new(path), :content_type => content_type)
243
227
  end
244
228
 
245
229
  #
@@ -249,10 +233,14 @@ module Ronin
249
233
  # _options_ may include the following keys:
250
234
  # <tt>:status</tt>:: The HTTP Response status code, defaults to 200.
251
235
  #
252
- # response("<data>lol</data>", :content_type => 'text/xml')
236
+ # server.response("<data>lol</data>", :content_type => 'text/xml')
253
237
  #
254
238
  def response(body='',options={},&block)
255
- status = (options.delete(:status) || options.delete('Status') || 200)
239
+ status = (
240
+ options.delete(:status) ||
241
+ options.delete('Status') ||
242
+ 200
243
+ )
256
244
  headers = {}
257
245
 
258
246
  options.each do |name,value|
@@ -270,15 +258,19 @@ module Ronin
270
258
  # Returns the server that handles requests for the specified host
271
259
  # _name_.
272
260
  #
273
- def virtual_host(name)
261
+ def vhost(name)
274
262
  name = name.to_s
275
263
 
276
- if @virtual_hosts.has_key?(name)
277
- return @virtual_hosts[name]
264
+ @vhosts_mutex.synchronize do
265
+ if @vhosts.has_key?(name)
266
+ return @vhosts[name]
267
+ end
278
268
  end
279
269
 
280
- @virtual_host_patterns.each do |pattern,server|
281
- return server if name.match(pattern)
270
+ @vhost_patterns_mutex.synchronize do
271
+ @vhost_patterns.each do |pattern,server|
272
+ return server if name.match(pattern)
273
+ end
282
274
  end
283
275
 
284
276
  return nil
@@ -288,12 +280,15 @@ module Ronin
288
280
  # Use the given _server_ or _block_ as the default route for all
289
281
  # other requests.
290
282
  #
291
- # default do |request|
283
+ # server.default do |request|
292
284
  # [200, {'Content-Type' => 'text/html'}, 'lol train']
293
285
  # end
294
286
  #
295
287
  def default(server=nil,&block)
296
- @default = (server || block)
288
+ @default_mutex.synchronize do
289
+ @default = (server || block)
290
+ end
291
+
297
292
  return self
298
293
  end
299
294
 
@@ -301,26 +296,35 @@ module Ronin
301
296
  # Registers the given _server_ or _block_ to be called when receiving
302
297
  # requests to host names which match the specified _pattern_.
303
298
  #
304
- # hosts_like(/^a[0-9]\./) do
305
- # map('/download/') do |request|
299
+ # server.hosts_like(/^a[0-9]\./) do |vhost|
300
+ # vhost.map('/download/') do |request|
306
301
  # ...
307
302
  # end
308
303
  # end
309
304
  #
310
305
  def hosts_like(pattern,server=nil,&block)
311
- @virtual_host_patterns[pattern] = (server || self.class.new(&block))
306
+ server ||= self.class.new(&block)
307
+
308
+ @vhost_patterns_mutex.synchronize do
309
+ @vhost_patterns[pattern] = server
310
+ end
311
+
312
+ return server
312
313
  end
313
314
 
314
315
  #
315
316
  # Registers the given _server_ or _block_ to be called when receiving
316
317
  # requests for paths which match the specified _pattern_.
317
318
  #
318
- # paths_like(/\.xml$/) do |request|
319
+ # server.paths_like(/\.xml$/) do |request|
319
320
  # ...
320
321
  # end
321
322
  #
322
323
  def paths_like(pattern,server=nil,&block)
323
- @path_patterns[pattern] = (server || block)
324
+ @patterns_mutex.synchronize do
325
+ @patterns[pattern] = (server || block)
326
+ end
327
+
324
328
  return self
325
329
  end
326
330
 
@@ -329,23 +333,32 @@ module Ronin
329
333
  # connects it as a virtual host representing the specified host
330
334
  # _name_.
331
335
  #
332
- # host('cdn.evil.com') do
336
+ # server.host('cdn.evil.com') do |server|
333
337
  # ...
334
338
  # end
335
339
  #
336
340
  def host(name,server=nil,&block)
337
- @virtual_hosts[name.to_s] = (server || self.class.new(&block))
341
+ server ||= self.class.new(&block)
342
+
343
+ @vhosts_mutex.synchronize do
344
+ @vhosts[name.to_s] = server
345
+ end
346
+
347
+ return server
338
348
  end
339
349
 
340
350
  #
341
351
  # Binds the specified URL _path_ to the given _server_ or _block_.
342
352
  #
343
- # bind '/secrets.xml' do |request|
353
+ # server.bind '/secrets.xml' do |request|
344
354
  # [200, {'Content-Type' => 'text/xml'}, "Made you look."]
345
355
  # end
346
356
  #
347
357
  def bind(path,server=nil,&block)
348
- @paths[path] = (server || block)
358
+ @paths_mutex.synchronize do
359
+ @paths[path] = (server || block)
360
+ end
361
+
349
362
  return self
350
363
  end
351
364
 
@@ -353,15 +366,18 @@ module Ronin
353
366
  # Binds the specified URL directory _path_ to the given
354
367
  # _server_ or _block_.
355
368
  #
356
- # map '/downloads' do |request|
357
- # response(
369
+ # server.map '/downloads' do |request|
370
+ # server.response(
358
371
  # "Your somewhere inside the downloads directory",
359
372
  # :content_type' => 'text/xml'
360
373
  # )
361
374
  # end
362
375
  #
363
376
  def map(path,server=nil,&block)
364
- @directories[path] = (server || block)
377
+ @directories_mutex.synchronize do
378
+ @directories[path] = (server || block)
379
+ end
380
+
365
381
  return self
366
382
  end
367
383
 
@@ -369,15 +385,19 @@ module Ronin
369
385
  # Binds the contents of the specified _file_ to the specified URL
370
386
  # _path_, using the given _options_.
371
387
  #
372
- # file '/robots.txt', '/path/to/my_robots.txt'
388
+ # _options_ may contain the following keys:
389
+ # <tt>content_type</tt>:: The content-type to use when serving
390
+ # the file at the specified _path_.
391
+ #
392
+ # server.file '/robots.txt', '/path/to/my_robots.txt'
373
393
  #
374
394
  def file(path,file,options={})
375
395
  file = File.expand_path(file)
376
- content_type = (options[:content_type] || content_type_for(file))
396
+ content_type = options[:content_type]
377
397
 
378
398
  bind(path) do |request|
379
399
  if File.file?(file)
380
- return_file(file,request)
400
+ return_file(file,request,content_type)
381
401
  else
382
402
  not_found(request)
383
403
  end
@@ -388,7 +408,7 @@ module Ronin
388
408
  # Binds the contents of the specified _directory_ to the given
389
409
  # prefix _path_.
390
410
  #
391
- # directory '/download/', '/tmp/files/'
411
+ # server.directory '/download/', '/tmp/files/'
392
412
  #
393
413
  def directory(path,directory)
394
414
  sub_dirs = path.split('/')
@@ -406,10 +426,24 @@ module Ronin
406
426
  end
407
427
 
408
428
  #
409
- # Starts the server.
429
+ # Starts the server. Mongrel will be used to run the server, if it
430
+ # is installed, otherwise WEBrick will be used.
410
431
  #
411
432
  def start
412
- Server.run(self, :host => @host, :port => @port)
433
+ rack_handler = [@handler, 'Mongrel', 'WEBrick'].find do |name|
434
+ name && Rack::Handler.const_defined?(name)
435
+ end
436
+
437
+ unless rack_handler
438
+ raise(StandardError,"unable to find any Rack handlers",caller)
439
+ end
440
+
441
+ rack_options = {
442
+ 'Host' => (@host || Server.default_host),
443
+ 'Port' => (@port || Server.default_port)
444
+ }
445
+
446
+ Rack::Handler.const_get(rack_handler).run(self,rack_options)
413
447
  return self
414
448
  end
415
449
 
@@ -422,36 +456,48 @@ module Ronin
422
456
  request = Rack::Request.new(env)
423
457
 
424
458
  if http_host
425
- if (server = virtual_host(http_host))
459
+ if (server = vhost(http_host))
426
460
  return server.call(env)
427
461
  end
428
462
  end
429
463
 
430
464
  if http_path
431
- if (block = @paths[http_path])
432
- return block.call(request)
465
+ @paths_mutex.synchronize do
466
+ if (block = @paths[http_path])
467
+ return block.call(request)
468
+ end
433
469
  end
434
470
 
435
- @path_patterns.each do |pattern,block|
436
- if http_path.match(pattern)
437
- return block.call(request)
471
+ @patterns_mutex.synchronize do
472
+ @patterns.each do |pattern,block|
473
+ if http_path.match(pattern)
474
+ return block.call(request)
475
+ end
438
476
  end
439
477
  end
440
478
 
441
479
  http_dirs = http_path.split('/')
442
480
 
443
- sub_dir = @directories.keys.select { |path|
444
- dirs = path.split('/')
481
+ @directories_mutex.synchronize do
482
+ sub_dir = @directories.keys.select { |path|
483
+ dirs = path.split('/')
445
484
 
446
- http_dirs[0...dirs.length] == dirs
447
- }.sort.last
485
+ http_dirs[0...dirs.length] == dirs
486
+ }.sort.last
448
487
 
449
- if (sub_dir && (block = @directories[sub_dir]))
450
- return block.call(request)
488
+ if (sub_dir && (block = @directories[sub_dir]))
489
+ return block.call(request)
490
+ end
451
491
  end
452
492
  end
453
493
 
454
- return @default.call(request)
494
+ resp = nil
495
+
496
+ @default_mutex.synchronize do
497
+ resp = @default.call(request)
498
+ end
499
+
500
+ return resp
455
501
  end
456
502
 
457
503
  #
@@ -43,8 +43,8 @@ module Ronin
43
43
  # <tt>:user_agent</tt>:: The User-Agent string to send. Defaults to
44
44
  # Web.user_agent.
45
45
  # <tt>:referer</tt>:: The referer URL to send.
46
- # <tt>:delay</tt>:: Duration in seconds to pause between spidering each
47
- # link. Defaults to 0.
46
+ # <tt>:delay</tt>:: Duration in seconds to pause between spidering
47
+ # each link. Defaults to 0.
48
48
  # <tt>:host</tt>:: The host-name to visit.
49
49
  # <tt>:hosts</tt>:: An +Array+ of host patterns to visit.
50
50
  # <tt>:ignore_hosts</tt>:: An +Array+ of host patterns to not visit.
@@ -57,7 +57,12 @@ module Ronin
57
57
  # visit.
58
58
  #
59
59
  def initialize(options={},&block)
60
- super(default_options.merge(options))
60
+ options = {
61
+ :proxy => Web.proxy,
62
+ :user_agent => Web.user_agent
63
+ }.merge(options)
64
+
65
+ super(options)
61
66
 
62
67
  every_url do |url|
63
68
  print_info("Spidering #{url}")
@@ -66,14 +71,6 @@ module Ronin
66
71
  block.call(self) if block
67
72
  end
68
73
 
69
- protected
70
-
71
- #
72
- # Returns the default options for Spider.
73
- #
74
- def default_options
75
- {:proxy => Web.proxy, :user_agent => Web.user_agent}
76
- end
77
74
  end
78
75
  end
79
76
  end
@@ -24,6 +24,6 @@
24
24
  module Ronin
25
25
  module Web
26
26
  # Ronin Web Version
27
- VERSION = '0.1.2'
27
+ VERSION = '0.1.3'
28
28
  end
29
29
  end
data/lib/ronin/web/web.rb CHANGED
@@ -34,16 +34,51 @@ module Ronin
34
34
  # Returns a Nokogiri::HTML::Document object for the specified _body_
35
35
  # of html.
36
36
  #
37
- def Web.html(body)
38
- Nokogiri::HTML(body)
37
+ def Web.html(body,&block)
38
+ doc = Nokogiri::HTML(body)
39
+
40
+ block.call(doc) if block
41
+ return doc
42
+ end
43
+
44
+ #
45
+ # Creates a new Nokogiri::HTML::Builder with the given _block_.
46
+ #
47
+ # Web.build_html do
48
+ # body {
49
+ # div(:style => 'display:none;') {
50
+ # object(:classid => 'blabla')
51
+ # }
52
+ # }
53
+ # end
54
+ #
55
+ def Web.build_html(&block)
56
+ Nokogiri::HTML::Builder.new(&block)
39
57
  end
40
58
 
41
59
  #
42
60
  # Returns a Nokogiri::XML::Document object for the specified _body_
43
- # of xml.
61
+ # of xml and the given _block_.
44
62
  #
45
- def Web.xml(body)
46
- Nokogiri::XML(body)
63
+ def Web.xml(body,&block)
64
+ doc = Nokogiri::XML(body)
65
+
66
+ block.call(doc) if block
67
+ return doc
68
+ end
69
+
70
+ #
71
+ # Creates a new Nokogiri::XML::Builder with the given _block_.
72
+ #
73
+ # Web.build_xml do
74
+ # post(:id => 2) {
75
+ # title { text("some example) }
76
+ # body { text("this is one contrived example.") }
77
+ # }
78
+ # end
79
+ #
80
+ def Web.build_xml(&block)
81
+ Nokogiri::XML::Builder.new(&block)
47
82
  end
48
83
 
49
84
  #
@@ -272,7 +307,9 @@ module Ronin
272
307
  # Web.post('http://www.rubyinside.com') # => WWW::Mechanize::Page
273
308
  #
274
309
  def Web.post(url,options={},&block)
275
- query = (options[:query] || {})
310
+ query = {}
311
+ query.merge!(options[:query]) if options[:query]
312
+
276
313
  page = Web.agent(options).post(url,query)
277
314
 
278
315
  block.call(page) if block
@@ -5,43 +5,43 @@ require 'web/helpers/server'
5
5
 
6
6
  describe Web::Server do
7
7
  before(:all) do
8
- @server = Web::Server.new do
9
- default do |env|
10
- response('This is default.')
8
+ @server = Web::Server.new do |server|
9
+ server.default do |env|
10
+ server.response('This is default.')
11
11
  end
12
12
 
13
- bind('/test/bind.xml') do |env|
14
- response('<secret/>', :content_type => 'text/xml')
13
+ server.bind('/test/bind.xml') do |env|
14
+ server.response('<secret/>', :content_type => 'text/xml')
15
15
  end
16
16
 
17
- paths_like(/path_patterns\/secret\./) do |env|
18
- response('No secrets here.')
17
+ server.paths_like(/path_patterns\/secret\./) do |env|
18
+ server.response('No secrets here.')
19
19
  end
20
20
 
21
- map('/test/map') do |env|
22
- response('mapped')
21
+ server.map('/test/map') do |env|
22
+ server.response('mapped')
23
23
  end
24
24
 
25
- file('/test/file.txt',File.join(WEB_SERVER_ROOT,'test.txt'))
25
+ server.file('/test/file.txt',File.join(WEB_SERVER_ROOT,'test.txt'))
26
26
 
27
- directory('/test/directory/',WEB_SERVER_ROOT)
27
+ server.directory('/test/directory/',WEB_SERVER_ROOT)
28
28
  end
29
29
 
30
- @virtual_host = Web::Server.new do
31
- bind('/test/virtual_host.xml') do |env|
32
- response('<virtual/>', :content_type => 'text/xml')
30
+ @vhost = Web::Server.new do |vhost|
31
+ vhost.bind('/test/virtual_host.xml') do |env|
32
+ vhost.response('<virtual/>', :content_type => 'text/xml')
33
33
  end
34
34
  end
35
35
 
36
- @server.host('virtual.host.com') do
37
- bind('/test/virtual_host.xml') do |env|
38
- response('<virtual/>', :content_type => 'text/xml')
36
+ @server.host('virtual.host.com') do |vhost|
37
+ vhost.bind('/test/virtual_host.xml') do |env|
38
+ vhost.response('<virtual/>', :content_type => 'text/xml')
39
39
  end
40
40
  end
41
41
 
42
- @server.hosts_like(/^virtual[0-9]\./) do
43
- bind('/test/virtual_host_patterns.xml') do |env|
44
- response('<virtual-patterns/>', :content_type => 'text/xml')
42
+ @server.hosts_like(/^virtual[0-9]\./) do |vhost|
43
+ vhost.bind('/test/virtual_host_patterns.xml') do |env|
44
+ vhost.response('<virtual-patterns/>', :content_type => 'text/xml')
45
45
  end
46
46
  end
47
47
  end
@@ -127,14 +127,14 @@ describe Web::Server do
127
127
  end
128
128
 
129
129
  it "should provide access to servers via their host-names" do
130
- virtual_host = @server.virtual_host('virtual.host.com')
130
+ virtual_host = @server.vhost('virtual.host.com')
131
131
  url = 'http://virtual.host.com/test/virtual_host.xml'
132
132
 
133
133
  get_url(virtual_host,url).body.should == ['<virtual/>']
134
134
  end
135
135
 
136
136
  it "should provide access to servers via their host-names that match virtual host patterns" do
137
- virtual_host = @server.virtual_host('virtual1.host.com')
137
+ virtual_host = @server.vhost('virtual1.host.com')
138
138
  url = 'http://virtual0.host.com/test/virtual_host_patterns.xml'
139
139
 
140
140
  get_url(virtual_host,url).body.should == ['<virtual-patterns/>']
metadata CHANGED
@@ -1,27 +1,38 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ronin-web
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Postmodern
8
8
  autorequire:
9
9
  bindir: bin
10
- cert_chain: []
10
+ cert_chain:
11
+ - |
12
+ -----BEGIN CERTIFICATE-----
13
+ MIIDQDCCAiigAwIBAgIBADANBgkqhkiG9w0BAQUFADBGMRgwFgYDVQQDDA9wb3N0
14
+ bW9kZXJuLm1vZDMxFTATBgoJkiaJk/IsZAEZFgVnbWFpbDETMBEGCgmSJomT8ixk
15
+ ARkWA2NvbTAeFw0wOTA2MDMwNDU5MDNaFw0xMDA2MDMwNDU5MDNaMEYxGDAWBgNV
16
+ BAMMD3Bvc3Rtb2Rlcm4ubW9kMzEVMBMGCgmSJomT8ixkARkWBWdtYWlsMRMwEQYK
17
+ CZImiZPyLGQBGRYDY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA
18
+ 1wvANkTDHFgVih5XLjuTwTZjgBq1lBGybXJiH6Id1lY2JOMqM5FB1DDHVvvij94i
19
+ mJabN0zkzu6VKWC70y0IwOxY7CPokr0eFdK/D0y7mCq1P8QITv76i2YqAl0eYqIt
20
+ W+IhIkANQ7E6uMZIZcdnfadC6lPAtlKkqtd9crvRbFgr6e3kyflmohbRnTEJHoRd
21
+ 7SHHsybE6DSn7oTDs6XBTNrNIn5VfZA0z01eeos/+zBm1zKJOK2+/7xtLLDuDU9G
22
+ +Rd+ltUBbvxUrMNZmDG29pnmN2xTRH+Q8HxD2AxlvM5SRpK6OeZaHV7PaCCAVZ4L
23
+ T9BFl1sfMvRlABeGEkSyuQIDAQABozkwNzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIE
24
+ sDAdBgNVHQ4EFgQUKwsd+PqEYmBvyaTyoL+uRuk+PhEwDQYJKoZIhvcNAQEFBQAD
25
+ ggEBAB4TvHsrlbcXcKg6gX5BIb9tI+zGkpzo0Z7jnxMEcNO7NGGwmzafDBI/xZYv
26
+ xkRH3/HXbGGYDOi6Q6gWt5GujSx0bOImDtYTJTH8jnzN92HzEK5WdScm1QpZKF1e
27
+ cezArMbxbSPaosxTCtG6LQTkE28lFQsmFZ5xzouugS4h5+LVJiVMmiP+l3EfkjFa
28
+ GOURU+rNEMPWo8MCWivGW7jes6BMzWHcW7DQ0scNVmIcCIgdyMmpscuAEOSeghy9
29
+ /fFs57Ey2OXBL55nDOyvN/ZQ2Vab05UH4t+GCxjAPeirzL/29FBtePT6VD44c38j
30
+ pDj+ws7QjtH/Qcrr1l9jfN0ehDs=
31
+ -----END CERTIFICATE-----
11
32
 
12
- date: 2009-03-28 00:00:00 -07:00
33
+ date: 2009-07-08 00:00:00 -07:00
13
34
  default_executable:
14
35
  dependencies:
15
- - !ruby/object:Gem::Dependency
16
- name: nokogiri
17
- type: :runtime
18
- version_requirement:
19
- version_requirements: !ruby/object:Gem::Requirement
20
- requirements:
21
- - - ">="
22
- - !ruby/object:Gem::Version
23
- version: 1.2.0
24
- version:
25
36
  - !ruby/object:Gem::Dependency
26
37
  name: mechanize
27
38
  type: :runtime
@@ -40,7 +51,7 @@ dependencies:
40
51
  requirements:
41
52
  - - ">="
42
53
  - !ruby/object:Gem::Version
43
- version: 0.1.3
54
+ version: 0.1.9
44
55
  version:
45
56
  - !ruby/object:Gem::Dependency
46
57
  name: rack
@@ -50,7 +61,7 @@ dependencies:
50
61
  requirements:
51
62
  - - ">="
52
63
  - !ruby/object:Gem::Version
53
- version: 0.9.1
64
+ version: 1.0.0
54
65
  version:
55
66
  - !ruby/object:Gem::Dependency
56
67
  name: ronin
@@ -60,7 +71,7 @@ dependencies:
60
71
  requirements:
61
72
  - - ">="
62
73
  - !ruby/object:Gem::Version
63
- version: 0.2.2
74
+ version: 0.2.4
64
75
  version:
65
76
  - !ruby/object:Gem::Dependency
66
77
  name: hoe
@@ -70,9 +81,15 @@ dependencies:
70
81
  requirements:
71
82
  - - ">="
72
83
  - !ruby/object:Gem::Version
73
- version: 1.11.0
84
+ version: 2.3.2
74
85
  version:
75
- description: Ronin Web is a Ruby library for Ronin that provides support for web scraping and spidering functionality. Ronin is a Ruby platform designed for information security and data exploration tasks. Ronin allows for the rapid development and distribution of code over many of the common Source-Code-Management (SCM) systems.
86
+ description: |-
87
+ Ronin Web is a Ruby library for Ronin that provides support for web
88
+ scraping and spidering functionality.
89
+
90
+ Ronin is a Ruby platform designed for information security and data
91
+ exploration tasks. Ronin allows for the rapid development and distribution
92
+ of code over many of the common Source-Code-Management (SCM) systems.
76
93
  email:
77
94
  - postmodern.mod3@gmail.com
78
95
  executables:
@@ -83,10 +100,12 @@ extra_rdoc_files:
83
100
  - History.txt
84
101
  - Manifest.txt
85
102
  - README.txt
103
+ - TODO.txt
86
104
  files:
87
105
  - History.txt
88
106
  - Manifest.txt
89
107
  - README.txt
108
+ - TODO.txt
90
109
  - Rakefile
91
110
  - bin/ronin-web
92
111
  - lib/ronin/sessions/web.rb
@@ -99,6 +118,7 @@ files:
99
118
  - lib/ronin/web/extensions/nokogiri/xml/attr.rb
100
119
  - lib/ronin/web/extensions/nokogiri/xml/element.rb
101
120
  - lib/ronin/web/extensions/nokogiri/xml/document.rb
121
+ - lib/ronin/web/fingerprint.rb
102
122
  - lib/ronin/web/server.rb
103
123
  - lib/ronin/web/proxy.rb
104
124
  - lib/ronin/web/spider.rb
@@ -113,6 +133,8 @@ files:
113
133
  - spec/web/server_spec.rb
114
134
  has_rdoc: true
115
135
  homepage: http://ronin.rubyforge.org/web/
136
+ licenses: []
137
+
116
138
  post_install_message:
117
139
  rdoc_options:
118
140
  - --main
@@ -134,9 +156,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
134
156
  requirements: []
135
157
 
136
158
  rubyforge_project: ronin
137
- rubygems_version: 1.3.1
159
+ rubygems_version: 1.3.4
138
160
  signing_key:
139
- specification_version: 2
161
+ specification_version: 3
140
162
  summary: Ronin Web is a Ruby library for Ronin that provides support for web scraping and spidering functionality
141
163
  test_files: []
142
164
 
metadata.gz.sig ADDED
Binary file