arachni 0.2.4 → 0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. data/CHANGELOG.md +33 -0
  2. data/README.md +2 -4
  3. data/Rakefile +15 -4
  4. data/bin/arachni +0 -0
  5. data/bin/arachni_web +0 -0
  6. data/bin/arachni_web_autostart +0 -0
  7. data/bin/arachni_xmlrpc +0 -0
  8. data/bin/arachni_xmlrpcd +0 -0
  9. data/bin/arachni_xmlrpcd_monitor +0 -0
  10. data/lib/arachni.rb +1 -1
  11. data/lib/framework.rb +36 -6
  12. data/lib/http.rb +12 -5
  13. data/lib/module/auditor.rb +482 -59
  14. data/lib/module/base.rb +17 -0
  15. data/lib/module/manager.rb +26 -2
  16. data/lib/module/trainer.rb +1 -12
  17. data/lib/module/utilities.rb +12 -0
  18. data/lib/parser/auditable.rb +8 -3
  19. data/lib/parser/elements.rb +11 -0
  20. data/lib/parser/page.rb +3 -1
  21. data/lib/parser/parser.rb +130 -18
  22. data/lib/rpc/xml/server/dispatcher.rb +21 -0
  23. data/lib/spider.rb +141 -82
  24. data/lib/ui/cli/cli.rb +2 -3
  25. data/lib/ui/web/addon_manager.rb +273 -0
  26. data/lib/ui/web/addons/autodeploy.rb +172 -0
  27. data/lib/ui/web/addons/autodeploy/lib/manager.rb +291 -0
  28. data/lib/ui/web/addons/autodeploy/views/index.erb +124 -0
  29. data/lib/ui/web/addons/sample.rb +78 -0
  30. data/lib/ui/web/addons/sample/views/index.erb +4 -0
  31. data/lib/ui/web/addons/scheduler.rb +139 -0
  32. data/lib/ui/web/addons/scheduler/views/index.erb +131 -0
  33. data/lib/ui/web/addons/scheduler/views/options.erb +93 -0
  34. data/lib/ui/web/dispatcher_manager.rb +80 -13
  35. data/lib/ui/web/instance_manager.rb +87 -0
  36. data/lib/ui/web/scheduler.rb +166 -0
  37. data/lib/ui/web/server.rb +142 -202
  38. data/lib/ui/web/server/public/js/jquery-ui-timepicker.js +985 -0
  39. data/lib/ui/web/server/public/plugins/sample/style.css +0 -0
  40. data/lib/ui/web/server/public/style.css +42 -0
  41. data/lib/ui/web/server/views/addon.erb +15 -0
  42. data/lib/ui/web/server/views/addons.erb +46 -0
  43. data/lib/ui/web/server/views/dispatchers.erb +1 -1
  44. data/lib/ui/web/server/views/instance.erb +9 -11
  45. data/lib/ui/web/server/views/layout.erb +14 -1
  46. data/lib/ui/web/server/views/welcome.erb +7 -6
  47. data/lib/ui/web/utilities.rb +134 -0
  48. data/modules/audit/code_injection_timing.rb +6 -2
  49. data/modules/audit/code_injection_timing/payloads.txt +2 -2
  50. data/modules/audit/os_cmd_injection_timing.rb +7 -3
  51. data/modules/audit/os_cmd_injection_timing/payloads.txt +1 -1
  52. data/modules/audit/sqli_blind_rdiff.rb +18 -233
  53. data/modules/audit/sqli_blind_rdiff/payloads.txt +5 -0
  54. data/modules/audit/sqli_blind_timing.rb +9 -2
  55. data/path_extractors/anchors.rb +1 -1
  56. data/path_extractors/forms.rb +1 -1
  57. data/path_extractors/frames.rb +1 -1
  58. data/path_extractors/generic.rb +1 -1
  59. data/path_extractors/links.rb +1 -1
  60. data/path_extractors/meta_refresh.rb +1 -1
  61. data/path_extractors/scripts.rb +1 -1
  62. data/path_extractors/sitemap.rb +1 -1
  63. data/plugins/proxy/server.rb +3 -2
  64. data/plugins/waf_detector.rb +0 -3
  65. metadata +37 -34
  66. data/lib/anemone/cookie_store.rb +0 -35
  67. data/lib/anemone/core.rb +0 -371
  68. data/lib/anemone/exceptions.rb +0 -5
  69. data/lib/anemone/http.rb +0 -144
  70. data/lib/anemone/page.rb +0 -338
  71. data/lib/anemone/page_store.rb +0 -160
  72. data/lib/anemone/storage.rb +0 -34
  73. data/lib/anemone/storage/base.rb +0 -75
  74. data/lib/anemone/storage/exceptions.rb +0 -15
  75. data/lib/anemone/storage/mongodb.rb +0 -89
  76. data/lib/anemone/storage/pstore.rb +0 -50
  77. data/lib/anemone/storage/redis.rb +0 -90
  78. data/lib/anemone/storage/tokyo_cabinet.rb +0 -57
  79. data/lib/anemone/tentacle.rb +0 -40
@@ -178,6 +178,10 @@ class Dispatcher < Base
178
178
  }
179
179
  end
180
180
 
181
+ def proc_info
182
+ unnil( proc( Process.pid ) )
183
+ end
184
+
181
185
  #
182
186
  # Outputs the Arachni banner.<br/>
183
187
  # Displays version number, revision number, author details etc.
@@ -235,6 +239,23 @@ USAGE
235
239
 
236
240
  private
237
241
 
242
+ #
243
+ # Recursively removes nils.
244
+ #
245
+ # @param [Hash] hash
246
+ #
247
+ # @return [Hash]
248
+ #
249
+ def unnil( hash )
250
+ hash.each_pair {
251
+ |k, v|
252
+ hash[k] = '' if v.nil?
253
+ hash[k] = unnil( v ) if v.is_a? Hash
254
+ }
255
+
256
+ return hash
257
+ end
258
+
238
259
  #
239
260
  # Initializes and updates the pool making sure that the number of
240
261
  # available server processes stays constant for any given moment
data/lib/spider.rb CHANGED
@@ -8,8 +8,9 @@
8
8
 
9
9
  =end
10
10
 
11
- require Arachni::Options.instance.dir['lib'] + 'anemone'
12
11
  require Arachni::Options.instance.dir['lib'] + 'module/utilities'
12
+ require 'nokogiri'
13
+ require Arachni::Options.instance.dir['lib'] + 'nokogiri/xml/node'
13
14
 
14
15
  module Arachni
15
16
 
@@ -21,7 +22,7 @@ module Arachni
21
22
  # @author: Tasos "Zapotek" Laskos
22
23
  # <tasos.laskos@gmail.com>
23
24
  # <zapotek@segfault.gr>
24
- # @version: 0.1
25
+ # @version: 0.2
25
26
  #
26
27
  class Spider
27
28
 
@@ -34,8 +35,6 @@ class Spider
34
35
  #
35
36
  attr_reader :opts
36
37
 
37
- attr_reader :pages
38
-
39
38
  #
40
39
  # Sitemap, array of links
41
40
  #
@@ -59,31 +58,6 @@ class Spider
59
58
  def initialize( opts )
60
59
  @opts = opts
61
60
 
62
- @anemone_opts = {
63
- :threads => 1,
64
- :discard_page_bodies => false,
65
- :delay => 0,
66
- :obey_robots_txt => false,
67
- :depth_limit => false,
68
- :link_count_limit => false,
69
- :redirect_limit => false,
70
- :storage => nil,
71
- :cookies => nil,
72
- :accept_cookies => true,
73
- :proxy_addr => nil,
74
- :proxy_port => nil,
75
- :proxy_user => nil,
76
- :proxy_pass => nil
77
- }
78
-
79
- hash_opts = @opts.to_h
80
- @anemone_opts.each_pair {
81
- |k, v|
82
- @anemone_opts[k] = hash_opts[k.to_s] if hash_opts[k.to_s]
83
- }
84
-
85
- @anemone_opts = @anemone_opts.merge( hash_opts )
86
-
87
61
  @sitemap = []
88
62
  @on_every_page_blocks = []
89
63
 
@@ -102,85 +76,170 @@ class Spider
102
76
  def run( &block )
103
77
  return if @opts.link_count_limit == 0
104
78
 
105
- i = 1
106
- # start the crawl
107
- Anemone.crawl( @opts.url, @anemone_opts ) {
108
- |anemone|
79
+ paths = []
80
+ paths << @opts.url.to_s
109
81
 
110
- # apply 'exclude' patterns
111
- anemone.skip_links_like( @opts.exclude ) if @opts.exclude
82
+ visited = []
112
83
 
113
- # apply 'include' patterns and grab matching pages
114
- # as they are discovered
115
- anemone.on_pages_like( @opts.include ) {
116
- |page|
84
+ while( !paths.empty? )
85
+ while( !paths.empty? && url = paths.pop )
86
+ url = url_sanitize( url )
87
+ next if skip?( url ) || !in_domain?( url )
117
88
 
118
- @pages = anemone.pages.keys || []
89
+ wait_if_paused
119
90
 
120
- url = url_sanitize( page.url.to_s )
91
+ visited << url
121
92
 
122
- # something went kaboom, tell the user and skip the page
123
- if page.error
124
- print_error( "[Error: " + (page.error.to_s) + "] " + url )
125
- print_debug_backtrace( page.error )
126
- next
127
- end
93
+ opts = {
94
+ :timeout => nil,
95
+ :remove_id => true,
96
+ :async => @opts.spider_first
97
+ }
128
98
 
129
- # push the url in the sitemap
130
- @sitemap.push( url )
131
-
132
- print_line
133
- print_status( "[HTTP: #{page.code}] " + url )
134
-
135
- # call the block...if we have one
136
- if block
137
- exception_jail{
138
- new_page = Arachni::Parser.new( @opts,
139
- Typhoeus::Response.new(
140
- :effective_url => url,
141
- :body => page.body,
142
- :headers_hash => page.headers
143
- )
144
- ).run
145
- new_page.code = page.code
146
- new_page.method = 'GET'
147
- block.call( new_page.clone )
99
+ Arachni::HTTP.instance.get( url, opts ).on_complete {
100
+ |res|
101
+
102
+ print_line
103
+ print_status( "[HTTP: #{res.code}] " + res.effective_url )
104
+
105
+ page = Arachni::Parser.new( @opts, res ).run
106
+ page.url = url_sanitize( res.effective_url )
107
+
108
+ @sitemap |= page.paths.map { |path| url_sanitize( path ) }
109
+ paths |= @sitemap - visited
110
+
111
+
112
+ # call the block...if we have one
113
+ if block
114
+ exception_jail{
115
+ block.call( page.clone )
116
+ }
117
+ end
118
+
119
+ # run blocks specified later
120
+ @on_every_page_blocks.each {
121
+ |block|
122
+ block.call( page )
148
123
  }
149
- end
150
124
 
151
- # run blocks specified later
152
- @on_every_page_blocks.each {
153
- |block|
154
- block.call( page )
155
125
  }
156
126
 
157
- # we don't need the HTML doc anymore
158
- page.discard_doc!( )
127
+ Arachni::HTTP.instance.run if !@opts.spider_first
159
128
 
160
129
  # make sure we obey the link count limit and
161
130
  # return if we have exceeded it.
162
131
  if( @opts.link_count_limit &&
163
- @opts.link_count_limit <= i )
132
+ @opts.link_count_limit <= visited.size )
133
+ Arachni::HTTP.instance.run if @opts.spider_first
164
134
  return @sitemap.uniq
165
135
  end
166
136
 
167
- i+=1
168
- }
169
- }
137
+
138
+ end
139
+
140
+ if @opts.spider_first
141
+ Arachni::HTTP.instance.run
142
+ else
143
+ break
144
+ end
145
+
146
+ end
170
147
 
171
148
  return @sitemap.uniq
172
149
  end
173
150
 
151
+ def skip?( url )
152
+ @opts.exclude.each {
153
+ |regexp|
154
+ return true if regexp =~ url
155
+ }
156
+
157
+ @opts.redundant.each_with_index {
158
+ |redundant, i|
159
+
160
+ if( url =~ redundant['regexp'] )
161
+
162
+ if( @opts.redundant[i]['count'] == 0 )
163
+ print_verbose( 'Discarding redundant page: \'' + url + '\'' )
164
+ return true
165
+ end
166
+
167
+ print_info( 'Matched redundancy rule: ' +
168
+ redundant['regexp'].to_s + ' for page \'' +
169
+ url + '\'' )
170
+
171
+ print_info( 'Count-down: ' + @opts.redundant[i]['count'].to_s )
172
+
173
+ @opts.redundant[i]['count'] -= 1
174
+ end
175
+ }
176
+
177
+
178
+ skip_cnt = 0
179
+ @opts.include.each {
180
+ |regexp|
181
+ skip_cnt += 1 if !(regexp =~ url)
182
+ }
183
+
184
+ return false if skip_cnt > 1
185
+
186
+ return false
187
+ end
188
+
189
+ def wait_if_paused
190
+ while( paused? )
191
+ ::IO::select( nil, nil, nil, 1 )
192
+ end
193
+ end
194
+
195
+ def pause!
196
+ @pause = true
197
+ end
198
+
199
+ def resume!
200
+ @pause = false
201
+ end
202
+
203
+ def paused?
204
+ @pause ||= false
205
+ return @pause
206
+ end
207
+
208
+ #
209
+ # Checks if the uri is in the same domain
174
210
  #
175
- # Decodes URLs to reverse multiple encodes and removes NULL characters
211
+ # @param [URI] url
176
212
  #
177
- def url_sanitize( url )
213
+ # @return [String]
214
+ #
215
+ def in_domain?( uri )
216
+
217
+ uri_1 = URI( uri.to_s )
218
+ uri_2 = URI( @opts.url.to_s )
178
219
 
179
- while( url =~ /%/ )
180
- url = ( URI.decode( url ).to_s.unpack( 'A*' )[0] )
220
+ if( @opts.follow_subdomains )
221
+ return extract_domain( uri_1 ) == extract_domain( uri_2 )
181
222
  end
182
223
 
183
- return url
224
+ uri_1.host == uri_2.host
225
+ end
226
+
227
+ #
228
+ # Extracts the domain from a URI object
229
+ #
230
+ # @param [URI] url
231
+ #
232
+ # @return [String]
233
+ #
234
+ def extract_domain( url )
235
+
236
+ if !url.host then return false end
237
+
238
+ splits = url.host.split( /\./ )
239
+
240
+ if splits.length == 1 then return true end
241
+
242
+ splits[-2] + "." + splits[-1]
184
243
  end
185
244
 
186
245
 
data/lib/ui/cli/cli.rb CHANGED
@@ -123,16 +123,15 @@ class CLI
123
123
 
124
124
  audited = stats[:auditmap_size]
125
125
  mapped = stats[:sitemap_size]
126
- progress = ( Float( audited ) / mapped ) * 100
127
126
 
128
127
  print_line
129
- print_info( "Audit progress: #{progress.to_s[0...5]}% ( #{audited}/#{mapped} pages )" )
128
+ print_info( "Audit progress: #{stats[:progress]}% ( Discovered #{mapped} pages )" )
130
129
  print_line
131
130
  print_info( "Sent #{stats[:requests]} requests." )
132
131
  print_info( "Received and analyzed #{stats[:responses]} responses." )
133
132
  print_info( 'In ' + stats[:time] )
134
133
 
135
- avg = 'Average: ' + stats[:avg] + ' requests/second.'
134
+ avg = 'Average: ' + stats[:avg].to_s + ' requests/second.'
136
135
  print_info( avg )
137
136
 
138
137
  print_line
@@ -0,0 +1,273 @@
1
+ =begin
2
+ Arachni
3
+ Copyright (c) 2010-2011 Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
4
+
5
+ This is free software; you can copy and distribute and modify
6
+ this program under the term of the GPL v2.0 License
7
+ (See LICENSE file for details)
8
+
9
+ =end
10
+
11
+
12
+ module Arachni
13
+ module UI
14
+ module Web
15
+
16
+ module Addons
17
+
18
+ #
19
+ # Base class for all add-ons.
20
+ #
21
+ #
22
+ # @author: Tasos "Zapotek" Laskos
23
+ # <tasos.laskos@gmail.com>
24
+ # <zapotek@segfault.gr>
25
+ # @version: 0.1
26
+ #
27
+ class Base
28
+
29
+ def initialize( settings, route )
30
+ @settings = settings
31
+ @route = '/addons/' + route
32
+
33
+ @settings.helpers do
34
+
35
+ def present( tpl, args )
36
+ views = current_addon.path_views
37
+ trv = ( '../' * views.split( '/' ).size ) + views + tpl.to_s
38
+
39
+ erb_args = []
40
+ erb_args << { :layout => true }
41
+ erb_args << { :tpl => trv.to_sym, :addon => addons.by_name( current_addon_name ), :tpl_args => args }
42
+
43
+ erb :addon, *erb_args
44
+ end
45
+
46
+ def partial( tpl, args )
47
+ views = current_addon.path_views
48
+ trv = ( '../' * views.split( '/' ).size ) + views + tpl.to_s
49
+
50
+ erb_args = []
51
+ erb_args << { :layout => false }
52
+ erb_args << args
53
+
54
+ erb trv.to_sym, *erb_args
55
+ end
56
+
57
+ def current_addon_name
58
+ env['PATH_INFO'].scan( /\/addons\/(.*?)\// ).flatten[0]
59
+ end
60
+
61
+ def current_addon
62
+ addons.running[current_addon_name]
63
+ end
64
+
65
+ end
66
+
67
+ end
68
+
69
+ def path_root
70
+ @route
71
+ end
72
+
73
+ def path_views
74
+ path_addon + '/views/'
75
+ end
76
+
77
+ def path_addon
78
+ Options.instance.dir['lib'] + 'ui/web' + path_root
79
+ end
80
+
81
+ def run
82
+
83
+ end
84
+
85
+ #
86
+ # This optional method allows you to specify the title which will be
87
+ # used for the menu (in case you want it to be dynamic).
88
+ #
89
+ # @return [String]
90
+ #
91
+ def title
92
+ ''
93
+ end
94
+
95
+
96
+ #
97
+ #
98
+ # *DO NOT MESS WITH THE FOLLOWING METHODS*
99
+ #
100
+ #
101
+
102
+
103
+ def settings
104
+ @settings
105
+ end
106
+
107
+ def get( path, &block )
108
+ settings.get( @route + path, &block )
109
+ end
110
+
111
+ def post( path, &block )
112
+ settings.post( @route + path, &block )
113
+ end
114
+
115
+ def put( path, &block )
116
+ settings.put( @route + path, &block )
117
+ end
118
+
119
+ def delete( path, &block )
120
+ settings.delete( @route + path, &block )
121
+ end
122
+
123
+ end
124
+ end
125
+
126
+
127
+ #
128
+ # Add-on manager.
129
+ #
130
+ #
131
+ # @author: Tasos "Zapotek" Laskos
132
+ # <tasos.laskos@gmail.com>
133
+ # <zapotek@segfault.gr>
134
+ # @version: 0.1
135
+ #
136
+ class AddonManager
137
+
138
+ include Utilities
139
+
140
+ class Addon
141
+ include DataMapper::Resource
142
+
143
+ property :id, Serial
144
+ property :name, String
145
+ end
146
+
147
+ class RestrictedComponentManager < Arachni::ComponentManager
148
+ def paths
149
+ cpaths = paths = Dir.glob( File.join( "#{@lib}", "*.rb" ) )
150
+ return paths.reject { |path| helper?( path ) }
151
+ end
152
+ end
153
+
154
+ def initialize( opts, settings )
155
+ @opts = opts
156
+ @settings = settings
157
+
158
+ lib = @opts.dir['lib'] + 'ui/web/addons/'
159
+ @@manager ||= RestrictedComponentManager.new( lib, Addons )
160
+
161
+ @@running ||= {}
162
+
163
+ DataMapper::setup( :default, "sqlite3://#{@settings.db}/default.db" )
164
+ DataMapper.finalize
165
+
166
+ Addon.auto_upgrade!
167
+
168
+ run( enabled )
169
+ end
170
+
171
+ #
172
+ # Runs addons.
173
+ #
174
+ # @param [Array] addons array holding the names of the addons
175
+ #
176
+ def run( addons )
177
+
178
+ begin
179
+ addons.each {
180
+ |name|
181
+ @@running[name] = @@manager[name].new( @settings, name )
182
+ @@running[name].run
183
+ }
184
+
185
+ rescue ::Exception => e
186
+ ap e.to_s
187
+ ap e.backtrace
188
+ end
189
+ end
190
+
191
+ def running
192
+ @@running
193
+ end
194
+
195
+ #
196
+ # Gets add-on info by name.
197
+ #
198
+ # @param [String] name
199
+ #
200
+ # @return [Hash]
201
+ #
202
+ def by_name( name )
203
+ available.each { |addon| return addon if addon['filename'] == name }
204
+ return nil
205
+ end
206
+
207
+ #
208
+ # Gets all available add-ons.
209
+ #
210
+ # @return [Array]
211
+ #
212
+ def available
213
+ @@available ||= populate_available
214
+
215
+ @@available.each {
216
+ |addon|
217
+
218
+ if @@running[addon['filename']] && !@@running[addon['filename']].title.empty?
219
+ addon['title'] = @@running[addon['filename']].title
220
+ else
221
+ addon['title'] = addon['name']
222
+ end
223
+ }
224
+
225
+ return @@available
226
+ end
227
+
228
+ #
229
+ # Enables and runs add-ons.
230
+ #
231
+ # @param [Array] addons array holding the names of the addons
232
+ #
233
+ def enable!( addons )
234
+ Addon.all.destroy
235
+ addons.each { |addon| Addon.create( :name => addon ); run( [addon] ) }
236
+ end
237
+
238
+ #
239
+ # Gets all enabled add-ons.
240
+ #
241
+ # @return [Array]
242
+ #
243
+ def enabled
244
+ Addon.all.map { |addon| addon.name }
245
+ end
246
+
247
+ private
248
+ def populate_available
249
+ @@available ||= []
250
+ return @@available if !@@available.empty?
251
+
252
+ @@available_classes ||= {}
253
+ @@manager.available.each {
254
+ |avail|
255
+
256
+ @@available << {
257
+ 'name' => @@manager[avail].info[:name],
258
+ 'filename' => avail,
259
+ 'description' => @@manager[avail].info[:description],
260
+ 'version' => @@manager[avail].info[:version],
261
+ 'author' => @@manager[avail].info[:author]
262
+ }
263
+
264
+ @@available_classes[avail] = @@manager[avail]
265
+
266
+ }
267
+ return @@available
268
+ end
269
+
270
+ end
271
+ end
272
+ end
273
+ end