panmind-tarantula 0.3.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. data/CHANGELOG +49 -0
  2. data/LICENSE +20 -0
  3. data/README.rdoc +171 -0
  4. data/Rakefile +83 -0
  5. data/VERSION.yml +4 -0
  6. data/examples/example_helper.rb +57 -0
  7. data/examples/relevance/core_extensions/ellipsize_example.rb +19 -0
  8. data/examples/relevance/core_extensions/file_example.rb +8 -0
  9. data/examples/relevance/core_extensions/response_example.rb +29 -0
  10. data/examples/relevance/core_extensions/test_case_example.rb +20 -0
  11. data/examples/relevance/tarantula/attack_handler_example.rb +29 -0
  12. data/examples/relevance/tarantula/basic_attack_example.rb +12 -0
  13. data/examples/relevance/tarantula/crawler_example.rb +375 -0
  14. data/examples/relevance/tarantula/form_example.rb +50 -0
  15. data/examples/relevance/tarantula/form_submission_example.rb +171 -0
  16. data/examples/relevance/tarantula/html_document_handler_example.rb +43 -0
  17. data/examples/relevance/tarantula/html_report_helper_example.rb +46 -0
  18. data/examples/relevance/tarantula/html_reporter_example.rb +82 -0
  19. data/examples/relevance/tarantula/invalid_html_handler_example.rb +33 -0
  20. data/examples/relevance/tarantula/io_reporter_example.rb +11 -0
  21. data/examples/relevance/tarantula/link_example.rb +84 -0
  22. data/examples/relevance/tarantula/log_grabber_example.rb +26 -0
  23. data/examples/relevance/tarantula/rails_integration_proxy_example.rb +101 -0
  24. data/examples/relevance/tarantula/result_example.rb +85 -0
  25. data/examples/relevance/tarantula/tidy_handler_example.rb +58 -0
  26. data/examples/relevance/tarantula/transform_example.rb +20 -0
  27. data/examples/relevance/tarantula_example.rb +23 -0
  28. data/laf/images/header_bg.jpg +0 -0
  29. data/laf/images/logo.png +0 -0
  30. data/laf/images/tagline.png +0 -0
  31. data/laf/javascripts/jquery-1.2.3.js +3408 -0
  32. data/laf/javascripts/jquery-ui-tabs.js +890 -0
  33. data/laf/javascripts/jquery.tablesorter.js +861 -0
  34. data/laf/javascripts/tarantula.js +10 -0
  35. data/laf/stylesheets/tarantula.css +346 -0
  36. data/lib/relevance/core_extensions/ellipsize.rb +34 -0
  37. data/lib/relevance/core_extensions/file.rb +9 -0
  38. data/lib/relevance/core_extensions/metaclass.rb +78 -0
  39. data/lib/relevance/core_extensions/response.rb +9 -0
  40. data/lib/relevance/core_extensions/string_chars_fix.rb +11 -0
  41. data/lib/relevance/core_extensions/test_case.rb +19 -0
  42. data/lib/relevance/tarantula.rb +59 -0
  43. data/lib/relevance/tarantula/attack.rb +18 -0
  44. data/lib/relevance/tarantula/attack_handler.rb +37 -0
  45. data/lib/relevance/tarantula/basic_attack.rb +40 -0
  46. data/lib/relevance/tarantula/crawler.rb +254 -0
  47. data/lib/relevance/tarantula/detail.html.erb +81 -0
  48. data/lib/relevance/tarantula/form.rb +23 -0
  49. data/lib/relevance/tarantula/form_submission.rb +88 -0
  50. data/lib/relevance/tarantula/html_document_handler.rb +36 -0
  51. data/lib/relevance/tarantula/html_report_helper.rb +39 -0
  52. data/lib/relevance/tarantula/html_reporter.rb +105 -0
  53. data/lib/relevance/tarantula/index.html.erb +37 -0
  54. data/lib/relevance/tarantula/invalid_html_handler.rb +18 -0
  55. data/lib/relevance/tarantula/io_reporter.rb +34 -0
  56. data/lib/relevance/tarantula/link.rb +76 -0
  57. data/lib/relevance/tarantula/log_grabber.rb +16 -0
  58. data/lib/relevance/tarantula/rails_integration_proxy.rb +87 -0
  59. data/lib/relevance/tarantula/recording.rb +12 -0
  60. data/lib/relevance/tarantula/response.rb +13 -0
  61. data/lib/relevance/tarantula/result.rb +77 -0
  62. data/lib/relevance/tarantula/test_report.html.erb +32 -0
  63. data/lib/relevance/tarantula/tidy_handler.rb +32 -0
  64. data/lib/relevance/tarantula/transform.rb +17 -0
  65. data/tasks/tarantula_tasks.rake +42 -0
  66. data/template/tarantula_test.rb +22 -0
  67. metadata +213 -0
@@ -0,0 +1,9 @@
1
+ # dynamically mixed in to response objects
2
+ module Relevance::CoreExtensions::Response
3
+ def html?
4
+ # some versions of Rails integration tests don't set content type
5
+ # so we are treating nil as html. A better fix would be welcome here.
6
+ ((content_type =~ %r{^text/html}) != nil) || content_type == nil
7
+ end
8
+ end
9
+
@@ -0,0 +1,11 @@
1
+ if RUBY_VERSION == "1.8.7" # fix interaction between Ruby 187 and Rails 202, so we can at least run the test suite on that combination
2
+ unless '1.9'.respond_to?(:force_encoding)
3
+ String.class_eval do
4
+ begin
5
+ remove_method :chars
6
+ rescue NameError
7
+ # OK
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,19 @@
1
+ require 'action_controller/integration'
2
+
3
+ module Relevance::CoreExtensions::TestCaseExtensions
4
+
5
+ def tarantula_crawl(integration_test, options = {})
6
+ url = options[:url] || "/"
7
+ t = tarantula_crawler(integration_test, options)
8
+ t.crawl url
9
+ end
10
+
11
+ def tarantula_crawler(integration_test, options = {})
12
+ Relevance::Tarantula::RailsIntegrationProxy.rails_integration_test(integration_test, options)
13
+ end
14
+
15
+ end
16
+
17
+ if defined? ActionController::IntegrationTest
18
+ ActionController::IntegrationTest.class_eval { include Relevance::CoreExtensions::TestCaseExtensions }
19
+ end
@@ -0,0 +1,59 @@
1
+ TARANTULA_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "../.."))
2
+
3
+ require 'forwardable'
4
+ require 'erb'
5
+ require 'active_support'
6
+ require 'action_controller'
7
+
8
+ # bringing in xss-shield requires a bunch of other dependencies
9
+ # still not certain about this, if it ruins your world please let me know
10
+ #xss_shield_path = File.join(TARANTULA_ROOT, %w{vendor xss-shield})
11
+ #$: << File.join(xss_shield_path, "lib")
12
+ #require File.join(xss_shield_path, "init")
13
+
14
+ require 'htmlentities'
15
+
16
+ module Relevance; end
17
+ module Relevance; module CoreExtensions; end; end
18
+ module Relevance
19
+ module Tarantula
20
+ def tarantula_home
21
+ File.expand_path(File.join(File.dirname(__FILE__), "../.."))
22
+ end
23
+ def log(msg)
24
+ puts msg if verbose
25
+ end
26
+ def rails_root
27
+ ::RAILS_ROOT
28
+ end
29
+ def verbose
30
+ ENV["VERBOSE"]
31
+ end
32
+ end
33
+ end
34
+
35
+ require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "test_case"))
36
+ require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "ellipsize"))
37
+ require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "file"))
38
+ require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "response"))
39
+ require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "metaclass"))
40
+ require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "string_chars_fix"))
41
+
42
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_reporter"))
43
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_report_helper"))
44
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "io_reporter"))
45
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "recording"))
46
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "response"))
47
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "result"))
48
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "log_grabber"))
49
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "invalid_html_handler"))
50
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "transform"))
51
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "crawler"))
52
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "basic_attack"))
53
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form"))
54
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form_submission"))
55
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack"))
56
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_handler"))
57
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "link"))
58
+
59
+ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "tidy_handler")) if ENV['TIDY_PATH']
@@ -0,0 +1,18 @@
1
+ class Relevance::Tarantula::Attack
2
+ HASHABLE_ATTRS = [:name, :input, :output, :description]
3
+ attr_accessor *HASHABLE_ATTRS
4
+ def initialize(hash)
5
+ hash.each do |k,v|
6
+ raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
7
+ self.instance_variable_set("@#{k}", v)
8
+ end
9
+ end
10
+ def ==(other)
11
+ Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
12
+ end
13
+ def input(input_field=nil)
14
+ @input
15
+ end
16
+ end
17
+
18
+
@@ -0,0 +1,37 @@
1
+ require 'hpricot'
2
+
3
+ class Relevance::Tarantula::AttackHandler
4
+ include ERB::Util
5
+
6
+ def attacks
7
+ Relevance::Tarantula::FormSubmission.attacks.select(&:output)
8
+ end
9
+
10
+ def handle(result)
11
+ return unless attacks.size > 0
12
+ regexp = '(' + attacks.map {|a| Regexp.escape a.output}.join('|') + ')'
13
+ response = result.response
14
+ return unless response.html?
15
+ if n = (response.body =~ /#{regexp}/)
16
+ error_result = result.dup
17
+ error_result.success = false
18
+ error_result.description = "XSS error found, match was: #{h($1)}"
19
+ error_result.data = <<-STR
20
+ ########################################################################
21
+ # Text around unescaped string: #{$1}
22
+ ########################################################################
23
+ #{response.body[[0, n - 200].max , 400]}
24
+
25
+
26
+
27
+
28
+
29
+ ########################################################################
30
+ # Attack information:
31
+ ########################################################################
32
+ #{attacks.select {|a| a.output == $1}[0].to_yaml}
33
+ STR
34
+ error_result
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,40 @@
1
+ class Relevance::Tarantula::BasicAttack
2
+ ATTRS = [:name, :output, :description]
3
+
4
+ attr_reader *ATTRS
5
+
6
+ def initialize
7
+ @name = "Tarantula Basic Fuzzer"
8
+ @output = nil
9
+ @description = "Supplies purely random but simplistically generated form input."
10
+ end
11
+
12
+ def ==(other)
13
+ Relevance::Tarantula::BasicAttack === other && ATTRS.all? { |attr| send(attr) == other.send(attr)}
14
+ end
15
+
16
+ def input(input_field)
17
+ case input_field['name']
18
+ when /amount/ then random_int
19
+ when /_id$/ then random_whole_number
20
+ when /uploaded_data/ then nil
21
+ when nil then input['value']
22
+ else
23
+ random_int
24
+ end
25
+ end
26
+
27
+ def big_number
28
+ 10000 # arbitrary
29
+ end
30
+
31
+ def random_int
32
+ rand(big_number) - (big_number/2)
33
+ end
34
+
35
+ def random_whole_number
36
+ rand(big_number)
37
+ end
38
+ end
39
+
40
+
@@ -0,0 +1,254 @@
1
+ require 'active_record'
2
+ require 'active_record/base'
3
+ require File.expand_path(File.join(File.dirname(__FILE__), "rails_integration_proxy"))
4
+ require File.expand_path(File.join(File.dirname(__FILE__), "html_document_handler.rb"))
5
+
6
+ class Relevance::Tarantula::Crawler
7
+ extend Forwardable
8
+ include Relevance::Tarantula
9
+
10
+ class CrawlTimeout < RuntimeError; end
11
+
12
+ attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
13
+ :reporters, :crawl_queue, :links_queued,
14
+ :form_signatures_queued, :max_url_length, :response_code_handler,
15
+ :times_to_crawl, :fuzzers, :test_name, :crawl_timeout
16
+ attr_reader :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
17
+
18
+ def initialize
19
+ @max_url_length = 1024
20
+ @successes = []
21
+ @failures = []
22
+ @handlers = [@response_code_handler = Result]
23
+ @links_queued = Set.new
24
+ @form_signatures_queued = Set.new
25
+ @crawl_queue = []
26
+ @crawl_start_times, @crawl_end_times = [], []
27
+ @crawl_timeout = 20.minutes
28
+ @referrers = {}
29
+ @skip_uri_patterns = [
30
+ /^javascript/,
31
+ /^mailto/,
32
+ /^http/,
33
+ ]
34
+ self.transform_url_patterns = [
35
+ [/#.*$/, '']
36
+ ]
37
+ @reporters = [Relevance::Tarantula::IOReporter.new($stderr)]
38
+ @decoder = HTMLEntities.new
39
+ @times_to_crawl = 1
40
+ @fuzzers = [Relevance::Tarantula::FormSubmission]
41
+
42
+ @stdout_tty = $stdout.tty?
43
+ end
44
+
45
+ def method_missing(meth, *args)
46
+ super unless Result::ALLOW_NNN_FOR =~ meth.to_s
47
+ @response_code_handler.send(meth, *args)
48
+ end
49
+
50
+ def transform_url_patterns=(patterns)
51
+ @transform_url_patterns = patterns.map do |pattern|
52
+ Array === pattern ? Relevance::Tarantula::Transform.new(*pattern) : pattern
53
+ end
54
+ end
55
+
56
+ def crawl(url = "/")
57
+ orig_links_queued = @links_queued.dup
58
+ orig_form_signatures_queued = @form_signatures_queued.dup
59
+ orig_crawl_queue = @crawl_queue.dup
60
+ @times_to_crawl.times do |num|
61
+ queue_link url
62
+
63
+ begin
64
+ do_crawl num
65
+ rescue CrawlTimeout => e
66
+ puts
67
+ puts e.message
68
+ end
69
+
70
+ puts "#{(num+1).ordinalize} crawl" if @times_to_crawl > 1
71
+
72
+ if num + 1 < @times_to_crawl
73
+ @links_queued = orig_links_queued
74
+ @form_signatures_queued = orig_form_signatures_queued
75
+ @crawl_queue = orig_crawl_queue
76
+ @referrers = {}
77
+ end
78
+ end
79
+ rescue Interrupt
80
+ $stderr.puts "CTRL-C"
81
+ ensure
82
+ report_results
83
+ end
84
+
85
+ def finished?
86
+ @crawl_queue.empty?
87
+ end
88
+
89
+ def do_crawl(number)
90
+ while (!finished?)
91
+ @crawl_start_times << Time.now
92
+ crawl_the_queue(number)
93
+ @crawl_end_times << Time.now
94
+ end
95
+ end
96
+
97
+ def crawl_the_queue(number = 0)
98
+ while (request = @crawl_queue.pop)
99
+ request.crawl
100
+ blip(number)
101
+ end
102
+ end
103
+
104
+ def save_result(result)
105
+ reporters.each do |reporter|
106
+ reporter.report(result)
107
+ end
108
+ end
109
+
110
+ def handle_link_results(link, result)
111
+ handlers.each do |h|
112
+ begin
113
+ save_result h.handle(result)
114
+ rescue Exception => e
115
+ log "error handling #{link} #{e.message}"
116
+ # TODO: pass to results
117
+ end
118
+ end
119
+ end
120
+
121
+ def follow(method, url, data=nil)
122
+ proxy.send(method, url, data)
123
+ end
124
+
125
+ def submit(method, action, data)
126
+ proxy.send(method, action, data)
127
+ end
128
+
129
+ def elasped_time_for_pass(num)
130
+ Time.now - crawl_start_times[num]
131
+ end
132
+
133
+ def grab_log!
134
+ @log_grabber && @log_grabber.grab!
135
+ end
136
+
137
+ def make_result(options)
138
+ defaults = {
139
+ :log => grab_log!,
140
+ :test_name => test_name
141
+ }
142
+ Result.new(defaults.merge(options)).freeze
143
+ end
144
+
145
+ def handle_form_results(form, response)
146
+ handlers.each do |h|
147
+ save_result h.handle(Result.new(:method => form.method,
148
+ :url => form.action,
149
+ :response => response,
150
+ :log => grab_log!,
151
+ :referrer => form.action,
152
+ :data => form.data.inspect,
153
+ :test_name => test_name).freeze)
154
+ end
155
+ end
156
+
157
+ def should_skip_url?(url)
158
+ return true if url.blank?
159
+ if @skip_uri_patterns.any? {|pattern| pattern =~ url}
160
+ log "Skipping #{url}"
161
+ return true
162
+ end
163
+ if url.length > max_url_length
164
+ log "Skipping long url #{url}"
165
+ return true
166
+ end
167
+ end
168
+
169
+ def should_skip_link?(link)
170
+ should_skip_url?(link.href) || @links_queued.member?(link)
171
+ end
172
+
173
+ def should_skip_form_submission?(fs)
174
+ should_skip_url?(fs.action) || @form_signatures_queued.member?(fs.signature)
175
+ end
176
+
177
+ def transform_url(url)
178
+ return unless url
179
+ url = @decoder.decode(url)
180
+ @transform_url_patterns.each do |pattern|
181
+ url = pattern[url]
182
+ end
183
+ url
184
+ end
185
+
186
+ def queue_link(dest, referrer = nil)
187
+ dest = Link.new(dest, self, referrer)
188
+ return if should_skip_link?(dest)
189
+ @crawl_queue << dest
190
+ @links_queued << dest
191
+ dest
192
+ end
193
+
194
+ def queue_form(form, referrer = nil)
195
+ fuzzers.each do |fuzzer|
196
+ fuzzer.mutate(Form.new(form, self, referrer)).each do |fs|
197
+ # fs = fuzzer.new(Form.new(form, self, referrer))
198
+ fs.action = transform_url(fs.action)
199
+ return if should_skip_form_submission?(fs)
200
+ @referrers[fs.action] = referrer if referrer
201
+ @crawl_queue << fs
202
+ @form_signatures_queued << fs.signature
203
+ end
204
+ end
205
+ end
206
+
207
+ def report_dir
208
+ File.join(rails_root, "tmp", "tarantula")
209
+ end
210
+
211
+ def generate_reports
212
+ errors = []
213
+ reporters.each do |reporter|
214
+ begin
215
+ reporter.finish_report(test_name)
216
+ rescue RuntimeError => e
217
+ errors << e
218
+ end
219
+ end
220
+ unless errors.empty?
221
+ raise errors.map(&:message).join("\n")
222
+ end
223
+ end
224
+
225
+ def report_results
226
+ puts "Crawled #{total_links_count} links and forms."
227
+ generate_reports
228
+ end
229
+
230
+ def total_links_count
231
+ @links_queued.size + @form_signatures_queued.size
232
+ end
233
+
234
+ def links_remaining_count
235
+ @crawl_queue.size
236
+ end
237
+
238
+ def links_completed_count
239
+ total_links_count - links_remaining_count
240
+ end
241
+
242
+ def blip(number = 0)
243
+ unless verbose
244
+ print "\r #{links_completed_count} of #{total_links_count} links completed " if @stdout_tty
245
+ timeout_if_too_long(number)
246
+ end
247
+ end
248
+
249
+ def timeout_if_too_long(number = 0)
250
+ if elasped_time_for_pass(number) > crawl_timeout
251
+ raise CrawlTimeout, "Exceeded crawl timeout of #{crawl_timeout} seconds - skipping to the next crawl..."
252
+ end
253
+ end
254
+ end