panmind-tarantula 0.3.3
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +49 -0
- data/LICENSE +20 -0
- data/README.rdoc +171 -0
- data/Rakefile +83 -0
- data/VERSION.yml +4 -0
- data/examples/example_helper.rb +57 -0
- data/examples/relevance/core_extensions/ellipsize_example.rb +19 -0
- data/examples/relevance/core_extensions/file_example.rb +8 -0
- data/examples/relevance/core_extensions/response_example.rb +29 -0
- data/examples/relevance/core_extensions/test_case_example.rb +20 -0
- data/examples/relevance/tarantula/attack_handler_example.rb +29 -0
- data/examples/relevance/tarantula/basic_attack_example.rb +12 -0
- data/examples/relevance/tarantula/crawler_example.rb +375 -0
- data/examples/relevance/tarantula/form_example.rb +50 -0
- data/examples/relevance/tarantula/form_submission_example.rb +171 -0
- data/examples/relevance/tarantula/html_document_handler_example.rb +43 -0
- data/examples/relevance/tarantula/html_report_helper_example.rb +46 -0
- data/examples/relevance/tarantula/html_reporter_example.rb +82 -0
- data/examples/relevance/tarantula/invalid_html_handler_example.rb +33 -0
- data/examples/relevance/tarantula/io_reporter_example.rb +11 -0
- data/examples/relevance/tarantula/link_example.rb +84 -0
- data/examples/relevance/tarantula/log_grabber_example.rb +26 -0
- data/examples/relevance/tarantula/rails_integration_proxy_example.rb +101 -0
- data/examples/relevance/tarantula/result_example.rb +85 -0
- data/examples/relevance/tarantula/tidy_handler_example.rb +58 -0
- data/examples/relevance/tarantula/transform_example.rb +20 -0
- data/examples/relevance/tarantula_example.rb +23 -0
- data/laf/images/header_bg.jpg +0 -0
- data/laf/images/logo.png +0 -0
- data/laf/images/tagline.png +0 -0
- data/laf/javascripts/jquery-1.2.3.js +3408 -0
- data/laf/javascripts/jquery-ui-tabs.js +890 -0
- data/laf/javascripts/jquery.tablesorter.js +861 -0
- data/laf/javascripts/tarantula.js +10 -0
- data/laf/stylesheets/tarantula.css +346 -0
- data/lib/relevance/core_extensions/ellipsize.rb +34 -0
- data/lib/relevance/core_extensions/file.rb +9 -0
- data/lib/relevance/core_extensions/metaclass.rb +78 -0
- data/lib/relevance/core_extensions/response.rb +9 -0
- data/lib/relevance/core_extensions/string_chars_fix.rb +11 -0
- data/lib/relevance/core_extensions/test_case.rb +19 -0
- data/lib/relevance/tarantula.rb +59 -0
- data/lib/relevance/tarantula/attack.rb +18 -0
- data/lib/relevance/tarantula/attack_handler.rb +37 -0
- data/lib/relevance/tarantula/basic_attack.rb +40 -0
- data/lib/relevance/tarantula/crawler.rb +254 -0
- data/lib/relevance/tarantula/detail.html.erb +81 -0
- data/lib/relevance/tarantula/form.rb +23 -0
- data/lib/relevance/tarantula/form_submission.rb +88 -0
- data/lib/relevance/tarantula/html_document_handler.rb +36 -0
- data/lib/relevance/tarantula/html_report_helper.rb +39 -0
- data/lib/relevance/tarantula/html_reporter.rb +105 -0
- data/lib/relevance/tarantula/index.html.erb +37 -0
- data/lib/relevance/tarantula/invalid_html_handler.rb +18 -0
- data/lib/relevance/tarantula/io_reporter.rb +34 -0
- data/lib/relevance/tarantula/link.rb +76 -0
- data/lib/relevance/tarantula/log_grabber.rb +16 -0
- data/lib/relevance/tarantula/rails_integration_proxy.rb +87 -0
- data/lib/relevance/tarantula/recording.rb +12 -0
- data/lib/relevance/tarantula/response.rb +13 -0
- data/lib/relevance/tarantula/result.rb +77 -0
- data/lib/relevance/tarantula/test_report.html.erb +32 -0
- data/lib/relevance/tarantula/tidy_handler.rb +32 -0
- data/lib/relevance/tarantula/transform.rb +17 -0
- data/tasks/tarantula_tasks.rake +42 -0
- data/template/tarantula_test.rb +22 -0
- metadata +213 -0
@@ -0,0 +1,9 @@
|
|
1
|
+
# dynamically mixed in to response objects
|
2
|
+
module Relevance::CoreExtensions::Response
|
3
|
+
def html?
|
4
|
+
# some versions of Rails integration tests don't set content type
|
5
|
+
# so we are treating nil as html. A better fix would be welcome here.
|
6
|
+
((content_type =~ %r{^text/html}) != nil) || content_type == nil
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
@@ -0,0 +1,11 @@
|
|
1
|
+
if RUBY_VERSION == "1.8.7" # fix interaction between Ruby 187 and Rails 202, so we can at least run the test suite on that combination
|
2
|
+
unless '1.9'.respond_to?(:force_encoding)
|
3
|
+
String.class_eval do
|
4
|
+
begin
|
5
|
+
remove_method :chars
|
6
|
+
rescue NameError
|
7
|
+
# OK
|
8
|
+
end
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'action_controller/integration'
|
2
|
+
|
3
|
+
module Relevance::CoreExtensions::TestCaseExtensions
|
4
|
+
|
5
|
+
def tarantula_crawl(integration_test, options = {})
|
6
|
+
url = options[:url] || "/"
|
7
|
+
t = tarantula_crawler(integration_test, options)
|
8
|
+
t.crawl url
|
9
|
+
end
|
10
|
+
|
11
|
+
def tarantula_crawler(integration_test, options = {})
|
12
|
+
Relevance::Tarantula::RailsIntegrationProxy.rails_integration_test(integration_test, options)
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
16
|
+
|
17
|
+
if defined? ActionController::IntegrationTest
|
18
|
+
ActionController::IntegrationTest.class_eval { include Relevance::CoreExtensions::TestCaseExtensions }
|
19
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
TARANTULA_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "../.."))
|
2
|
+
|
3
|
+
require 'forwardable'
|
4
|
+
require 'erb'
|
5
|
+
require 'active_support'
|
6
|
+
require 'action_controller'
|
7
|
+
|
8
|
+
# bringing in xss-shield requires a bunch of other dependencies
|
9
|
+
# still not certain about this, if it ruins your world please let me know
|
10
|
+
#xss_shield_path = File.join(TARANTULA_ROOT, %w{vendor xss-shield})
|
11
|
+
#$: << File.join(xss_shield_path, "lib")
|
12
|
+
#require File.join(xss_shield_path, "init")
|
13
|
+
|
14
|
+
require 'htmlentities'
|
15
|
+
|
16
|
+
module Relevance; end
|
17
|
+
module Relevance; module CoreExtensions; end; end
|
18
|
+
module Relevance
|
19
|
+
module Tarantula
|
20
|
+
def tarantula_home
|
21
|
+
File.expand_path(File.join(File.dirname(__FILE__), "../.."))
|
22
|
+
end
|
23
|
+
def log(msg)
|
24
|
+
puts msg if verbose
|
25
|
+
end
|
26
|
+
def rails_root
|
27
|
+
::RAILS_ROOT
|
28
|
+
end
|
29
|
+
def verbose
|
30
|
+
ENV["VERBOSE"]
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "test_case"))
|
36
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "ellipsize"))
|
37
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "file"))
|
38
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "response"))
|
39
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "metaclass"))
|
40
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "string_chars_fix"))
|
41
|
+
|
42
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_reporter"))
|
43
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_report_helper"))
|
44
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "io_reporter"))
|
45
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "recording"))
|
46
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "response"))
|
47
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "result"))
|
48
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "log_grabber"))
|
49
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "invalid_html_handler"))
|
50
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "transform"))
|
51
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "crawler"))
|
52
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "basic_attack"))
|
53
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form"))
|
54
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form_submission"))
|
55
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack"))
|
56
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_handler"))
|
57
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "link"))
|
58
|
+
|
59
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "tidy_handler")) if ENV['TIDY_PATH']
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class Relevance::Tarantula::Attack
|
2
|
+
HASHABLE_ATTRS = [:name, :input, :output, :description]
|
3
|
+
attr_accessor *HASHABLE_ATTRS
|
4
|
+
def initialize(hash)
|
5
|
+
hash.each do |k,v|
|
6
|
+
raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
|
7
|
+
self.instance_variable_set("@#{k}", v)
|
8
|
+
end
|
9
|
+
end
|
10
|
+
def ==(other)
|
11
|
+
Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
|
12
|
+
end
|
13
|
+
def input(input_field=nil)
|
14
|
+
@input
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'hpricot'
|
2
|
+
|
3
|
+
class Relevance::Tarantula::AttackHandler
|
4
|
+
include ERB::Util
|
5
|
+
|
6
|
+
def attacks
|
7
|
+
Relevance::Tarantula::FormSubmission.attacks.select(&:output)
|
8
|
+
end
|
9
|
+
|
10
|
+
def handle(result)
|
11
|
+
return unless attacks.size > 0
|
12
|
+
regexp = '(' + attacks.map {|a| Regexp.escape a.output}.join('|') + ')'
|
13
|
+
response = result.response
|
14
|
+
return unless response.html?
|
15
|
+
if n = (response.body =~ /#{regexp}/)
|
16
|
+
error_result = result.dup
|
17
|
+
error_result.success = false
|
18
|
+
error_result.description = "XSS error found, match was: #{h($1)}"
|
19
|
+
error_result.data = <<-STR
|
20
|
+
########################################################################
|
21
|
+
# Text around unescaped string: #{$1}
|
22
|
+
########################################################################
|
23
|
+
#{response.body[[0, n - 200].max , 400]}
|
24
|
+
|
25
|
+
|
26
|
+
|
27
|
+
|
28
|
+
|
29
|
+
########################################################################
|
30
|
+
# Attack information:
|
31
|
+
########################################################################
|
32
|
+
#{attacks.select {|a| a.output == $1}[0].to_yaml}
|
33
|
+
STR
|
34
|
+
error_result
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
class Relevance::Tarantula::BasicAttack
|
2
|
+
ATTRS = [:name, :output, :description]
|
3
|
+
|
4
|
+
attr_reader *ATTRS
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
@name = "Tarantula Basic Fuzzer"
|
8
|
+
@output = nil
|
9
|
+
@description = "Supplies purely random but simplistically generated form input."
|
10
|
+
end
|
11
|
+
|
12
|
+
def ==(other)
|
13
|
+
Relevance::Tarantula::BasicAttack === other && ATTRS.all? { |attr| send(attr) == other.send(attr)}
|
14
|
+
end
|
15
|
+
|
16
|
+
def input(input_field)
|
17
|
+
case input_field['name']
|
18
|
+
when /amount/ then random_int
|
19
|
+
when /_id$/ then random_whole_number
|
20
|
+
when /uploaded_data/ then nil
|
21
|
+
when nil then input['value']
|
22
|
+
else
|
23
|
+
random_int
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def big_number
|
28
|
+
10000 # arbitrary
|
29
|
+
end
|
30
|
+
|
31
|
+
def random_int
|
32
|
+
rand(big_number) - (big_number/2)
|
33
|
+
end
|
34
|
+
|
35
|
+
def random_whole_number
|
36
|
+
rand(big_number)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
|
@@ -0,0 +1,254 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_record/base'
|
3
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "rails_integration_proxy"))
|
4
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "html_document_handler.rb"))
|
5
|
+
|
6
|
+
class Relevance::Tarantula::Crawler
|
7
|
+
extend Forwardable
|
8
|
+
include Relevance::Tarantula
|
9
|
+
|
10
|
+
class CrawlTimeout < RuntimeError; end
|
11
|
+
|
12
|
+
attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
|
13
|
+
:reporters, :crawl_queue, :links_queued,
|
14
|
+
:form_signatures_queued, :max_url_length, :response_code_handler,
|
15
|
+
:times_to_crawl, :fuzzers, :test_name, :crawl_timeout
|
16
|
+
attr_reader :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
@max_url_length = 1024
|
20
|
+
@successes = []
|
21
|
+
@failures = []
|
22
|
+
@handlers = [@response_code_handler = Result]
|
23
|
+
@links_queued = Set.new
|
24
|
+
@form_signatures_queued = Set.new
|
25
|
+
@crawl_queue = []
|
26
|
+
@crawl_start_times, @crawl_end_times = [], []
|
27
|
+
@crawl_timeout = 20.minutes
|
28
|
+
@referrers = {}
|
29
|
+
@skip_uri_patterns = [
|
30
|
+
/^javascript/,
|
31
|
+
/^mailto/,
|
32
|
+
/^http/,
|
33
|
+
]
|
34
|
+
self.transform_url_patterns = [
|
35
|
+
[/#.*$/, '']
|
36
|
+
]
|
37
|
+
@reporters = [Relevance::Tarantula::IOReporter.new($stderr)]
|
38
|
+
@decoder = HTMLEntities.new
|
39
|
+
@times_to_crawl = 1
|
40
|
+
@fuzzers = [Relevance::Tarantula::FormSubmission]
|
41
|
+
|
42
|
+
@stdout_tty = $stdout.tty?
|
43
|
+
end
|
44
|
+
|
45
|
+
def method_missing(meth, *args)
|
46
|
+
super unless Result::ALLOW_NNN_FOR =~ meth.to_s
|
47
|
+
@response_code_handler.send(meth, *args)
|
48
|
+
end
|
49
|
+
|
50
|
+
def transform_url_patterns=(patterns)
|
51
|
+
@transform_url_patterns = patterns.map do |pattern|
|
52
|
+
Array === pattern ? Relevance::Tarantula::Transform.new(*pattern) : pattern
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def crawl(url = "/")
|
57
|
+
orig_links_queued = @links_queued.dup
|
58
|
+
orig_form_signatures_queued = @form_signatures_queued.dup
|
59
|
+
orig_crawl_queue = @crawl_queue.dup
|
60
|
+
@times_to_crawl.times do |num|
|
61
|
+
queue_link url
|
62
|
+
|
63
|
+
begin
|
64
|
+
do_crawl num
|
65
|
+
rescue CrawlTimeout => e
|
66
|
+
puts
|
67
|
+
puts e.message
|
68
|
+
end
|
69
|
+
|
70
|
+
puts "#{(num+1).ordinalize} crawl" if @times_to_crawl > 1
|
71
|
+
|
72
|
+
if num + 1 < @times_to_crawl
|
73
|
+
@links_queued = orig_links_queued
|
74
|
+
@form_signatures_queued = orig_form_signatures_queued
|
75
|
+
@crawl_queue = orig_crawl_queue
|
76
|
+
@referrers = {}
|
77
|
+
end
|
78
|
+
end
|
79
|
+
rescue Interrupt
|
80
|
+
$stderr.puts "CTRL-C"
|
81
|
+
ensure
|
82
|
+
report_results
|
83
|
+
end
|
84
|
+
|
85
|
+
def finished?
|
86
|
+
@crawl_queue.empty?
|
87
|
+
end
|
88
|
+
|
89
|
+
def do_crawl(number)
|
90
|
+
while (!finished?)
|
91
|
+
@crawl_start_times << Time.now
|
92
|
+
crawl_the_queue(number)
|
93
|
+
@crawl_end_times << Time.now
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def crawl_the_queue(number = 0)
|
98
|
+
while (request = @crawl_queue.pop)
|
99
|
+
request.crawl
|
100
|
+
blip(number)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def save_result(result)
|
105
|
+
reporters.each do |reporter|
|
106
|
+
reporter.report(result)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def handle_link_results(link, result)
|
111
|
+
handlers.each do |h|
|
112
|
+
begin
|
113
|
+
save_result h.handle(result)
|
114
|
+
rescue Exception => e
|
115
|
+
log "error handling #{link} #{e.message}"
|
116
|
+
# TODO: pass to results
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def follow(method, url, data=nil)
|
122
|
+
proxy.send(method, url, data)
|
123
|
+
end
|
124
|
+
|
125
|
+
def submit(method, action, data)
|
126
|
+
proxy.send(method, action, data)
|
127
|
+
end
|
128
|
+
|
129
|
+
def elasped_time_for_pass(num)
|
130
|
+
Time.now - crawl_start_times[num]
|
131
|
+
end
|
132
|
+
|
133
|
+
def grab_log!
|
134
|
+
@log_grabber && @log_grabber.grab!
|
135
|
+
end
|
136
|
+
|
137
|
+
def make_result(options)
|
138
|
+
defaults = {
|
139
|
+
:log => grab_log!,
|
140
|
+
:test_name => test_name
|
141
|
+
}
|
142
|
+
Result.new(defaults.merge(options)).freeze
|
143
|
+
end
|
144
|
+
|
145
|
+
def handle_form_results(form, response)
|
146
|
+
handlers.each do |h|
|
147
|
+
save_result h.handle(Result.new(:method => form.method,
|
148
|
+
:url => form.action,
|
149
|
+
:response => response,
|
150
|
+
:log => grab_log!,
|
151
|
+
:referrer => form.action,
|
152
|
+
:data => form.data.inspect,
|
153
|
+
:test_name => test_name).freeze)
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def should_skip_url?(url)
|
158
|
+
return true if url.blank?
|
159
|
+
if @skip_uri_patterns.any? {|pattern| pattern =~ url}
|
160
|
+
log "Skipping #{url}"
|
161
|
+
return true
|
162
|
+
end
|
163
|
+
if url.length > max_url_length
|
164
|
+
log "Skipping long url #{url}"
|
165
|
+
return true
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def should_skip_link?(link)
|
170
|
+
should_skip_url?(link.href) || @links_queued.member?(link)
|
171
|
+
end
|
172
|
+
|
173
|
+
def should_skip_form_submission?(fs)
|
174
|
+
should_skip_url?(fs.action) || @form_signatures_queued.member?(fs.signature)
|
175
|
+
end
|
176
|
+
|
177
|
+
def transform_url(url)
|
178
|
+
return unless url
|
179
|
+
url = @decoder.decode(url)
|
180
|
+
@transform_url_patterns.each do |pattern|
|
181
|
+
url = pattern[url]
|
182
|
+
end
|
183
|
+
url
|
184
|
+
end
|
185
|
+
|
186
|
+
def queue_link(dest, referrer = nil)
|
187
|
+
dest = Link.new(dest, self, referrer)
|
188
|
+
return if should_skip_link?(dest)
|
189
|
+
@crawl_queue << dest
|
190
|
+
@links_queued << dest
|
191
|
+
dest
|
192
|
+
end
|
193
|
+
|
194
|
+
def queue_form(form, referrer = nil)
|
195
|
+
fuzzers.each do |fuzzer|
|
196
|
+
fuzzer.mutate(Form.new(form, self, referrer)).each do |fs|
|
197
|
+
# fs = fuzzer.new(Form.new(form, self, referrer))
|
198
|
+
fs.action = transform_url(fs.action)
|
199
|
+
return if should_skip_form_submission?(fs)
|
200
|
+
@referrers[fs.action] = referrer if referrer
|
201
|
+
@crawl_queue << fs
|
202
|
+
@form_signatures_queued << fs.signature
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
def report_dir
|
208
|
+
File.join(rails_root, "tmp", "tarantula")
|
209
|
+
end
|
210
|
+
|
211
|
+
def generate_reports
|
212
|
+
errors = []
|
213
|
+
reporters.each do |reporter|
|
214
|
+
begin
|
215
|
+
reporter.finish_report(test_name)
|
216
|
+
rescue RuntimeError => e
|
217
|
+
errors << e
|
218
|
+
end
|
219
|
+
end
|
220
|
+
unless errors.empty?
|
221
|
+
raise errors.map(&:message).join("\n")
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
def report_results
|
226
|
+
puts "Crawled #{total_links_count} links and forms."
|
227
|
+
generate_reports
|
228
|
+
end
|
229
|
+
|
230
|
+
def total_links_count
|
231
|
+
@links_queued.size + @form_signatures_queued.size
|
232
|
+
end
|
233
|
+
|
234
|
+
def links_remaining_count
|
235
|
+
@crawl_queue.size
|
236
|
+
end
|
237
|
+
|
238
|
+
def links_completed_count
|
239
|
+
total_links_count - links_remaining_count
|
240
|
+
end
|
241
|
+
|
242
|
+
def blip(number = 0)
|
243
|
+
unless verbose
|
244
|
+
print "\r #{links_completed_count} of #{total_links_count} links completed " if @stdout_tty
|
245
|
+
timeout_if_too_long(number)
|
246
|
+
end
|
247
|
+
end
|
248
|
+
|
249
|
+
def timeout_if_too_long(number = 0)
|
250
|
+
if elasped_time_for_pass(number) > crawl_timeout
|
251
|
+
raise CrawlTimeout, "Exceeded crawl timeout of #{crawl_timeout} seconds - skipping to the next crawl..."
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|