codez-tarantula 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.autotest +14 -0
- data/.gitignore +12 -0
- data/.travis.yml +7 -0
- data/CHANGELOG +64 -0
- data/DSL_EXAMPLES.md +120 -0
- data/Gemfile +2 -0
- data/LICENSE +20 -0
- data/README.rdoc +136 -0
- data/Rakefile +36 -0
- data/ci/rails2.gemfile +4 -0
- data/ci/rails3.gemfile +4 -0
- data/laf/images/header_bg.jpg +0 -0
- data/laf/images/logo.png +0 -0
- data/laf/images/tagline.png +0 -0
- data/laf/javascripts/jquery-1.2.3.js +3408 -0
- data/laf/javascripts/jquery-ui-tabs.js +890 -0
- data/laf/javascripts/jquery.tablesorter.js +861 -0
- data/laf/javascripts/tarantula.js +10 -0
- data/laf/stylesheets/tarantula.css +346 -0
- data/lib/relevance/core_extensions/ellipsize.rb +38 -0
- data/lib/relevance/core_extensions/file.rb +15 -0
- data/lib/relevance/core_extensions/metaclass.rb +78 -0
- data/lib/relevance/core_extensions/response.rb +14 -0
- data/lib/relevance/core_extensions/test_case.rb +21 -0
- data/lib/relevance/tarantula.rb +55 -0
- data/lib/relevance/tarantula/attack.rb +22 -0
- data/lib/relevance/tarantula/attack_handler.rb +43 -0
- data/lib/relevance/tarantula/basic_attack.rb +44 -0
- data/lib/relevance/tarantula/crawler.rb +271 -0
- data/lib/relevance/tarantula/detail.html.erb +81 -0
- data/lib/relevance/tarantula/form.rb +29 -0
- data/lib/relevance/tarantula/form_submission.rb +98 -0
- data/lib/relevance/tarantula/html_document_handler.rb +42 -0
- data/lib/relevance/tarantula/html_report_helper.rb +46 -0
- data/lib/relevance/tarantula/html_reporter.rb +111 -0
- data/lib/relevance/tarantula/index.html.erb +37 -0
- data/lib/relevance/tarantula/invalid_html_handler.rb +27 -0
- data/lib/relevance/tarantula/io_reporter.rb +40 -0
- data/lib/relevance/tarantula/link.rb +105 -0
- data/lib/relevance/tarantula/log_grabber.rb +22 -0
- data/lib/relevance/tarantula/rails_integration_proxy.rb +90 -0
- data/lib/relevance/tarantula/recording.rb +12 -0
- data/lib/relevance/tarantula/response.rb +19 -0
- data/lib/relevance/tarantula/result.rb +83 -0
- data/lib/relevance/tarantula/test_report.html.erb +32 -0
- data/lib/relevance/tarantula/tidy_handler.rb +35 -0
- data/lib/relevance/tarantula/transform.rb +21 -0
- data/lib/relevance/tarantula/version.rb +5 -0
- data/lib/relevance/tasks/tarantula_tasks.rake +42 -0
- data/lib/tarantula-rails3.rb +9 -0
- data/spec/relevance/core_extensions/ellipsize_spec.rb +19 -0
- data/spec/relevance/core_extensions/file_spec.rb +7 -0
- data/spec/relevance/core_extensions/response_spec.rb +48 -0
- data/spec/relevance/core_extensions/test_case_spec.rb +19 -0
- data/spec/relevance/tarantula/attack_handler_spec.rb +29 -0
- data/spec/relevance/tarantula/basic_attack_spec.rb +12 -0
- data/spec/relevance/tarantula/crawler_spec.rb +409 -0
- data/spec/relevance/tarantula/form_spec.rb +50 -0
- data/spec/relevance/tarantula/form_submission_spec.rb +171 -0
- data/spec/relevance/tarantula/html_document_handler_spec.rb +43 -0
- data/spec/relevance/tarantula/html_report_helper_spec.rb +46 -0
- data/spec/relevance/tarantula/html_reporter_spec.rb +82 -0
- data/spec/relevance/tarantula/invalid_html_handler_spec.rb +33 -0
- data/spec/relevance/tarantula/io_reporter_spec.rb +11 -0
- data/spec/relevance/tarantula/link_spec.rb +132 -0
- data/spec/relevance/tarantula/log_grabber_spec.rb +26 -0
- data/spec/relevance/tarantula/rails_integration_proxy_spec.rb +100 -0
- data/spec/relevance/tarantula/result_spec.rb +85 -0
- data/spec/relevance/tarantula/tidy_handler_spec.rb +58 -0
- data/spec/relevance/tarantula/transform_spec.rb +20 -0
- data/spec/relevance/tarantula_spec.rb +23 -0
- data/spec/spec_helper.rb +43 -0
- data/tarantula.gemspec +25 -0
- data/template/tarantula_test.rb +22 -0
- data/vendor/xss-shield/MIT-LICENSE +20 -0
- data/vendor/xss-shield/README +76 -0
- data/vendor/xss-shield/init.rb +16 -0
- data/vendor/xss-shield/lib/xss_shield.rb +6 -0
- data/vendor/xss-shield/lib/xss_shield/erb_hacks.rb +111 -0
- data/vendor/xss-shield/lib/xss_shield/haml_hacks.rb +42 -0
- data/vendor/xss-shield/lib/xss_shield/safe_string.rb +47 -0
- data/vendor/xss-shield/lib/xss_shield/secure_helpers.rb +40 -0
- data/vendor/xss-shield/test/test_actionview_integration.rb +40 -0
- data/vendor/xss-shield/test/test_erb.rb +44 -0
- data/vendor/xss-shield/test/test_haml.rb +43 -0
- data/vendor/xss-shield/test/test_helpers.rb +25 -0
- data/vendor/xss-shield/test/test_safe_string.rb +55 -0
- metadata +247 -0
@@ -0,0 +1,14 @@
|
|
1
|
+
# dynamically mixed in to response objects
|
2
|
+
module Relevance
|
3
|
+
module CoreExtensions
|
4
|
+
module Response
|
5
|
+
def html?
|
6
|
+
# some versions of Rails integration tests don't set content type
|
7
|
+
# so we are treating nil as html. A better fix would be welcome here.
|
8
|
+
(content_type.respond_to?(:html?) ?
|
9
|
+
content_type.html? : content_type =~ %r{^text/html}) ||
|
10
|
+
content_type.nil?
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Relevance
|
2
|
+
module CoreExtensions
|
3
|
+
|
4
|
+
module TestCaseExtensions
|
5
|
+
def tarantula_crawl(integration_test, options = {})
|
6
|
+
url = options[:url] || "/"
|
7
|
+
t = tarantula_crawler(integration_test, options)
|
8
|
+
t.crawl url
|
9
|
+
end
|
10
|
+
|
11
|
+
def tarantula_crawler(integration_test, options = {})
|
12
|
+
Relevance::Tarantula::RailsIntegrationProxy.rails_integration_test(integration_test, options)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
if defined? ActionController::IntegrationTest
|
20
|
+
ActionController::IntegrationTest.class_eval { include Relevance::CoreExtensions::TestCaseExtensions }
|
21
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
require 'erb'
|
3
|
+
require 'active_support'
|
4
|
+
require 'action_controller'
|
5
|
+
require 'htmlentities'
|
6
|
+
|
7
|
+
if RUBY_VERSION < '1.9.1'
|
8
|
+
warn "***************************************************"
|
9
|
+
warn "tarantula will stop supporting ruby 1.8.x in 0.6.0."
|
10
|
+
warn "***************************************************"
|
11
|
+
end
|
12
|
+
|
13
|
+
module Relevance; end
|
14
|
+
module Relevance; module CoreExtensions; end; end
|
15
|
+
module Relevance
|
16
|
+
module Tarantula
|
17
|
+
def tarantula_home
|
18
|
+
File.expand_path(File.join(File.dirname(__FILE__), "../.."))
|
19
|
+
end
|
20
|
+
def log(msg)
|
21
|
+
puts msg if verbose
|
22
|
+
end
|
23
|
+
def rails_root
|
24
|
+
::Rails.root.to_s
|
25
|
+
end
|
26
|
+
def verbose
|
27
|
+
ENV["VERBOSE"]
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
require "relevance/core_extensions/test_case"
|
33
|
+
require "relevance/core_extensions/ellipsize"
|
34
|
+
require "relevance/core_extensions/file"
|
35
|
+
require "relevance/core_extensions/response"
|
36
|
+
require "relevance/core_extensions/metaclass"
|
37
|
+
|
38
|
+
require "relevance/tarantula/html_reporter"
|
39
|
+
require "relevance/tarantula/html_report_helper"
|
40
|
+
require "relevance/tarantula/io_reporter"
|
41
|
+
require "relevance/tarantula/recording"
|
42
|
+
require "relevance/tarantula/response"
|
43
|
+
require "relevance/tarantula/result"
|
44
|
+
require "relevance/tarantula/log_grabber"
|
45
|
+
require "relevance/tarantula/invalid_html_handler"
|
46
|
+
require "relevance/tarantula/transform"
|
47
|
+
require "relevance/tarantula/crawler"
|
48
|
+
require "relevance/tarantula/basic_attack"
|
49
|
+
require "relevance/tarantula/form"
|
50
|
+
require "relevance/tarantula/form_submission"
|
51
|
+
require "relevance/tarantula/attack"
|
52
|
+
require "relevance/tarantula/attack_handler"
|
53
|
+
require "relevance/tarantula/link"
|
54
|
+
|
55
|
+
require "relevance/tarantula/tidy_handler" if ENV['TIDY_PATH']
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Relevance
|
2
|
+
module Tarantula
|
3
|
+
|
4
|
+
class Attack
|
5
|
+
HASHABLE_ATTRS = [:name, :input, :output, :description]
|
6
|
+
attr_accessor *HASHABLE_ATTRS
|
7
|
+
def initialize(hash)
|
8
|
+
hash.each do |k,v|
|
9
|
+
raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
|
10
|
+
self.instance_variable_set("@#{k}", v)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
def ==(other)
|
14
|
+
Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
|
15
|
+
end
|
16
|
+
def input(input_field=nil)
|
17
|
+
@input
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'hpricot'
|
2
|
+
|
3
|
+
module Relevance
|
4
|
+
module Tarantula
|
5
|
+
|
6
|
+
class AttackHandler
|
7
|
+
include ERB::Util
|
8
|
+
|
9
|
+
def attacks
|
10
|
+
Relevance::Tarantula::FormSubmission.attacks.select(&:output)
|
11
|
+
end
|
12
|
+
|
13
|
+
def handle(result)
|
14
|
+
return unless attacks.size > 0
|
15
|
+
regexp = '(' + attacks.map {|a| Regexp.escape a.output}.join('|') + ')'
|
16
|
+
response = result.response
|
17
|
+
return unless response.html?
|
18
|
+
if n = (response.body =~ /#{regexp}/)
|
19
|
+
error_result = result.dup
|
20
|
+
error_result.success = false
|
21
|
+
error_result.description = "XSS error found, match was: #{h($1)}"
|
22
|
+
error_result.data = <<-STR
|
23
|
+
########################################################################
|
24
|
+
# Text around unescaped string: #{$1}
|
25
|
+
########################################################################
|
26
|
+
#{response.body[[0, n - 200].max , 400]}
|
27
|
+
|
28
|
+
|
29
|
+
|
30
|
+
|
31
|
+
|
32
|
+
########################################################################
|
33
|
+
# Attack information:
|
34
|
+
########################################################################
|
35
|
+
#{attacks.select {|a| a.output == $1}[0].to_yaml}
|
36
|
+
STR
|
37
|
+
error_result
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
end
|
43
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Relevance
|
2
|
+
module Tarantula
|
3
|
+
|
4
|
+
class BasicAttack
|
5
|
+
ATTRS = [:name, :output, :description]
|
6
|
+
|
7
|
+
attr_reader *ATTRS
|
8
|
+
|
9
|
+
def initialize
|
10
|
+
@name = "Tarantula Basic Fuzzer"
|
11
|
+
@output = nil
|
12
|
+
@description = "Supplies purely random but simplistically generated form input."
|
13
|
+
end
|
14
|
+
|
15
|
+
def ==(other)
|
16
|
+
Relevance::Tarantula::BasicAttack === other && ATTRS.all? { |attr| send(attr) == other.send(attr)}
|
17
|
+
end
|
18
|
+
|
19
|
+
def input(input_field)
|
20
|
+
case input_field['name']
|
21
|
+
when /amount/ then random_int
|
22
|
+
when /_id$/ then random_whole_number
|
23
|
+
when /uploaded_data/ then nil
|
24
|
+
when nil then input['value']
|
25
|
+
else
|
26
|
+
random_int
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def big_number
|
31
|
+
10000 # arbitrary
|
32
|
+
end
|
33
|
+
|
34
|
+
def random_int
|
35
|
+
rand(big_number) - (big_number/2)
|
36
|
+
end
|
37
|
+
|
38
|
+
def random_whole_number
|
39
|
+
rand(big_number)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,271 @@
|
|
1
|
+
require 'active_record'
|
2
|
+
require 'active_record/base'
|
3
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "rails_integration_proxy"))
|
4
|
+
require File.expand_path(File.join(File.dirname(__FILE__), "html_document_handler.rb"))
|
5
|
+
|
6
|
+
module Relevance
|
7
|
+
module Tarantula
|
8
|
+
|
9
|
+
class Crawler
|
10
|
+
extend Forwardable
|
11
|
+
include Relevance::Tarantula
|
12
|
+
|
13
|
+
class CrawlTimeout < RuntimeError; end
|
14
|
+
|
15
|
+
attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
|
16
|
+
:reporters, :crawl_queue, :links_queued,
|
17
|
+
:form_signatures_queued, :max_url_length, :response_code_handler,
|
18
|
+
:times_to_crawl, :fuzzers, :test_name, :crawl_timeout
|
19
|
+
attr_reader :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
|
20
|
+
|
21
|
+
def initialize
|
22
|
+
@max_url_length = 1024
|
23
|
+
@successes = []
|
24
|
+
@failures = []
|
25
|
+
@handlers = [@response_code_handler = Result]
|
26
|
+
@links_queued = Set.new
|
27
|
+
@form_signatures_queued = Set.new
|
28
|
+
@crawl_queue = []
|
29
|
+
@crawl_start_times, @crawl_end_times = [], []
|
30
|
+
@crawl_timeout = 20.minutes
|
31
|
+
@referrers = {}
|
32
|
+
@skip_uri_patterns = [
|
33
|
+
/^javascript/,
|
34
|
+
/^mailto/,
|
35
|
+
/^http/,
|
36
|
+
]
|
37
|
+
self.transform_url_patterns = [
|
38
|
+
[/#.*$/, '']
|
39
|
+
]
|
40
|
+
@reporters = [Relevance::Tarantula::IOReporter.new($stderr)]
|
41
|
+
@decoder = HTMLEntities.new
|
42
|
+
@times_to_crawl = 1
|
43
|
+
@fuzzers = [Relevance::Tarantula::FormSubmission]
|
44
|
+
|
45
|
+
@stdout_tty = $stdout.tty?
|
46
|
+
end
|
47
|
+
|
48
|
+
def method_missing(meth, *args)
|
49
|
+
super unless Result::ALLOW_NNN_FOR =~ meth.to_s
|
50
|
+
@response_code_handler.send(meth, *args)
|
51
|
+
end
|
52
|
+
|
53
|
+
def transform_url_patterns=(patterns)
|
54
|
+
@transform_url_patterns = patterns.map do |pattern|
|
55
|
+
Array === pattern ? Relevance::Tarantula::Transform.new(*pattern) : pattern
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
def crawl(url = "/")
|
60
|
+
orig_links_queued = @links_queued.dup
|
61
|
+
orig_form_signatures_queued = @form_signatures_queued.dup
|
62
|
+
orig_crawl_queue = @crawl_queue.dup
|
63
|
+
@times_to_crawl.times do |num|
|
64
|
+
queue_link url
|
65
|
+
|
66
|
+
begin
|
67
|
+
do_crawl num
|
68
|
+
rescue CrawlTimeout => e
|
69
|
+
puts
|
70
|
+
puts e.message
|
71
|
+
end
|
72
|
+
|
73
|
+
puts "#{ActiveSupport::Inflector.ordinalize((num+1))} crawl" if @times_to_crawl > 1
|
74
|
+
|
75
|
+
if num + 1 < @times_to_crawl
|
76
|
+
@links_queued = orig_links_queued
|
77
|
+
@form_signatures_queued = orig_form_signatures_queued
|
78
|
+
@crawl_queue = orig_crawl_queue
|
79
|
+
@referrers = {}
|
80
|
+
end
|
81
|
+
end
|
82
|
+
rescue Interrupt
|
83
|
+
$stderr.puts "CTRL-C"
|
84
|
+
ensure
|
85
|
+
report_results
|
86
|
+
end
|
87
|
+
|
88
|
+
def finished?
|
89
|
+
@crawl_queue.empty?
|
90
|
+
end
|
91
|
+
|
92
|
+
def do_crawl(number)
|
93
|
+
while (!finished?)
|
94
|
+
@crawl_start_times << Time.now
|
95
|
+
crawl_the_queue(number)
|
96
|
+
@crawl_end_times << Time.now
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def crawl_the_queue(number = 0)
|
101
|
+
while (request = @crawl_queue.shift)
|
102
|
+
request.crawl
|
103
|
+
blip(number)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def save_result(result)
|
108
|
+
reporters.each do |reporter|
|
109
|
+
reporter.report(result)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def handle_link_results(link, result)
|
114
|
+
handlers.each do |h|
|
115
|
+
begin
|
116
|
+
save_result h.handle(result)
|
117
|
+
rescue Exception => e
|
118
|
+
log "error handling #{link} #{e.message}"
|
119
|
+
# TODO: pass to results
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
def follow(method, url, data=nil)
|
125
|
+
proxy.send(method, url, data)
|
126
|
+
end
|
127
|
+
|
128
|
+
def submit(method, action, data)
|
129
|
+
proxy.send(method, action, data)
|
130
|
+
end
|
131
|
+
|
132
|
+
def elasped_time_for_pass(num)
|
133
|
+
Time.now - crawl_start_times[num]
|
134
|
+
end
|
135
|
+
|
136
|
+
def grab_log!
|
137
|
+
@log_grabber && @log_grabber.grab!
|
138
|
+
end
|
139
|
+
|
140
|
+
def make_result(options)
|
141
|
+
defaults = {
|
142
|
+
:log => grab_log!,
|
143
|
+
:test_name => test_name
|
144
|
+
}
|
145
|
+
Result.new(defaults.merge(options)).freeze
|
146
|
+
end
|
147
|
+
|
148
|
+
def handle_form_results(form, response)
|
149
|
+
handlers.each do |h|
|
150
|
+
save_result h.handle(Result.new(:method => form.meth,
|
151
|
+
:url => form.action,
|
152
|
+
:response => response,
|
153
|
+
:log => grab_log!,
|
154
|
+
:referrer => form.action,
|
155
|
+
:data => form.data.inspect,
|
156
|
+
:test_name => test_name).freeze)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def should_skip_url?(url)
|
161
|
+
return true if url.blank?
|
162
|
+
if @skip_uri_patterns.any? {|pattern| pattern =~ url}
|
163
|
+
log "Skipping #{url}"
|
164
|
+
return true
|
165
|
+
end
|
166
|
+
if url.length > max_url_length
|
167
|
+
log "Skipping long url #{url}"
|
168
|
+
return true
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
def should_skip_link?(link)
|
173
|
+
should_skip_url?(link.href) || @links_queued.member?(link)
|
174
|
+
end
|
175
|
+
|
176
|
+
def should_skip_form_submission?(fs)
|
177
|
+
should_skip_url?(fs.action) || @form_signatures_queued.member?(fs.signature)
|
178
|
+
end
|
179
|
+
|
180
|
+
def transform_url(url)
|
181
|
+
return unless url
|
182
|
+
url = @decoder.decode(url)
|
183
|
+
@transform_url_patterns.each do |pattern|
|
184
|
+
url = pattern[url]
|
185
|
+
end
|
186
|
+
url
|
187
|
+
end
|
188
|
+
|
189
|
+
def queue_link(dest, referrer = nil)
|
190
|
+
dest = Link.new(dest, self, referrer)
|
191
|
+
return if should_skip_link?(dest)
|
192
|
+
append_to_queue(dest)
|
193
|
+
@links_queued << dest
|
194
|
+
dest
|
195
|
+
end
|
196
|
+
|
197
|
+
def queue_form(form, referrer = nil)
|
198
|
+
fuzzers.each do |fuzzer|
|
199
|
+
fuzzer.mutate(Form.new(form, self, referrer)).each do |fs|
|
200
|
+
# fs = fuzzer.new(Form.new(form, self, referrer))
|
201
|
+
fs.action = transform_url(fs.action)
|
202
|
+
return if should_skip_form_submission?(fs)
|
203
|
+
@referrers[fs.action] = referrer if referrer
|
204
|
+
append_to_queue(fs)
|
205
|
+
@form_signatures_queued << fs.signature
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
# append delete requests to the end of the queue, all others just before the first delete request
|
211
|
+
def append_to_queue(request)
|
212
|
+
if request.meth != 'delete' && index = @crawl_queue.index {|r| r.meth == 'delete' }
|
213
|
+
@crawl_queue.insert(index, request)
|
214
|
+
elsif request.meth == 'delete' && parent_index = @crawl_queue.index {|r| r.meth == 'delete' && request.url.start_with?(r.url) }
|
215
|
+
@crawl_queue.insert(parent_index, request)
|
216
|
+
else
|
217
|
+
@crawl_queue << request
|
218
|
+
end
|
219
|
+
end
|
220
|
+
|
221
|
+
def report_dir
|
222
|
+
File.join(rails_root, "tmp", "tarantula")
|
223
|
+
end
|
224
|
+
|
225
|
+
def generate_reports
|
226
|
+
errors = []
|
227
|
+
reporters.each do |reporter|
|
228
|
+
begin
|
229
|
+
reporter.finish_report(test_name)
|
230
|
+
rescue RuntimeError => e
|
231
|
+
errors << e
|
232
|
+
end
|
233
|
+
end
|
234
|
+
unless errors.empty?
|
235
|
+
raise errors.map(&:message).join("\n")
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
def report_results
|
240
|
+
puts "Crawled #{total_links_count} links and forms."
|
241
|
+
generate_reports
|
242
|
+
end
|
243
|
+
|
244
|
+
def total_links_count
|
245
|
+
@links_queued.size + @form_signatures_queued.size
|
246
|
+
end
|
247
|
+
|
248
|
+
def links_remaining_count
|
249
|
+
@crawl_queue.size
|
250
|
+
end
|
251
|
+
|
252
|
+
def links_completed_count
|
253
|
+
total_links_count - links_remaining_count
|
254
|
+
end
|
255
|
+
|
256
|
+
def blip(number = 0)
|
257
|
+
unless verbose
|
258
|
+
print "\r #{links_completed_count} of #{total_links_count} links completed " if @stdout_tty
|
259
|
+
timeout_if_too_long(number)
|
260
|
+
end
|
261
|
+
end
|
262
|
+
|
263
|
+
def timeout_if_too_long(number = 0)
|
264
|
+
if elasped_time_for_pass(number) > crawl_timeout
|
265
|
+
raise CrawlTimeout, "Exceeded crawl timeout of #{crawl_timeout} seconds - skipping to the next crawl..."
|
266
|
+
end
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
270
|
+
end
|
271
|
+
end
|