tarantula 0.3.3 → 0.4.0

data/lib/relevance/core_extensions/file.rb

@@ -1,9 +1,15 @@
-module Relevance::CoreExtensions::File 
-  def extension(path)
-    extname(path)[1..-1]
+module Relevance
+  module CoreExtensions
+
+    module File 
+      def extension(path)
+        extname(path)[1..-1]
+      end
+    end
+
   end
 end
 
 class File
   extend Relevance::CoreExtensions::File
-end
+end

data/lib/relevance/core_extensions/response.rb

@@ -1,9 +1,12 @@
 # dynamically mixed in to response objects
-module Relevance::CoreExtensions::Response 
-  def html?                                           
-    # some versions of Rails integration tests don't set content type
-    # so we are treating nil as html. A better fix would be welcome here.
-    ((content_type =~ %r{^text/html}) != nil)  || content_type == nil
+module Relevance
+  module CoreExtensions
+    module Response 
+      def html?                                           
+        # some versions of Rails integration tests don't set content type
+        # so we are treating nil as html. A better fix would be welcome here.
+        ((content_type =~ %r{^text/html}) != nil)  || content_type == nil
+      end
+    end
   end
 end
-

data/lib/relevance/core_extensions/test_case.rb

@@ -1,19 +1,21 @@
-require 'action_controller/integration'
+module Relevance
+  module CoreExtensions
 
-module Relevance::CoreExtensions::TestCaseExtensions
+    module TestCaseExtensions
+      def tarantula_crawl(integration_test, options = {})
+        url = options[:url] || "/"
+        t = tarantula_crawler(integration_test, options)
+        t.crawl url
+      end
+
+      def tarantula_crawler(integration_test, options = {})
+        Relevance::Tarantula::RailsIntegrationProxy.rails_integration_test(integration_test, options)
+      end
+    end
 
-  def tarantula_crawl(integration_test, options = {})
-    url = options[:url] || "/"
-    t = tarantula_crawler(integration_test, options)
-    t.crawl url
-  end
-  
-  def tarantula_crawler(integration_test, options = {})
-    Relevance::Tarantula::RailsIntegrationProxy.rails_integration_test(integration_test, options)
   end
-  
 end
 
 if defined? ActionController::IntegrationTest
   ActionController::IntegrationTest.class_eval { include Relevance::CoreExtensions::TestCaseExtensions }
-end
+end

data/lib/relevance/tarantula.rb

@@ -4,7 +4,6 @@ require 'forwardable'
 require 'erb'
 require 'active_support'
 require 'action_controller'
-
 # bringing in xss-shield requires a bunch of other dependencies
 # still not certain about this, if it ruins your world please let me know
 #xss_shield_path = File.join(TARANTULA_ROOT, %w{vendor xss-shield})
@@ -24,7 +23,7 @@ module Relevance
       puts msg if verbose
     end
     def rails_root
-      ::RAILS_ROOT
+      ::Rails.root.to_s
     end
     def verbose
       ENV["VERBOSE"]
@@ -32,28 +31,28 @@ module Relevance
   end
 end
 
-require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "test_case"))
-require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "ellipsize"))
-require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "file"))
-require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "response"))
-require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "metaclass"))
-require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "string_chars_fix"))
+require "relevance/core_extensions/test_case"
+require "relevance/core_extensions/ellipsize"
+require "relevance/core_extensions/file"
+require "relevance/core_extensions/response"
+require "relevance/core_extensions/metaclass"
+require "relevance/core_extensions/string_chars_fix"
 
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_reporter"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_report_helper"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "io_reporter"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "recording"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "response"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "result"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "log_grabber"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "invalid_html_handler"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "transform"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "crawler"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "basic_attack"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form_submission"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_handler"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "link"))
+require "relevance/tarantula/html_reporter"
+require "relevance/tarantula/html_report_helper"
+require "relevance/tarantula/io_reporter"
+require "relevance/tarantula/recording"
+require "relevance/tarantula/response"
+require "relevance/tarantula/result"
+require "relevance/tarantula/log_grabber"
+require "relevance/tarantula/invalid_html_handler"
+require "relevance/tarantula/transform"
+require "relevance/tarantula/crawler"
+require "relevance/tarantula/basic_attack"
+require "relevance/tarantula/form"
+require "relevance/tarantula/form_submission"
+require "relevance/tarantula/attack"
+require "relevance/tarantula/attack_handler"
+require "relevance/tarantula/link"
 
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "tidy_handler")) if ENV['TIDY_PATH']
+require "relevance/tarantula/tidy_handler" if ENV['TIDY_PATH']

data/lib/relevance/tarantula/attack.rb

@@ -1,18 +1,22 @@
-class Relevance::Tarantula::Attack
-  HASHABLE_ATTRS = [:name, :input, :output, :description]
-  attr_accessor *HASHABLE_ATTRS
-  def initialize(hash)
-    hash.each do |k,v|
-      raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
-      self.instance_variable_set("@#{k}", v)
+module Relevance
+  module Tarantula
+
+    class Attack
+      HASHABLE_ATTRS = [:name, :input, :output, :description]
+      attr_accessor *HASHABLE_ATTRS
+      def initialize(hash)
+        hash.each do |k,v|
+          raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
+          self.instance_variable_set("@#{k}", v)
+        end
+      end
+      def ==(other)
+        Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
+      end
+      def input(input_field=nil)
+        @input
+      end
     end
-  end
-  def ==(other)
-    Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
-  end
-  def input(input_field=nil)
-    @input
+
   end
 end
-
-

data/lib/relevance/tarantula/attack_handler.rb

@@ -1,37 +1,43 @@
 require 'hpricot'
 
-class Relevance::Tarantula::AttackHandler 
-  include ERB::Util
-  
-  def attacks
-    Relevance::Tarantula::FormSubmission.attacks.select(&:output)
-  end
-  
-  def handle(result)
-    return unless attacks.size > 0
-    regexp = '(' + attacks.map {|a| Regexp.escape a.output}.join('|') + ')'
-    response = result.response
-    return unless response.html?
-    if n = (response.body =~ /#{regexp}/)
-      error_result = result.dup
-      error_result.success = false
-      error_result.description = "XSS error found, match was: #{h($1)}"
-      error_result.data = <<-STR
+module Relevance
+  module Tarantula
+
+    class AttackHandler 
+      include ERB::Util
+
+      def attacks
+        Relevance::Tarantula::FormSubmission.attacks.select(&:output)
+      end
+
+      def handle(result)
+        return unless attacks.size > 0
+        regexp = '(' + attacks.map {|a| Regexp.escape a.output}.join('|') + ')'
+        response = result.response
+        return unless response.html?
+        if n = (response.body =~ /#{regexp}/)
+          error_result = result.dup
+          error_result.success = false
+          error_result.description = "XSS error found, match was: #{h($1)}"
+          error_result.data = <<-STR
         ########################################################################
         # Text around unescaped string: #{$1}
         ########################################################################
-        #{response.body[[0, n - 200].max , 400]}
-        
-        
-        
-        
-        
+          #{response.body[[0, n - 200].max , 400]}
+
+
+
+
+
         ########################################################################
         # Attack information:
         ########################################################################
-        #{attacks.select {|a| a.output == $1}[0].to_yaml}
-      STR
-      error_result
+          #{attacks.select {|a| a.output == $1}[0].to_yaml}
+          STR
+          error_result
+        end
+      end
     end
+
   end
 end

data/lib/relevance/tarantula/basic_attack.rb

@@ -1,40 +1,44 @@
-class Relevance::Tarantula::BasicAttack
-  ATTRS = [:name, :output, :description]
+module Relevance
+  module Tarantula
 
-  attr_reader *ATTRS
+    class BasicAttack
+      ATTRS = [:name, :output, :description]
 
-  def initialize
-    @name = "Tarantula Basic Fuzzer"
-    @output = nil
-    @description = "Supplies purely random but simplistically generated form input."
-  end
+      attr_reader *ATTRS
 
-  def ==(other)
-    Relevance::Tarantula::BasicAttack === other && ATTRS.all? { |attr| send(attr) == other.send(attr)}
-  end
+      def initialize
+        @name = "Tarantula Basic Fuzzer"
+        @output = nil
+        @description = "Supplies purely random but simplistically generated form input."
+      end
+
+      def ==(other)
+        Relevance::Tarantula::BasicAttack === other && ATTRS.all? { |attr| send(attr) == other.send(attr)}
+      end
+
+      def input(input_field)
+        case input_field['name']
+        when /amount/         then random_int
+        when /_id$/           then random_whole_number
+        when /uploaded_data/  then nil
+        when nil              then input['value']
+        else
+          random_int
+        end
+      end
+
+      def big_number
+        10000   # arbitrary
+      end
+
+      def random_int
+        rand(big_number) - (big_number/2)
+      end
 
-  def input(input_field)
-    case input_field['name']
-      when /amount/         then random_int
-      when /_id$/           then random_whole_number
-      when /uploaded_data/  then nil
-      when nil              then input['value']
-      else
-        random_int
+      def random_whole_number
+        rand(big_number)
+      end
     end
-  end
 
-  def big_number
-    10000   # arbitrary
-  end
-  
-  def random_int
-    rand(big_number) - (big_number/2)
-  end
-  
-  def random_whole_number
-    rand(big_number)
   end
 end
-
-

data/lib/relevance/tarantula/crawler.rb

@@ -3,252 +3,258 @@ require 'active_record/base'
 require File.expand_path(File.join(File.dirname(__FILE__), "rails_integration_proxy"))
 require File.expand_path(File.join(File.dirname(__FILE__), "html_document_handler.rb"))
 
-class Relevance::Tarantula::Crawler
-  extend Forwardable
-  include Relevance::Tarantula
-
-  class CrawlTimeout < RuntimeError; end
-
-  attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
-                :reporters, :crawl_queue, :links_queued,
-                :form_signatures_queued, :max_url_length, :response_code_handler,
-                :times_to_crawl, :fuzzers, :test_name, :crawl_timeout
-  attr_reader   :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
-
-  def initialize
-    @max_url_length = 1024
-    @successes = []
-    @failures = []
-    @handlers = [@response_code_handler = Result]
-    @links_queued = Set.new
-    @form_signatures_queued = Set.new
-    @crawl_queue = []
-    @crawl_start_times, @crawl_end_times = [], []
-    @crawl_timeout = 20.minutes
-    @referrers = {}
-    @skip_uri_patterns = [
-      /^javascript/,
-      /^mailto/,
-      /^http/,
-    ]
-    self.transform_url_patterns = [
-      [/#.*$/, '']
-    ]
-    @reporters = [Relevance::Tarantula::IOReporter.new($stderr)]
-    @decoder = HTMLEntities.new
-    @times_to_crawl = 1
-    @fuzzers = [Relevance::Tarantula::FormSubmission]
-    
-    @stdout_tty = $stdout.tty?
-  end
+module Relevance
+  module Tarantula
 
-  def method_missing(meth, *args)
-    super unless Result::ALLOW_NNN_FOR =~ meth.to_s
-    @response_code_handler.send(meth, *args)
-  end
+    class Crawler
+      extend Forwardable
+      include Relevance::Tarantula
 
-  def transform_url_patterns=(patterns)
-    @transform_url_patterns = patterns.map do |pattern|
-      Array === pattern ? Relevance::Tarantula::Transform.new(*pattern) : pattern
-    end
-  end
+      class CrawlTimeout < RuntimeError; end
+
+      attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
+        :reporters, :crawl_queue, :links_queued,
+        :form_signatures_queued, :max_url_length, :response_code_handler,
+        :times_to_crawl, :fuzzers, :test_name, :crawl_timeout
+      attr_reader   :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
 
-  def crawl(url = "/")
-    orig_links_queued = @links_queued.dup
-    orig_form_signatures_queued = @form_signatures_queued.dup
-    orig_crawl_queue = @crawl_queue.dup
-    @times_to_crawl.times do |num|
-      queue_link url
-      
-      begin 
-        do_crawl num
-      rescue CrawlTimeout => e
-        puts
-        puts e.message
-      end
-      
-      puts "#{(num+1).ordinalize} crawl" if @times_to_crawl > 1
-
-      if num + 1 < @times_to_crawl
-        @links_queued = orig_links_queued
-        @form_signatures_queued = orig_form_signatures_queued
-        @crawl_queue = orig_crawl_queue
+      def initialize
+        @max_url_length = 1024
+        @successes = []
+        @failures = []
+        @handlers = [@response_code_handler = Result]
+        @links_queued = Set.new
+        @form_signatures_queued = Set.new
+        @crawl_queue = []
+        @crawl_start_times, @crawl_end_times = [], []
+        @crawl_timeout = 20.minutes
         @referrers = {}
+        @skip_uri_patterns = [
+          /^javascript/,
+          /^mailto/,
+          /^http/,
+        ]
+        self.transform_url_patterns = [
+          [/#.*$/, '']
+        ]
+        @reporters = [Relevance::Tarantula::IOReporter.new($stderr)]
+        @decoder = HTMLEntities.new
+        @times_to_crawl = 1
+        @fuzzers = [Relevance::Tarantula::FormSubmission]
+
+        @stdout_tty = $stdout.tty?
       end
-    end
-  rescue Interrupt
-    $stderr.puts "CTRL-C"
-  ensure
-    report_results
-  end
 
-  def finished?
-    @crawl_queue.empty?
-  end
+      def method_missing(meth, *args)
+        super unless Result::ALLOW_NNN_FOR =~ meth.to_s
+        @response_code_handler.send(meth, *args)
+      end
 
-  def do_crawl(number)
-    while (!finished?)
-      @crawl_start_times << Time.now
-      crawl_the_queue(number)
-      @crawl_end_times << Time.now
-    end
-  end
+      def transform_url_patterns=(patterns)
+        @transform_url_patterns = patterns.map do |pattern|
+          Array === pattern ? Relevance::Tarantula::Transform.new(*pattern) : pattern
+        end
+      end
 
-  def crawl_the_queue(number = 0)
-    while (request = @crawl_queue.pop)
-      request.crawl
-      blip(number)
-    end
-  end
+      def crawl(url = "/")
+        orig_links_queued = @links_queued.dup
+        orig_form_signatures_queued = @form_signatures_queued.dup
+        orig_crawl_queue = @crawl_queue.dup
+        @times_to_crawl.times do |num|
+          queue_link url
 
-  def save_result(result)
-    reporters.each do |reporter|
-      reporter.report(result)
-    end
-  end
+          begin 
+            do_crawl num
+          rescue CrawlTimeout => e
+            puts
+            puts e.message
+          end
+
+          puts "#{ActiveSupport::Inflector.ordinalize((num+1))} crawl" if @times_to_crawl > 1
 
-  def handle_link_results(link, result)
-    handlers.each do |h|
-      begin
-        save_result h.handle(result)
-      rescue Exception => e
-        log "error handling #{link} #{e.message}"
-        # TODO: pass to results
+          if num + 1 < @times_to_crawl
+            @links_queued = orig_links_queued
+            @form_signatures_queued = orig_form_signatures_queued
+            @crawl_queue = orig_crawl_queue
+            @referrers = {}
+          end
+        end
+      rescue Interrupt
+        $stderr.puts "CTRL-C"
+      ensure
+        report_results
       end
-    end
-  end
 
-  def follow(method, url, data=nil)
-    proxy.send(method, url, data)
-  end
-  
-  def submit(method, action, data)
-    proxy.send(method, action, data)
-  end
+      def finished?
+        @crawl_queue.empty?
+      end
 
-  def elasped_time_for_pass(num)
-    Time.now - crawl_start_times[num]
-  end
+      def do_crawl(number)
+        while (!finished?)
+          @crawl_start_times << Time.now
+          crawl_the_queue(number)
+          @crawl_end_times << Time.now
+        end
+      end
 
-  def grab_log!
-    @log_grabber && @log_grabber.grab!
-  end
-  
-  def make_result(options)
-    defaults = {
-      :log       => grab_log!,
-      :test_name => test_name      
-    }
-    Result.new(defaults.merge(options)).freeze
-  end
+      def crawl_the_queue(number = 0)
+        while (request = @crawl_queue.pop)
+          request.crawl
+          blip(number)
+        end
+      end
 
-  def handle_form_results(form, response)
-    handlers.each do |h|
-      save_result h.handle(Result.new(:method => form.method,
-                                     :url => form.action,
-                                     :response => response,
-                                     :log => grab_log!,
-                                     :referrer => form.action,
-                                     :data => form.data.inspect,
-                                     :test_name => test_name).freeze)
-    end
-  end
+      def save_result(result)
+        reporters.each do |reporter|
+          reporter.report(result)
+        end
+      end
 
-  def should_skip_url?(url)
-    return true if url.blank?
-    if @skip_uri_patterns.any? {|pattern| pattern =~ url}
-      log "Skipping #{url}"
-      return true
-    end
-    if url.length > max_url_length
-      log "Skipping long url #{url}"
-      return true
-    end
-  end
+      def handle_link_results(link, result)
+        handlers.each do |h|
+          begin
+            save_result h.handle(result)
+          rescue Exception => e
+            log "error handling #{link} #{e.message}"
+            # TODO: pass to results
+          end
+        end
+      end
 
-  def should_skip_link?(link)
-    should_skip_url?(link.href) || @links_queued.member?(link)
-  end
+      def follow(method, url, data=nil)
+        proxy.send(method, url, data)
+      end
 
-  def should_skip_form_submission?(fs)
-    should_skip_url?(fs.action) || @form_signatures_queued.member?(fs.signature)
-  end
+      def submit(method, action, data)
+        proxy.send(method, action, data)
+      end
 
-  def transform_url(url)
-    return unless url
-    url = @decoder.decode(url)
-    @transform_url_patterns.each do |pattern|
-      url = pattern[url]
-    end
-    url
-  end
+      def elasped_time_for_pass(num)
+        Time.now - crawl_start_times[num]
+      end
 
-  def queue_link(dest, referrer = nil)
-    dest = Link.new(dest, self, referrer)
-    return if should_skip_link?(dest)
-    @crawl_queue << dest
-    @links_queued << dest
-    dest
-  end
+      def grab_log!
+        @log_grabber && @log_grabber.grab!
+      end
 
-  def queue_form(form, referrer = nil)
-    fuzzers.each do |fuzzer|
-      fuzzer.mutate(Form.new(form, self, referrer)).each do |fs|
-        # fs = fuzzer.new(Form.new(form, self, referrer))
-        fs.action = transform_url(fs.action)
-        return if should_skip_form_submission?(fs)
-        @referrers[fs.action] = referrer if referrer
-        @crawl_queue << fs
-        @form_signatures_queued << fs.signature
+      def make_result(options)
+        defaults = {
+          :log       => grab_log!,
+          :test_name => test_name      
+        }
+        Result.new(defaults.merge(options)).freeze
       end
-    end
-  end
 
-  def report_dir
-    File.join(rails_root, "tmp", "tarantula")
-  end
+      def handle_form_results(form, response)
+        handlers.each do |h|
+          save_result h.handle(Result.new(:method => form.method,
+                                          :url => form.action,
+                                          :response => response,
+                                          :log => grab_log!,
+                                          :referrer => form.action,
+                                          :data => form.data.inspect,
+                                          :test_name => test_name).freeze)
+        end
+      end
 
-  def generate_reports
-    errors = []
-    reporters.each do |reporter|
-      begin
-        reporter.finish_report(test_name)
-      rescue RuntimeError => e
-        errors << e
+      def should_skip_url?(url)
+        return true if url.blank?
+        if @skip_uri_patterns.any? {|pattern| pattern =~ url}
+          log "Skipping #{url}"
+          return true
+        end
+        if url.length > max_url_length
+          log "Skipping long url #{url}"
+          return true
+        end
       end
-    end
-    unless errors.empty?
-      raise errors.map(&:message).join("\n")
-    end
-  end
 
-  def report_results
-    puts "Crawled #{total_links_count} links and forms."
-    generate_reports
-  end
+      def should_skip_link?(link)
+        should_skip_url?(link.href) || @links_queued.member?(link)
+      end
 
-  def total_links_count
-    @links_queued.size + @form_signatures_queued.size
-  end
+      def should_skip_form_submission?(fs)
+        should_skip_url?(fs.action) || @form_signatures_queued.member?(fs.signature)
+      end
 
-  def links_remaining_count
-    @crawl_queue.size
-  end
+      def transform_url(url)
+        return unless url
+        url = @decoder.decode(url)
+        @transform_url_patterns.each do |pattern|
+          url = pattern[url]
+        end
+        url
+      end
 
-  def links_completed_count
-      total_links_count - links_remaining_count
-  end
+      def queue_link(dest, referrer = nil)
+        dest = Link.new(dest, self, referrer)
+        return if should_skip_link?(dest)
+        @crawl_queue << dest
+        @links_queued << dest
+        dest
+      end
 
-  def blip(number = 0)
-    unless verbose
-      print "\r #{links_completed_count} of #{total_links_count} links completed               " if @stdout_tty
-      timeout_if_too_long(number)
-    end
-  end
-  
-  def timeout_if_too_long(number = 0)
-    if elasped_time_for_pass(number) > crawl_timeout
-      raise CrawlTimeout, "Exceeded crawl timeout of #{crawl_timeout} seconds - skipping to the next crawl..."
+      def queue_form(form, referrer = nil)
+        fuzzers.each do |fuzzer|
+          fuzzer.mutate(Form.new(form, self, referrer)).each do |fs|
+            # fs = fuzzer.new(Form.new(form, self, referrer))
+            fs.action = transform_url(fs.action)
+            return if should_skip_form_submission?(fs)
+            @referrers[fs.action] = referrer if referrer
+            @crawl_queue << fs
+            @form_signatures_queued << fs.signature
+          end
+        end
+      end
+
+      def report_dir
+        File.join(rails_root, "tmp", "tarantula")
+      end
+
+      def generate_reports
+        errors = []
+        reporters.each do |reporter|
+          begin
+            reporter.finish_report(test_name)
+          rescue RuntimeError => e
+            errors << e
+          end
+        end
+        unless errors.empty?
+          raise errors.map(&:message).join("\n")
+        end
+      end
+
+      def report_results
+        puts "Crawled #{total_links_count} links and forms."
+        generate_reports
+      end
+
+      def total_links_count
+        @links_queued.size + @form_signatures_queued.size
+      end
+
+      def links_remaining_count
+        @crawl_queue.size
+      end
+
+      def links_completed_count
+        total_links_count - links_remaining_count
+      end
+
+      def blip(number = 0)
+        unless verbose
+          print "\r #{links_completed_count} of #{total_links_count} links completed               " if @stdout_tty
+          timeout_if_too_long(number)
+        end
+      end
+
+      def timeout_if_too_long(number = 0)
+        if elasped_time_for_pass(number) > crawl_timeout
+          raise CrawlTimeout, "Exceeded crawl timeout of #{crawl_timeout} seconds - skipping to the next crawl..."
+        end
+      end
     end
+
   end
 end