tarantula 0.0.5

data/rails/init.rb

@@ -0,0 +1 @@
+load File.expand_path(File.join(File.dirname(__FILE__) + ".." "tasks", "tarantula.rake"))

data/tarantula.gemspec

@@ -0,0 +1,48 @@
+Gem::Specification.new do |s|
+  s.name = %q{tarantula}
+  s.version = "0.0.5"
+
+  s.required_rubygems_version = Gem::Requirement.new("= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Relevance"]
+  s.date = %q{2008-09-26}
+  s.description = %q{A big hairy fuzzy spider that crawls your site, wreaking havoc}
+  s.email = %q{opensource@thinkrelevance.com}
+  s.extra_rdoc_files = ["CHANGELOG", "lib/relevance/core_extensions/ellipsize.rb", "lib/relevance/core_extensions/file.rb", "lib/relevance/core_extensions/response.rb", "lib/relevance/core_extensions/test_case.rb", "lib/relevance/tarantula/attack.rb", "lib/relevance/tarantula/attack_form_submission.rb", "lib/relevance/tarantula/attack_handler.rb", "lib/relevance/tarantula/crawler.rb", "lib/relevance/tarantula/detail.html.erb", "lib/relevance/tarantula/form.rb", "lib/relevance/tarantula/form_submission.rb", "lib/relevance/tarantula/html_document_handler.rb", "lib/relevance/tarantula/html_report_helper.rb", "lib/relevance/tarantula/html_reporter.rb", "lib/relevance/tarantula/index.html.erb", "lib/relevance/tarantula/invalid_html_handler.rb", "lib/relevance/tarantula/io_reporter.rb", "lib/relevance/tarantula/link.rb", "lib/relevance/tarantula/log_grabber.rb", "lib/relevance/tarantula/rails_integration_proxy.rb", "lib/relevance/tarantula/recording.rb", "lib/relevance/tarantula/response.rb", "lib/relevance/tarantula/result.rb", "lib/relevance/tarantula/test_report.html.erb", "lib/relevance/tarantula/tidy_handler.rb", "lib/relevance/tarantula/transform.rb", "lib/relevance/tarantula.rb", "MIT-LICENSE", "README.rdoc", "vendor/xss-shield/MIT-LICENSE"]
+  s.files = ["CHANGELOG", "init.rb", "install.rb", "laf/images/background.jpg", "laf/images/relevance-os-logo.gif", "laf/images/tab.png", "laf/images/table-sort.gif", "laf/images/tarantula-sprites.png", "laf/javascripts/jquery-1.2.3.js", "laf/javascripts/jquery-ui-tabs.js", "laf/javascripts/jquery.tablesorter.js", "laf/javascripts/tarantula.js", "laf/stylesheets/tarantula.css", "laf/stylesheets/ui.tabs.css", "lib/relevance/core_extensions/ellipsize.rb", "lib/relevance/core_extensions/file.rb", "lib/relevance/core_extensions/response.rb", "lib/relevance/core_extensions/test_case.rb", "lib/relevance/tarantula/attack.rb", "lib/relevance/tarantula/attack_form_submission.rb", "lib/relevance/tarantula/attack_handler.rb", "lib/relevance/tarantula/crawler.rb", "lib/relevance/tarantula/detail.html.erb", "lib/relevance/tarantula/form.rb", "lib/relevance/tarantula/form_submission.rb", "lib/relevance/tarantula/html_document_handler.rb", "lib/relevance/tarantula/html_report_helper.rb", "lib/relevance/tarantula/html_reporter.rb", "lib/relevance/tarantula/index.html.erb", "lib/relevance/tarantula/invalid_html_handler.rb", "lib/relevance/tarantula/io_reporter.rb", "lib/relevance/tarantula/link.rb", "lib/relevance/tarantula/log_grabber.rb", "lib/relevance/tarantula/rails_integration_proxy.rb", "lib/relevance/tarantula/recording.rb", "lib/relevance/tarantula/response.rb", "lib/relevance/tarantula/result.rb", "lib/relevance/tarantula/test_report.html.erb", "lib/relevance/tarantula/tidy_handler.rb", "lib/relevance/tarantula/transform.rb", "lib/relevance/tarantula.rb", "manifest.txt", "MIT-LICENSE", "rails/init.rb", "Rakefile", "README.rdoc", "tarantula.gemspec", "tasks/tarantula_tasks.rake", "template/tarantula_test.rb", "test/relevance/core_extensions/ellipsize_test.rb", "test/relevance/core_extensions/file_test.rb", "test/relevance/core_extensions/response_test.rb", "test/relevance/core_extensions/test_case_test.rb", "test/relevance/tarantula/attack_form_submission_test.rb", "test/relevance/tarantula/attack_handler_test.rb", "test/relevance/tarantula/crawler_test.rb", "test/relevance/tarantula/form_submission_test.rb", "test/relevance/tarantula/form_test.rb", "test/relevance/tarantula/html_document_handler_test.rb", "test/relevance/tarantula/html_report_helper_test.rb", "test/relevance/tarantula/html_reporter_test.rb", "test/relevance/tarantula/invalid_html_handler_test.rb", "test/relevance/tarantula/io_reporter_test.rb", "test/relevance/tarantula/link_test.rb", "test/relevance/tarantula/log_grabber_test.rb", "test/relevance/tarantula/rails_integration_proxy_test.rb", "test/relevance/tarantula/result_test.rb", "test/relevance/tarantula/tidy_handler_test.rb", "test/relevance/tarantula/transform_test.rb", "test/relevance/tarantula_test.rb", "test/test_helper.rb", "tmp/test_output/images/background.jpg", "tmp/test_output/images/relevance-os-logo.gif", "tmp/test_output/images/tab.png", "tmp/test_output/images/table-sort.gif", "tmp/test_output/images/tarantula-sprites.png", "tmp/test_output/index.html", "tmp/test_output/javascripts/jquery-1.2.3.js", "tmp/test_output/javascripts/jquery-ui-tabs.js", "tmp/test_output/javascripts/jquery.tablesorter.js", "tmp/test_output/javascripts/tarantula.js", "tmp/test_output/stylesheets/tarantula.css", "tmp/test_output/stylesheets/ui.tabs.css", "tmp/test_output/test_user_pages/1.html", "tmp/test_output/test_user_pages/10.html", "tmp/test_output/test_user_pages/11.html", "tmp/test_output/test_user_pages/12.html", "tmp/test_output/test_user_pages/13.html", "tmp/test_output/test_user_pages/14.html", "tmp/test_output/test_user_pages/15.html", "tmp/test_output/test_user_pages/16.html", "tmp/test_output/test_user_pages/17.html", "tmp/test_output/test_user_pages/18.html", "tmp/test_output/test_user_pages/19.html", "tmp/test_output/test_user_pages/2.html", "tmp/test_output/test_user_pages/20.html", "tmp/test_output/test_user_pages/3.html", "tmp/test_output/test_user_pages/4.html", "tmp/test_output/test_user_pages/5.html", "tmp/test_output/test_user_pages/6.html", "tmp/test_output/test_user_pages/7.html", "tmp/test_output/test_user_pages/8.html", "tmp/test_output/test_user_pages/9.html", "uninstall.rb", "vendor/xss-shield/init.rb", "vendor/xss-shield/lib/xss_shield/erb_hacks.rb", "vendor/xss-shield/lib/xss_shield/haml_hacks.rb", "vendor/xss-shield/lib/xss_shield/safe_string.rb", "vendor/xss-shield/lib/xss_shield/secure_helpers.rb", "vendor/xss-shield/lib/xss_shield.rb", "vendor/xss-shield/MIT-LICENSE", "vendor/xss-shield/README", "vendor/xss-shield/test/test_actionview_integration.rb", "vendor/xss-shield/test/test_erb.rb", "vendor/xss-shield/test/test_haml.rb", "vendor/xss-shield/test/test_helpers.rb", "vendor/xss-shield/test/test_safe_string.rb"]
+  s.has_rdoc = true
+  s.homepage = %q{http://opensource.thinkrelevance.com/wiki/tarantula}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Tarantula", "--main", "README.rdoc"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{thinkrelevance}
+  s.rubygems_version = %q{1.2.0}
+  s.summary = %q{A big hairy fuzzy spider that crawls your site, wreaking havoc}
+  s.test_files = ["test/relevance/core_extensions/ellipsize_test.rb", "test/relevance/core_extensions/file_test.rb", "test/relevance/core_extensions/response_test.rb", "test/relevance/core_extensions/test_case_test.rb", "test/relevance/tarantula/attack_form_submission_test.rb", "test/relevance/tarantula/attack_handler_test.rb", "test/relevance/tarantula/crawler_test.rb", "test/relevance/tarantula/form_submission_test.rb", "test/relevance/tarantula/form_test.rb", "test/relevance/tarantula/html_document_handler_test.rb", "test/relevance/tarantula/html_report_helper_test.rb", "test/relevance/tarantula/html_reporter_test.rb", "test/relevance/tarantula/invalid_html_handler_test.rb", "test/relevance/tarantula/io_reporter_test.rb", "test/relevance/tarantula/link_test.rb", "test/relevance/tarantula/log_grabber_test.rb", "test/relevance/tarantula/rails_integration_proxy_test.rb", "test/relevance/tarantula/result_test.rb", "test/relevance/tarantula/tidy_handler_test.rb", "test/relevance/tarantula/transform_test.rb", "test/relevance/tarantula_test.rb"]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if current_version >= 3 then
+      s.add_runtime_dependency(%q<htmlentities>, [">= 0"])
+      s.add_runtime_dependency(%q<hpricot>, [">= 0"])
+      s.add_runtime_dependency(%q<facets>, [">= 2.4.3"])
+      s.add_development_dependency(%q<ruby-debug>, [">= 0"])
+      s.add_development_dependency(%q<test-spec>, [">= 0"])
+      s.add_development_dependency(%q<mocha>, [">= 0"])
+    else
+      s.add_dependency(%q<htmlentities>, [">= 0"])
+      s.add_dependency(%q<hpricot>, [">= 0"])
+      s.add_dependency(%q<facets>, [">= 2.4.3"])
+      s.add_dependency(%q<ruby-debug>, [">= 0"])
+      s.add_dependency(%q<test-spec>, [">= 0"])
+      s.add_dependency(%q<mocha>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<htmlentities>, [">= 0"])
+    s.add_dependency(%q<hpricot>, [">= 0"])
+    s.add_dependency(%q<facets>, [">= 2.4.3"])
+    s.add_dependency(%q<ruby-debug>, [">= 0"])
+    s.add_dependency(%q<test-spec>, [">= 0"])
+    s.add_dependency(%q<mocha>, [">= 0"])
+  end
+end

data/tasks/tarantula_tasks.rake

@@ -0,0 +1,34 @@
+namespace :tarantula do
+
+  desc 'Run tarantula tests and (Mac only) open results in your browser.'
+  task :test do
+    rm_rf "tmp/tarantula"
+    task = Rake::TestTask.new(:tarantula_test) do |t|
+      t.libs << 'test'
+      t.pattern = 'test/tarantula/**/*_test.rb'
+      t.verbose = true
+    end
+
+    begin
+      Rake::Task[:tarantula_test].invoke
+    rescue RuntimeError => e
+      puts e.message
+    end
+    Dir.glob("tmp/tarantula/**/index.html") do |file|
+      if PLATFORM['darwin']
+        system("open #{file}")
+      elsif PLATFORM[/linux/]
+        system("firefox #{file}")
+      else
+        puts "You can view tarantula results at #{file}"
+      end
+    end
+  end
+
+  desc 'Generate a default tarantula test'
+  task :setup do
+    mkdir_p "test/tarantula"
+    template_path = File.expand_path(File.join(File.dirname(__FILE__), "..", "template", "tarantula_test.rb"))
+    cp template_path, "test/tarantula/"
+  end
+end

data/template/tarantula_test.rb

@@ -0,0 +1,12 @@
+require "#{File.dirname(__FILE__)}/../test_helper"
+require "relevance/tarantula"
+
+class TarantulaTest < ActionController::IntegrationTest
+  fixtures :all
+
+  def test_tarantula
+    post '/session', :login => 'quentin', :password => 'monkey'
+    follow_redirect!
+    tarantula_crawl(self)
+  end
+end

data/test/relevance/core_extensions/ellipsize_test.rb

@@ -0,0 +1,19 @@
+require File.join(File.dirname(__FILE__), "../..", "test_helper.rb")
+
+describe "Relevance::CoreExtensions::Object#ellipsize" do
+  it "converts nil to empty string" do
+    nil.ellipsize.should == ""
+  end
+  
+  it "doesn't touch short strings" do
+    "hello".ellipsize.should == "hello"
+  end
+  
+  it "calls inspect on non-strings" do
+    [1,2,3].ellipsize.should == "[1, 2, 3]"
+  end
+
+  it "shortens long strings and adds ..." do
+    "long-string".ellipsize(5).should == "long-..."
+  end
+end

data/test/relevance/core_extensions/file_test.rb

@@ -0,0 +1,8 @@
+require File.join(File.dirname(__FILE__), "../..", "test_helper.rb")
+require 'relevance/core_extensions/file'
+
+describe "Relevance::CoreExtensions::File#extension" do
+  it "should return the extension without the leading dot" do
+    File.extension("foo.bar").should == "bar"
+  end
+end

data/test/relevance/core_extensions/response_test.rb

@@ -0,0 +1,29 @@
+require File.join(File.dirname(__FILE__), "../..", "test_helper.rb")
+require 'relevance/core_extensions/file'
+
+describe "Relevance::CoreExtensions::Response#html?" do
+  before do
+    @response = OpenStruct.new
+    @response.extend(Relevance::CoreExtensions::Response)
+  end         
+  
+  it "should be html if the content-type is 'text/html'" do
+    @response.content_type = "text/html"
+    @response.should.be.html
+    @response.content_type = "text/html;charset=iso-8859-2"
+    @response.should.be.html
+  end
+
+  it "should not be html if the content-type isn't an html type" do
+    @response.content_type = "text/plain"
+    @response.should.not.be.html
+  end
+
+  # better ideas welcome, but be careful not to  
+  # castrate tarantula for proxies that don't set the content-type
+  it "should pretend we have html if the content-type is nil" do
+    @response.content_type = nil
+    @response.should.be.html
+  end
+  
+end

data/test/relevance/core_extensions/test_case_test.rb

@@ -0,0 +1,16 @@
+require File.join(File.dirname(__FILE__), "../..", "test_helper.rb")
+require 'relevance/core_extensions/test_case'
+
+describe "TestCase extensions" do
+  it "can create the crawler" do
+    Relevance::Tarantula::RailsIntegrationProxy.stubs(:rails_root).returns("STUB_RAILS_ROOT")
+    Relevance::Tarantula::Crawler.any_instance.stubs(:rails_root).returns("STUB_RAILS_ROOT")
+    tarantula_crawler(stub_everything)
+  end
+
+  it "can crawl" do
+    (crawler = mock).expects(:crawl).with("/foo")
+    expects(:tarantula_crawler).returns(crawler)
+    tarantula_crawl(:integration_test_stub, :url => "/foo")
+  end
+end

data/test/relevance/tarantula/attack_form_submission_test.rb

@@ -0,0 +1,79 @@
+require File.join(File.dirname(__FILE__), "..", "..", "test_helper.rb")
+
+describe "Relevance::Tarantula::AttackFormSubmission" do
+  
+  # TODO: add more from field types to this example form as needed
+  before do
+    @tag = Hpricot(<<END)
+<form action="/session" method="post">
+  <input id="email" name="email" size="30" type="text" />
+  <textarea id="comment" name="comment"value="1" />
+  <input name="commit" type="submit" value="Postit" />
+  <input name="secret" type="hidden" value="secret" />
+  <select id="foo_opened_on_1i" name="foo[opened_on(1i)]">
+    <option value="2003">2003</option>
+    <option value="2004">2004</option>
+  </select> 
+</form>
+END
+    @form = Relevance::Tarantula::Form.new(@tag.at('form'))
+    @fs = Relevance::Tarantula::AttackFormSubmission.new(@form, Relevance::Tarantula::Attack.new({:name => 'foo_name', :input => 'foo_code', :output => 'foo_code'}))
+  end
+  
+  it "can mutate text areas" do
+    @fs.mutate_text_areas(@form).should == {"comment" => "foo_code"}
+  end
+  
+  it "can mutate selects" do
+    Hpricot::Elements.any_instance.stubs(:rand).returns(stub(:[] => "2006-stub"))
+    @fs.mutate_selects(@form).should == {"foo[opened_on(1i)]" => "2006-stub"}
+  end
+  
+  it "can mutate inputs" do
+    @fs.mutate_inputs(@form).should == {"commit"=>"foo_code", "secret"=>"foo_code", "email"=>"foo_code"}
+  end
+
+  it "has a signature based on action,  fields, and attack name" do
+    @fs.signature.should == ['/session', [
+      "comment", 
+      "commit", 
+      "email", 
+      "foo[opened_on(1i)]", 
+      "secret"],
+      "foo_name"
+    ]
+  end
+  
+  it "has a friendly to_s" do
+    @fs.to_s.should =~ %r{^/session post}
+  end
+  
+  it "processes all its attacks" do
+    Relevance::Tarantula::AttackFormSubmission.stubs(:attacks).returns([
+      Relevance::Tarantula::Attack.new({:name => 'foo_name1', :input => 'foo_input', :output => 'foo_output'}),
+      Relevance::Tarantula::Attack.new({:name => 'foo_name2', :input => 'foo_input', :output => 'foo_output'}),
+    ])
+    Relevance::Tarantula::AttackFormSubmission.mutate(@form).size.should == 2
+  end
+  
+  it "maps hash attacks to Attack instances" do
+    Relevance::Tarantula::AttackFormSubmission.instance_variable_set("@attacks", [{ :name => "attack name"}])
+    Relevance::Tarantula::AttackFormSubmission.attacks.should == [Relevance::Tarantula::Attack.new({:name => "attack name"})]
+  end
+end
+
+describe "Relevance::Tarantula::AttackFormSubmission for a crummy form" do
+  before do
+    @tag = Hpricot(<<END)
+<form action="/session" method="post">
+  <input value="no_name" />
+</form>
+END
+    @form = Relevance::Tarantula::Form.new(@tag.at('form'))
+    @fs = Relevance::Tarantula::AttackFormSubmission.new(@form, {:name => 'foo_name', :input => 'foo_code', :output => 'foo_code'})
+  end
+  
+  it "ignores unnamed inputs" do
+    @fs.mutate_inputs(@form).should == {}
+  end
+end

data/test/relevance/tarantula/attack_handler_test.rb

@@ -0,0 +1,29 @@
+require File.join(File.dirname(__FILE__), "..", "..", "test_helper.rb")
+
+describe "Relevance::Tarantula::AttackHandler" do
+  before do
+    @handler = Relevance::Tarantula::AttackHandler.new
+    attack = Relevance::Tarantula::Attack.new({:name => 'foo_name', :input => 'foo_code', :output => '<bad>'})
+    @handler.stubs(:attacks).returns([attack])
+  end
+  
+  it "lets safe documents through" do
+    result = @handler.handle(Relevance::Tarantula::Result.new(:response => stub(:html? => true, :body => '<a href="/foo">good</a>')))
+    result.should == nil
+  end
+  
+  it "detects the supplied code" do
+    result = @handler.handle(Relevance::Tarantula::Result.new(:response => stub(:html? => true, :body => '<a href="/foo"><bad></a>')))
+    result.success.should == false
+  end
+end
+
+describe "Attacks without an output specified" do
+  it "never matches anything" do
+    handler = Relevance::Tarantula::AttackHandler.new
+    attack = Relevance::Tarantula::Attack.new({:name => 'foo_name', :input => 'foo_code'})
+    Relevance::Tarantula::AttackFormSubmission.stubs(:attacks).returns([attack])
+    result = handler.handle(Relevance::Tarantula::Result.new(:response => stub(:html? => true, :body => '<a href="/foo">good</a>')))
+    result.should == nil
+  end
+end

data/test/relevance/tarantula/crawler_test.rb

@@ -0,0 +1,296 @@
+require File.join(File.dirname(__FILE__), "..", "..", "test_helper.rb")
+
+describe 'Relevance::Tarantula::Crawler#transform_url' do
+  before {@crawler = Relevance::Tarantula::Crawler.new}
+  it "de-obfuscates unicode obfuscated urls" do
+    obfuscated_mailto = "mailto:"
+    @crawler.transform_url(obfuscated_mailto).should == "mailto:"
+  end
+  
+  it "strips the trailing name portion of a link" do
+    @crawler.transform_url('http://host/path#name').should == 'http://host/path'
+  end
+end
+
+describe 'Relevance::Tarantula::Crawler log grabbing' do
+  it "returns nil if no grabber is specified" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.grab_log!.should == nil
+  end
+  
+  it "returns grabber.grab if grabber is specified" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.log_grabber = stub(:grab! => "fake log entry")
+    crawler.grab_log!.should == "fake log entry"
+  end
+end
+
+describe 'Relevance::Tarantula::Crawler interruption' do
+  it 'catches interruption and writes the partial report' do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.stubs(:queue_link)
+    crawler.stubs(:do_crawl).raises(Interrupt)
+    crawler.expects(:report_results)
+    $stderr.expects(:puts).with("CTRL-C")
+    crawler.crawl
+  end
+end
+
+describe 'Relevance::Tarantula::Crawler handle_form_results' do
+  it 'captures the result values (bugfix)' do
+    response = stub_everything
+    result_args = {:url => :action_stub, 
+                    :data => 'nil', 
+                    :response => response, 
+                    :referrer => :action_stub, 
+                    :log => nil, 
+                    :method => :stub_method,
+                    :test_name => nil}
+    result = Relevance::Tarantula::Result.new(result_args)
+    Relevance::Tarantula::Result.expects(:new).with(result_args).returns(result)
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.handle_form_results(stub_everything(:method => :stub_method, :action => :action_stub), 
+                                response)
+  end
+end
+
+describe 'Relevance::Tarantula::Crawler#crawl' do
+  it 'queues the first url, does crawl, and then reports results' do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.expects(:queue_link).with("/foobar")
+    crawler.expects(:do_crawl)
+    crawler.expects(:report_results)
+    crawler.crawl("/foobar")
+  end
+  
+  it 'reports results even if the crawl fails' do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.expects(:do_crawl).raises(RuntimeError)
+    crawler.expects(:report_results)
+    lambda {crawler.crawl('/')}.should.raise(RuntimeError)
+  end
+end
+
+describe 'Relevance::Tarantula::Crawler queuing' do
+  it 'queues and remembers links' do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.expects(:transform_url).with("/url").returns("/transformed")
+    crawler.queue_link("/url")
+    crawler.links_to_crawl.should == [Relevance::Tarantula::Link.new("/transformed")]
+    crawler.links_queued.should == Set.new([Relevance::Tarantula::Link.new("/transformed")])
+  end
+  
+  it 'queues and remembers forms' do
+    crawler = Relevance::Tarantula::Crawler.new
+    form = Hpricot('<form action="/action" method="post"/>').at('form')
+    signature = Relevance::Tarantula::FormSubmission.new(Relevance::Tarantula::Form.new(form)).signature
+    crawler.queue_form(form)
+    crawler.forms_to_crawl.size.should == 1
+    crawler.form_signatures_queued.should == Set.new([signature])
+  end
+  
+  it 'remembers link referrer if there is one' do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.queue_link("/url", "/some-referrer")
+    crawler.referrers.should == {Relevance::Tarantula::Link.new("/url") => "/some-referrer"}
+  end
+  
+end
+
+describe 'Relevance::Tarantula::Crawler#report_results' do
+  it "delegates to generate_reports" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.expects(:generate_reports)
+    crawler.report_results
+  end
+end
+
+describe 'Relevance::Tarantula::Crawler#crawling' do
+
+  it "converts ActiveRecord::RecordNotFound into a 404" do
+    (proxy = stub_everything).expects(:send).raises(ActiveRecord::RecordNotFound)
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.proxy = proxy
+    response = crawler.crawl_form stub_everything(:method => nil)
+    response.code.should == "404"
+    response.content_type.should == "text/plain"
+    response.body.should == "ActiveRecord::RecordNotFound"
+  end
+
+  it "does four things with each link: get, log, handle, and blip" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.proxy = stub
+    response = stub(:code => "200")
+    crawler.links_to_crawl = [stub(:href => "/foo1", :method => :get), stub(:href => "/foo2", :method => :get)]
+    crawler.proxy.expects(:get).returns(response).times(2)
+    crawler.expects(:log).times(2)
+    crawler.expects(:handle_link_results).times(2)
+    crawler.expects(:blip).times(2)
+    crawler.crawl_queued_links
+    crawler.links_to_crawl.should == []
+  end
+    
+  it "invokes queued forms, logs responses, and calls handlers" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.forms_to_crawl << stub_everything(:method => "get", 
+                                              :action => "/foo",
+                                              :data => "some data",
+                                              :to_s => "stub")
+    crawler.proxy = stub_everything(:send => stub(:code => "200" ))
+    crawler.expects(:log).with("Response 200 for stub")
+    crawler.expects(:blip)
+    crawler.crawl_queued_forms
+  end
+  
+  it "resets to the initial links/forms on subsequent crawls when times_to_crawl > 1" do
+    crawler = Relevance::Tarantula::Crawler.new
+    stub_puts_and_print(crawler)
+    crawler.proxy = stub
+    response = stub(:code => "200")
+    crawler.links_to_crawl = [stub(:href => "/foo", :method => :get)]
+    crawler.proxy.expects(:get).returns(response).times(4) # (stub and "/") * 2
+    crawler.forms_to_crawl << stub_everything(:method => "post", 
+                                              :action => "/foo",
+                                              :data => "some data",
+                                              :to_s => "stub")
+    crawler.proxy.expects(:post).returns(response).times(2)
+    crawler.expects(:links_completed_count).returns(*(0..6).to_a).times(6)
+    crawler.times_to_crawl = 2
+    crawler.crawl
+  end
+end
+
+describe 'Crawler blip' do
+  it "blips the current progress if !verbose" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.stubs(:verbose).returns false
+    crawler.expects(:print).with("\r 0 of 0 links completed               ")
+    crawler.blip
+  end
+  it "blips nothing if verbose" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.stubs(:verbose).returns true
+    crawler.expects(:print).never
+    crawler.blip
+  end
+end
+
+describe 'Relevance::Tarantula::Crawler' do
+  it "is finished when the links and forms are crawled" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.finished?.should == true
+  end
+
+  it "isn't finished when links remain" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.links_to_crawl = [:stub_link]
+    crawler.finished?.should == false
+  end
+
+  it "isn't finished when links remain" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.forms_to_crawl = [:stub_form]
+    crawler.finished?.should == false
+  end
+  
+  it "crawls links and forms again and again until finished?==true" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.expects(:finished?).times(3).returns(false, false, true)
+    crawler.expects(:crawl_queued_links).times(2)
+    crawler.expects(:crawl_queued_forms).times(2)
+    crawler.do_crawl
+  end
+  
+  it "asks each reporter to write its report in report_dir" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.stubs(:report_dir).returns(test_output_dir)
+    reporter = stub_everything
+    reporter.expects(:report)
+    reporter.expects(:finish_report)
+    crawler.reporters = [reporter]
+    crawler.save_result stub(:code => "404", :url => "/uh-oh")
+    crawler.generate_reports
+  end
+  
+  it "builds a report dir relative to rails root" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.expects(:rails_root).returns("faux_rails_root")
+    crawler.report_dir.should == "faux_rails_root/tmp/tarantula"
+  end
+  
+  it "skips links that are already queued" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.should_skip_link?(Relevance::Tarantula::Link.new("/foo")).should == false
+    crawler.queue_link("/foo").should == Relevance::Tarantula::Link.new("/foo")
+    crawler.should_skip_link?(Relevance::Tarantula::Link.new("/foo")).should == true
+  end
+  
+end
+                         
+describe "Crawler link skipping" do   
+  before do
+    @crawler = Relevance::Tarantula::Crawler.new
+  end
+  
+  it "skips links that are too long" do
+    @crawler.should_skip_link?(Relevance::Tarantula::Link.new("/foo")).should == false
+    @crawler.max_url_length = 2
+    @crawler.expects(:log).with("Skipping long url /foo")
+    @crawler.should_skip_link?(Relevance::Tarantula::Link.new("/foo")).should == true
+  end
+  
+  it "skips outbound links (those that begin with http)" do
+    @crawler.expects(:log).with("Skipping http-anything")
+    @crawler.should_skip_link?(Relevance::Tarantula::Link.new("http-anything")).should == true
+  end
+
+  it "skips javascript links (those that begin with javascript)" do
+    @crawler.expects(:log).with("Skipping javascript-anything")
+    @crawler.should_skip_link?(Relevance::Tarantula::Link.new("javascript-anything")).should == true
+  end
+
+  it "skips mailto links (those that begin with http)" do
+    @crawler.expects(:log).with("Skipping mailto-anything")
+    @crawler.should_skip_link?(Relevance::Tarantula::Link.new("mailto-anything")).should == true
+  end
+  
+  it 'skips blank links' do
+    @crawler.queue_link(nil)
+    @crawler.links_to_crawl.should == []
+    @crawler.queue_link("")
+    @crawler.links_to_crawl.should == []
+  end
+  
+  it "logs and skips links that match a pattern" do
+    @crawler.expects(:log).with("Skipping /the-red-button")
+    @crawler.skip_uri_patterns << /red-button/
+    @crawler.queue_link("/blue-button").should == Relevance::Tarantula::Link.new("/blue-button")
+    @crawler.queue_link("/the-red-button").should == nil
+  end   
+  
+  it "logs and skips form submissions that match a pattern" do
+    @crawler.expects(:log).with("Skipping /reset-password-form")
+    @crawler.skip_uri_patterns << /reset-password/             
+    fs = stub_everything(:action => "/reset-password-form")
+    @crawler.should_skip_form_submission?(fs).should == true
+  end
+end
+
+describe "allow_nnn_for" do
+  it "installs result as a response_code_handler" do
+    crawler = Relevance::Tarantula::Crawler.new
+    crawler.response_code_handler.should == Relevance::Tarantula::Result
+  end
+  
+  it "delegates to the response_code_handler" do
+    crawler = Relevance::Tarantula::Crawler.new
+    (response_code_handler = mock).expects(:allow_404_for).with(:stub)
+    crawler.response_code_handler = response_code_handler
+    crawler.allow_404_for(:stub)
+  end
+  
+  it "chains up to super for method_missing" do
+    crawler = Relevance::Tarantula::Crawler.new
+    lambda{crawler.foo}.should.raise(NoMethodError)
+  end
+end