relevance-tarantula 0.2.1 → 0.3.2

data/examples/relevance/tarantula/link_example.rb

@@ -4,47 +4,64 @@ describe "Relevance::Tarantula::Link" do
   include ActionView::Helpers::UrlHelper
   
   it "does not raise an error when initializing without href attribtue" do
-    link = Relevance::Tarantula::Link.new(Hpricot('<a="/foo">foo</a>').at('a'))
+    link = make_link(Hpricot('<a="/foo">foo</a>').at('a'))
     link.href.should == nil
     link.method.should == :get    
   end
 
   it "parses anchor tags" do
-    link = Relevance::Tarantula::Link.new(Hpricot('<a href="/foo">foo</a>').at('a'))
+    link = make_link(Hpricot('<a href="/foo">foo</a>').at('a'))
     link.href.should == '/foo'
     link.method.should == :get
   end
 
   it "parses anchor tags with POST 'method'" do
-    link = Relevance::Tarantula::Link.new(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:post)}">foo</a>}).at('a'))
+    link = make_link(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:post)}">foo</a>}).at('a'))
     link.href.should == '/foo'
     link.method.should == :post
   end
 
   it "parses anchor tags with PUT 'method'" do
-    link = Relevance::Tarantula::Link.new(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:put)}">foo</a>}).at('a'))
+    link = make_link(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:put)}">foo</a>}).at('a'))
     link.href.should == '/foo'
     link.method.should == :put
   end
 
   it "parses anchor tags with DELETE 'method'" do
-    link = Relevance::Tarantula::Link.new(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:delete)}">foo</a>}).at('a'))
+    link = make_link(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:delete)}">foo</a>}).at('a'))
     link.href.should == '/foo'
     link.method.should == :delete
   end
 
   it "parses link tags with text" do
-    link = Relevance::Tarantula::Link.new(Hpricot('<link href="/bar">bar</a>').at('link'))
+    link = make_link(Hpricot('<link href="/bar">bar</a>').at('link'))
     link.href.should == '/bar'
     link.method.should == :get
   end
   
   it "parses link tags without text" do
-    link = Relevance::Tarantula::Link.new(Hpricot('<link href="/bar" />').at('link'))
+    link = make_link(Hpricot('<link href="/bar" />').at('link'))
     link.href.should == '/bar'
     link.method.should == :get
   end
   
+  it 'remembers link referrer if there is one' do
+    link = make_link('/url', stub_everything, '/some-referrer')
+    link.referrer.should == '/some-referrer'
+  end
+  
+  it "does two things when crawled: follow, log, and handle" do
+    crawler = Relevance::Tarantula::Crawler.new
+    link = make_link('/foo', crawler)
+    
+    response = stub(:code => "200")
+    crawler.expects(:follow).returns(response)
+    link.expects(:log)
+    crawler.expects(:handle_link_results)
+    
+    link.crawl
+  end
+  
   # method_javascript_function needs this method
   def protect_against_forgery?
     false

data/lib/relevance/tarantula/attack.rb

@@ -10,6 +10,9 @@ class Relevance::Tarantula::Attack
   def ==(other)
     Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
   end
+  def input(input_field=nil)
+    @input
+  end
 end
 
 

data/lib/relevance/tarantula/attack_handler.rb

@@ -4,7 +4,7 @@ class Relevance::Tarantula::AttackHandler
   include ERB::Util
   
   def attacks
-    Relevance::Tarantula::AttackFormSubmission.attacks.select(&:output)
+    Relevance::Tarantula::FormSubmission.attacks.select(&:output)
   end
   
   def handle(result)

data/lib/relevance/tarantula/basic_attack.rb

@@ -0,0 +1,40 @@
+class Relevance::Tarantula::BasicAttack
+  ATTRS = [:name, :output, :description]
+
+  attr_reader *ATTRS
+
+  def initialize
+    @name = "Tarantula Basic Fuzzer"
+    @output = nil
+    @description = "Supplies purely random but simplistically generated form input."
+  end
+
+  def ==(other)
+    Relevance::Tarantula::BasicAttack === other && ATTRS.all? { |attr| send(attr) == other.send(attr)}
+  end
+
+  def input(input_field)
+    case input_field['name']
+      when /amount/         then random_int
+      when /_id$/           then random_whole_number
+      when /uploaded_data/  then nil
+      when nil              then input['value']
+      else
+        random_int
+    end
+  end
+
+  def big_number
+    10000   # arbitrary
+  end
+  
+  def random_int
+    rand(big_number) - (big_number/2)
+  end
+  
+  def random_whole_number
+    rand(big_number)
+  end
+end
+
+

data/lib/relevance/tarantula/crawler.rb

@@ -10,7 +10,7 @@ class Relevance::Tarantula::Crawler
   class CrawlTimeout < RuntimeError; end
 
   attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
-                :reporters, :links_to_crawl, :links_queued, :forms_to_crawl,
+                :reporters, :crawl_queue, :links_queued,
                 :form_signatures_queued, :max_url_length, :response_code_handler,
                 :times_to_crawl, :fuzzers, :test_name, :crawl_timeout
   attr_reader   :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
@@ -22,8 +22,7 @@ class Relevance::Tarantula::Crawler
     @handlers = [@response_code_handler = Result]
     @links_queued = Set.new
     @form_signatures_queued = Set.new
-    @links_to_crawl = []
-    @forms_to_crawl = []
+    @crawl_queue = []
     @crawl_start_times, @crawl_end_times = [], []
     @crawl_timeout = 20.minutes
     @referrers = {}
@@ -55,8 +54,7 @@ class Relevance::Tarantula::Crawler
   def crawl(url = "/")
     orig_links_queued = @links_queued.dup
     orig_form_signatures_queued = @form_signatures_queued.dup
-    orig_links_to_crawl = @links_to_crawl.dup
-    orig_forms_to_crawl = @forms_to_crawl.dup
+    orig_crawl_queue = @crawl_queue.dup
     @times_to_crawl.times do |num|
       queue_link url
       
@@ -71,8 +69,7 @@ class Relevance::Tarantula::Crawler
       if num + 1 < @times_to_crawl
         @links_queued = orig_links_queued
         @form_signatures_queued = orig_form_signatures_queued
-        @links_to_crawl = orig_links_to_crawl
-        @forms_to_crawl = orig_forms_to_crawl
+        @crawl_queue = orig_crawl_queue
         @referrers = {}
       end
     end
@@ -83,23 +80,20 @@ class Relevance::Tarantula::Crawler
   end
 
   def finished?
-    @links_to_crawl.empty? && @forms_to_crawl.empty?
+    @crawl_queue.empty?
   end
 
   def do_crawl(number)
     while (!finished?)
       @crawl_start_times << Time.now
-      crawl_queued_links(number)
-      crawl_queued_forms(number)
+      crawl_the_queue(number)
       @crawl_end_times << Time.now
     end
   end
 
-  def crawl_queued_links(number = 0)
-    while (link = @links_to_crawl.pop)
-      response = proxy.send(link.method, link.href)
-      log "Response #{response.code} for #{link}"
-      handle_link_results(link, response)
+  def crawl_the_queue(number = 0)
+    while (request = @crawl_queue.pop)
+      request.crawl
       blip(number)
     end
   end
@@ -110,15 +104,10 @@ class Relevance::Tarantula::Crawler
     end
   end
 
-  def handle_link_results(link, response)
+  def handle_link_results(link, result)
     handlers.each do |h|
       begin
-        save_result h.handle(Result.new(:method => link.method,
-                                       :url => link.href,
-                                       :response => response,
-                                       :log => grab_log!,
-                                       :referrer => referrers[link],
-                                       :test_name => test_name).freeze)
+        save_result h.handle(result)
       rescue Exception => e
         log "error handling #{link} #{e.message}"
         # TODO: pass to results
@@ -126,23 +115,14 @@ class Relevance::Tarantula::Crawler
     end
   end
 
-  def crawl_form(form)
-    response = proxy.send(form.method, form.action, form.data)
-    log "Response #{response.code} for #{form}"
-    response
-  rescue ActiveRecord::RecordNotFound => e
-    log "Skipping #{form.action}, presumed ok that record is missing"
-    Relevance::Tarantula::Response.new(:code => "404", :body => e.message, :content_type => "text/plain")
-  end
-
-  def crawl_queued_forms(number = 0)
-    while (form = @forms_to_crawl.pop)
-      response = crawl_form(form)
-      handle_form_results(form, response)
-      blip(number)
-    end
+  def follow(method, url, data=nil)
+    proxy.send(method, url, data)
   end
   
+  def submit(method, action, data)
+    proxy.send(method, action, data)
+  end
+
   def elasped_time_for_pass(num)
     Time.now - crawl_start_times[num]
   end
@@ -150,6 +130,14 @@ class Relevance::Tarantula::Crawler
   def grab_log!
     @log_grabber && @log_grabber.grab!
   end
+  
+  def make_result(options)
+    defaults = {
+      :log       => grab_log!,
+      :test_name => test_name      
+    }
+    Result.new(defaults.merge(options)).freeze
+  end
 
   def handle_form_results(form, response)
     handlers.each do |h|
@@ -193,23 +181,21 @@ class Relevance::Tarantula::Crawler
   end
 
   def queue_link(dest, referrer = nil)
-    dest = Link.new(dest)
-    dest.href = transform_url(dest.href)
+    dest = Link.new(dest, self, referrer)
     return if should_skip_link?(dest)
-    @referrers[dest] = referrer if referrer
-    @links_to_crawl << dest
+    @crawl_queue << dest
     @links_queued << dest
     dest
   end
 
   def queue_form(form, referrer = nil)
     fuzzers.each do |fuzzer|
-      fuzzer.mutate(Form.new(form)).each do |fs|
-        # fs = fuzzer.new(Form.new(form))
+      fuzzer.mutate(Form.new(form, self, referrer)).each do |fs|
+        # fs = fuzzer.new(Form.new(form, self, referrer))
         fs.action = transform_url(fs.action)
         return if should_skip_form_submission?(fs)
         @referrers[fs.action] = referrer if referrer
-        @forms_to_crawl << fs
+        @crawl_queue << fs
         @form_signatures_queued << fs.signature
       end
     end
@@ -234,6 +220,7 @@ class Relevance::Tarantula::Crawler
   end
 
   def report_results
+    puts "Crawled #{total_links_count} links and forms."
     generate_reports
   end
 
@@ -242,7 +229,7 @@ class Relevance::Tarantula::Crawler
   end
 
   def links_remaining_count
-    @links_to_crawl.size + @forms_to_crawl.size
+    @crawl_queue.size
   end
 
   def links_completed_count
@@ -251,7 +238,7 @@ class Relevance::Tarantula::Crawler
 
   def blip(number = 0)
     unless verbose
-      print "\r #{links_completed_count} of #{total_links_count} links completed               "
+      print "\r #{links_completed_count} of #{total_links_count} links completed               " if $stdout.tty?
       timeout_if_too_long(number)
     end
   end

data/lib/relevance/tarantula/form.rb

@@ -2,8 +2,10 @@ class Relevance::Tarantula::Form
   extend Forwardable
   def_delegators("@tag", :search)
   
-  def initialize(tag)
-    @tag = tag
+  attr_accessor :crawler, :referrer
+  
+  def initialize(tag, crawler, referrer)
+    @tag, @crawler, @referrer = tag, crawler, referrer
   end
   
   def action

data/lib/relevance/tarantula/form_submission.rb

@@ -1,25 +1,58 @@
 class Relevance::Tarantula::FormSubmission
-  attr_accessor :method, :action, :data
-  def initialize(form)
+  include Relevance::Tarantula
+  attr_accessor :method, :action, :data, :attack, :form
+
+  class << self
+    def attacks
+      # normalize from hash input to Attack
+      @attacks = @attacks.map do |val|
+        Hash === val ? Relevance::Tarantula::Attack.new(val) : val
+      end
+      @attacks
+    end
+    def attacks=(atts)
+      # normalize from hash input to Attack
+      @attacks = atts.map do |val|
+        Hash === val ? Relevance::Tarantula::Attack.new(val) : val
+      end
+    end
+  end
+  @attacks = [Relevance::Tarantula::BasicAttack.new]
+
+  def initialize(form, attack = Relevance::Tarantula::BasicAttack.new)
+    @form = form
     @method = form.method
     @action = form.action
+    @attack = attack
     @data = mutate_selects(form).merge(mutate_text_areas(form)).merge(mutate_inputs(form))
   end
   
+  def crawl
+    begin
+      response = form.crawler.submit(method, action, data)
+      log "Response #{response.code} for #{self}"
+    rescue ActiveRecord::RecordNotFound => e
+      log "Skipping #{action}, presumed ok that record is missing"
+      response = Relevance::Tarantula::Response.new(:code => "404", :body => e.message, :content_type => "text/plain")
+    end
+    form.crawler.handle_form_results(self, response)
+    response
+  end
+
   def self.mutate(form)
-    [self.new(form)]
+    attacks.map{|attack| new(form, attack)} if attacks
   end
-  
+
   def to_s
-    "#{action} #{method} #{data.inspect}"
+    "#{action} #{method} #{data.inspect} #{attack.inspect}"
   end
-  
+
   # a form's signature is what makes it unique (e.g. action + fields)
   # used to keep track of which forms we have submitted already
   def signature
-    [action, data.keys.sort]
+    [action, data.keys.sort, attack.name]
   end
-  
+
   def create_random_data_for(form, tag_selector)
     form.search(tag_selector).inject({}) do |form_args, input|
       # TODO: test
@@ -35,37 +68,21 @@ class Relevance::Tarantula::FormSubmission
   def mutate_text_areas(form)
     create_random_data_for(form, 'textarea')
   end
-  
+
   def mutate_selects(form)
     form.search('select').inject({}) do |form_args, select|
       options = select.search('option')
       option = options.rand
-      form_args[select['name']] = option['value'] 
+      form_args[select['name']] = option['value']
       form_args
     end
   end
-  
+
   def random_data(input)
     case input['name']
-      when /amount/         then random_int
-      when /_id$/           then random_whole_number
-      when /uploaded_data/  then nil
-      when /^_method$/      then input['value']
-      when nil              then input['value']
-      else                  
-        random_int
+    when /^_method$/      then input['value']
+    else
+      attack.input(input)
     end
   end
-  
-  def big_number
-    10000   # arbitrary
-  end
-  
-  def random_int
-    rand(big_number) - (big_number/2)
-  end
-  
-  def random_whole_number
-    rand(big_number)
-  end
 end

data/lib/relevance/tarantula/link.rb

@@ -1,4 +1,5 @@
 class Relevance::Tarantula::Link
+  include Relevance::Tarantula
   
   class << self
     include ActionView::Helpers::UrlHelper
@@ -17,18 +18,33 @@ class Relevance::Tarantula::Link
     METHOD_REGEXPS[m] = /#{s}/
   end
   
-  attr_accessor :href
+  attr_accessor :href, :crawler, :referrer
   
-  def initialize(link)
+  def initialize(link, crawler, referrer)
+    @crawler, @referrer = crawler, referrer
+    
     if String === link || link.nil?
-      @href = link
+      @href = transform_url(link)
       @method = :get
     else # should be a tag
-      @href = link['href'] ? link['href'].downcase : nil
+      @href = link['href'] ? transform_url(link['href'].downcase) : nil
       @tag = link
     end
   end
   
+  def crawl
+    response = crawler.follow(method, href)
+    log "Response #{response.code} for #{self}"
+    crawler.handle_link_results(self, make_result(response))
+  end
+  
+  def make_result(response)
+    crawler.make_result(:method    => method,
+                        :url       => href,
+                        :response  => response,
+                        :referrer  => referrer)
+  end
+  
   def method
     @method ||= begin
       (@tag &&
@@ -39,6 +55,10 @@ class Relevance::Tarantula::Link
     end
   end
   
+  def transform_url(link)
+    crawler.transform_url(link)
+  end
+  
   def ==(obj)
     obj.respond_to?(:href) && obj.respond_to?(:method) &&
       self.href.to_s == obj.href.to_s && self.method.to_s == obj.method.to_s

data/lib/relevance/tarantula/rails_integration_proxy.rb

@@ -41,7 +41,7 @@ class Relevance::Tarantula::RailsIntegrationProxy
     if response.code == '404'
       if File.exist?(static_content_path(url))
         case ext = File.extension(url)
-        when /html|te?xt|css|js|jpe?g|gif|psd|png|eps|pdf/
+        when /html|te?xt|css|js|jpe?g|gif|psd|png|eps|pdf|ico/
           response.body = static_content_file(url)
           response.headers["type"] = "text/#{ext}"  # readable as response.content_type
           response.meta.attr_accessor :code

data/lib/relevance/tarantula.rb

@@ -49,10 +49,10 @@ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "log_gra
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "invalid_html_handler"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "transform"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "crawler"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "basic_attack"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form_submission"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_form_submission"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_handler"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "link"))
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: relevance-tarantula
 version: !ruby/object:Gem::Version 
-  version: 0.2.1
+  version: 0.3.2
 platform: ruby
 authors: 
 - Relevance, Inc.
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-08-21 00:00:00 -07:00
+date: 2009-09-24 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -51,8 +51,8 @@ files:
 - examples/relevance/core_extensions/file_example.rb
 - examples/relevance/core_extensions/response_example.rb
 - examples/relevance/core_extensions/test_case_example.rb
-- examples/relevance/tarantula/attack_form_submission_example.rb
 - examples/relevance/tarantula/attack_handler_example.rb
+- examples/relevance/tarantula/basic_attack_example.rb
 - examples/relevance/tarantula/crawler_example.rb
 - examples/relevance/tarantula/form_example.rb
 - examples/relevance/tarantula/form_submission_example.rb
@@ -84,8 +84,8 @@ files:
 - lib/relevance/core_extensions/test_case.rb
 - lib/relevance/tarantula.rb
 - lib/relevance/tarantula/attack.rb
-- lib/relevance/tarantula/attack_form_submission.rb
 - lib/relevance/tarantula/attack_handler.rb
+- lib/relevance/tarantula/basic_attack.rb
 - lib/relevance/tarantula/crawler.rb
 - lib/relevance/tarantula/detail.html.erb
 - lib/relevance/tarantula/form.rb
@@ -107,7 +107,7 @@ files:
 - lib/relevance/tarantula/transform.rb
 - tasks/tarantula_tasks.rake
 - template/tarantula_test.rb
-has_rdoc: true
+has_rdoc: false
 homepage: http://github.com/relevance/tarantula
 licenses: 
 post_install_message: 
@@ -132,7 +132,7 @@ requirements: []
 rubyforge_project: thinkrelevance
 rubygems_version: 1.3.5
 signing_key: 
-specification_version: 2
+specification_version: 3
 summary: A big hairy fuzzy spider that crawls your site, wreaking havoc
 test_files: 
 - examples/example_helper.rb
@@ -140,8 +140,8 @@ test_files:
 - examples/relevance/core_extensions/file_example.rb
 - examples/relevance/core_extensions/response_example.rb
 - examples/relevance/core_extensions/test_case_example.rb
-- examples/relevance/tarantula/attack_form_submission_example.rb
 - examples/relevance/tarantula/attack_handler_example.rb
+- examples/relevance/tarantula/basic_attack_example.rb
 - examples/relevance/tarantula/crawler_example.rb
 - examples/relevance/tarantula/form_example.rb
 - examples/relevance/tarantula/form_submission_example.rb

data/examples/relevance/tarantula/attack_form_submission_example.rb

@@ -1,79 +0,0 @@
-require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
-
-describe "Relevance::Tarantula::AttackFormSubmission" do
-  
-  # TODO: add more from field types to this example form as needed
-  before do
-    @tag = Hpricot(<<END)
-<form action="/session" method="post">
-  <input id="email" name="email" size="30" type="text" />
-  <textarea id="comment" name="comment"value="1" />
-  <input name="commit" type="submit" value="Postit" />
-  <input name="secret" type="hidden" value="secret" />
-  <select id="foo_opened_on_1i" name="foo[opened_on(1i)]">
-    <option value="2003">2003</option>
-    <option value="2004">2004</option>
-  </select> 
-</form>
-END
-    @form = Relevance::Tarantula::Form.new(@tag.at('form'))
-    @fs = Relevance::Tarantula::AttackFormSubmission.new(@form, Relevance::Tarantula::Attack.new({:name => 'foo_name', :input => 'foo_code', :output => 'foo_code'}))
-  end
-  
-  it "can mutate text areas" do
-    @fs.mutate_text_areas(@form).should == {"comment" => "foo_code"}
-  end
-  
-  it "can mutate selects" do
-    Hpricot::Elements.any_instance.stubs(:rand).returns(stub(:[] => "2006-stub"))
-    @fs.mutate_selects(@form).should == {"foo[opened_on(1i)]" => "2006-stub"}
-  end
-  
-  it "can mutate inputs" do
-    @fs.mutate_inputs(@form).should == {"commit"=>"foo_code", "secret"=>"foo_code", "email"=>"foo_code"}
-  end
-
-  it "has a signature based on action,  fields, and attack name" do
-    @fs.signature.should == ['/session', [
-      "comment", 
-      "commit", 
-      "email", 
-      "foo[opened_on(1i)]", 
-      "secret"],
-      "foo_name"
-    ]
-  end
-  
-  it "has a friendly to_s" do
-    @fs.to_s.should =~ %r{^/session post}
-  end
-  
-  it "processes all its attacks" do
-    Relevance::Tarantula::AttackFormSubmission.stubs(:attacks).returns([
-      Relevance::Tarantula::Attack.new({:name => 'foo_name1', :input => 'foo_input', :output => 'foo_output'}),
-      Relevance::Tarantula::Attack.new({:name => 'foo_name2', :input => 'foo_input', :output => 'foo_output'}),
-    ])
-    Relevance::Tarantula::AttackFormSubmission.mutate(@form).size.should == 2
-  end
-  
-  it "maps hash attacks to Attack instances" do
-    Relevance::Tarantula::AttackFormSubmission.instance_variable_set("@attacks", [{ :name => "attack name"}])
-    Relevance::Tarantula::AttackFormSubmission.attacks.should == [Relevance::Tarantula::Attack.new({:name => "attack name"})]
-  end
-end
-
-describe "Relevance::Tarantula::AttackFormSubmission for a crummy form" do
-  before do
-    @tag = Hpricot(<<END)
-<form action="/session" method="post">
-  <input value="no_name" />
-</form>
-END
-    @form = Relevance::Tarantula::Form.new(@tag.at('form'))
-    @fs = Relevance::Tarantula::AttackFormSubmission.new(@form, {:name => 'foo_name', :input => 'foo_code', :output => 'foo_code'})
-  end
-  
-  it "ignores unnamed inputs" do
-    @fs.mutate_inputs(@form).should == {}
-  end
-end