tarantula 0.2.0 → 0.3.3

data/examples/relevance/tarantula/link_example.rb

@@ -4,47 +4,64 @@ describe "Relevance::Tarantula::Link" do
   include ActionView::Helpers::UrlHelper
   
   it "does not raise an error when initializing without href attribtue" do
-    link = Relevance::Tarantula::Link.new(Hpricot('<a="/foo">foo</a>').at('a'))
+    link = make_link(Hpricot('<a="/foo">foo</a>').at('a'))
     link.href.should == nil
     link.method.should == :get    
   end
 
   it "parses anchor tags" do
-    link = Relevance::Tarantula::Link.new(Hpricot('<a href="/foo">foo</a>').at('a'))
+    link = make_link(Hpricot('<a href="/foo">foo</a>').at('a'))
     link.href.should == '/foo'
     link.method.should == :get
   end
 
   it "parses anchor tags with POST 'method'" do
-    link = Relevance::Tarantula::Link.new(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:post)}">foo</a>}).at('a'))
+    link = make_link(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:post)}">foo</a>}).at('a'))
     link.href.should == '/foo'
     link.method.should == :post
   end
 
   it "parses anchor tags with PUT 'method'" do
-    link = Relevance::Tarantula::Link.new(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:put)}">foo</a>}).at('a'))
+    link = make_link(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:put)}">foo</a>}).at('a'))
     link.href.should == '/foo'
     link.method.should == :put
   end
 
   it "parses anchor tags with DELETE 'method'" do
-    link = Relevance::Tarantula::Link.new(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:delete)}">foo</a>}).at('a'))
+    link = make_link(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:delete)}">foo</a>}).at('a'))
     link.href.should == '/foo'
     link.method.should == :delete
   end
 
   it "parses link tags with text" do
-    link = Relevance::Tarantula::Link.new(Hpricot('<link href="/bar">bar</a>').at('link'))
+    link = make_link(Hpricot('<link href="/bar">bar</a>').at('link'))
     link.href.should == '/bar'
     link.method.should == :get
   end
   
   it "parses link tags without text" do
-    link = Relevance::Tarantula::Link.new(Hpricot('<link href="/bar" />').at('link'))
+    link = make_link(Hpricot('<link href="/bar" />').at('link'))
     link.href.should == '/bar'
     link.method.should == :get
   end
   
+  it 'remembers link referrer if there is one' do
+    link = make_link('/url', stub_everything, '/some-referrer')
+    link.referrer.should == '/some-referrer'
+  end
+  
+  it "does two things when crawled: follow, log, and handle" do
+    crawler = Relevance::Tarantula::Crawler.new
+    link = make_link('/foo', crawler)
+    
+    response = stub(:code => "200")
+    crawler.expects(:follow).returns(response)
+    link.expects(:log)
+    crawler.expects(:handle_link_results)
+    
+    link.crawl
+  end
+  
   # method_javascript_function needs this method
   def protect_against_forgery?
     false

data/examples/relevance/tarantula/rails_integration_proxy_example.rb

@@ -41,7 +41,7 @@ describe "Relevance::Tarantula::RailsIntegrationProxy" do
   it "adds a response accessor to its delegate rails integration test" do
     o = Object.new
     Relevance::Tarantula::RailsIntegrationProxy.new(o)
-    o.methods(false).sort.should == %w{response response=}
+    o.methods(false).map(&:to_s).sort.should == %w{response response=}
   end
 
 end

data/lib/relevance/tarantula/attack.rb

@@ -10,6 +10,9 @@ class Relevance::Tarantula::Attack
   def ==(other)
     Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
   end
+  def input(input_field=nil)
+    @input
+  end
 end
 
 

data/lib/relevance/tarantula/attack_handler.rb

@@ -4,7 +4,7 @@ class Relevance::Tarantula::AttackHandler
   include ERB::Util
   
   def attacks
-    Relevance::Tarantula::AttackFormSubmission.attacks.select(&:output)
+    Relevance::Tarantula::FormSubmission.attacks.select(&:output)
   end
   
   def handle(result)

data/lib/relevance/tarantula/basic_attack.rb

@@ -0,0 +1,40 @@
+class Relevance::Tarantula::BasicAttack
+  ATTRS = [:name, :output, :description]
+
+  attr_reader *ATTRS
+
+  def initialize
+    @name = "Tarantula Basic Fuzzer"
+    @output = nil
+    @description = "Supplies purely random but simplistically generated form input."
+  end
+
+  def ==(other)
+    Relevance::Tarantula::BasicAttack === other && ATTRS.all? { |attr| send(attr) == other.send(attr)}
+  end
+
+  def input(input_field)
+    case input_field['name']
+      when /amount/         then random_int
+      when /_id$/           then random_whole_number
+      when /uploaded_data/  then nil
+      when nil              then input['value']
+      else
+        random_int
+    end
+  end
+
+  def big_number
+    10000   # arbitrary
+  end
+  
+  def random_int
+    rand(big_number) - (big_number/2)
+  end
+  
+  def random_whole_number
+    rand(big_number)
+  end
+end
+
+

data/lib/relevance/tarantula/crawler.rb

@@ -10,7 +10,7 @@ class Relevance::Tarantula::Crawler
   class CrawlTimeout < RuntimeError; end
 
   attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
-                :reporters, :links_to_crawl, :links_queued, :forms_to_crawl,
+                :reporters, :crawl_queue, :links_queued,
                 :form_signatures_queued, :max_url_length, :response_code_handler,
                 :times_to_crawl, :fuzzers, :test_name, :crawl_timeout
   attr_reader   :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
@@ -22,8 +22,7 @@ class Relevance::Tarantula::Crawler
     @handlers = [@response_code_handler = Result]
     @links_queued = Set.new
     @form_signatures_queued = Set.new
-    @links_to_crawl = []
-    @forms_to_crawl = []
+    @crawl_queue = []
     @crawl_start_times, @crawl_end_times = [], []
     @crawl_timeout = 20.minutes
     @referrers = {}
@@ -39,6 +38,8 @@ class Relevance::Tarantula::Crawler
     @decoder = HTMLEntities.new
     @times_to_crawl = 1
     @fuzzers = [Relevance::Tarantula::FormSubmission]
+    
+    @stdout_tty = $stdout.tty?
   end
 
   def method_missing(meth, *args)
@@ -55,14 +56,14 @@ class Relevance::Tarantula::Crawler
   def crawl(url = "/")
     orig_links_queued = @links_queued.dup
     orig_form_signatures_queued = @form_signatures_queued.dup
-    orig_links_to_crawl = @links_to_crawl.dup
-    orig_forms_to_crawl = @forms_to_crawl.dup
+    orig_crawl_queue = @crawl_queue.dup
     @times_to_crawl.times do |num|
       queue_link url
       
       begin 
         do_crawl num
       rescue CrawlTimeout => e
+        puts
         puts e.message
       end
       
@@ -71,8 +72,7 @@ class Relevance::Tarantula::Crawler
       if num + 1 < @times_to_crawl
         @links_queued = orig_links_queued
         @form_signatures_queued = orig_form_signatures_queued
-        @links_to_crawl = orig_links_to_crawl
-        @forms_to_crawl = orig_forms_to_crawl
+        @crawl_queue = orig_crawl_queue
         @referrers = {}
       end
     end
@@ -83,23 +83,20 @@ class Relevance::Tarantula::Crawler
   end
 
   def finished?
-    @links_to_crawl.empty? && @forms_to_crawl.empty?
+    @crawl_queue.empty?
   end
 
   def do_crawl(number)
     while (!finished?)
       @crawl_start_times << Time.now
-      crawl_queued_links(number)
-      crawl_queued_forms(number)
+      crawl_the_queue(number)
       @crawl_end_times << Time.now
     end
   end
 
-  def crawl_queued_links(number = 0)
-    while (link = @links_to_crawl.pop)
-      response = proxy.send(link.method, link.href)
-      log "Response #{response.code} for #{link}"
-      handle_link_results(link, response)
+  def crawl_the_queue(number = 0)
+    while (request = @crawl_queue.pop)
+      request.crawl
       blip(number)
     end
   end
@@ -110,15 +107,10 @@ class Relevance::Tarantula::Crawler
     end
   end
 
-  def handle_link_results(link, response)
+  def handle_link_results(link, result)
     handlers.each do |h|
       begin
-        save_result h.handle(Result.new(:method => link.method,
-                                       :url => link.href,
-                                       :response => response,
-                                       :log => grab_log!,
-                                       :referrer => referrers[link],
-                                       :test_name => test_name).freeze)
+        save_result h.handle(result)
       rescue Exception => e
         log "error handling #{link} #{e.message}"
         # TODO: pass to results
@@ -126,23 +118,14 @@ class Relevance::Tarantula::Crawler
     end
   end
 
-  def crawl_form(form)
-    response = proxy.send(form.method, form.action, form.data)
-    log "Response #{response.code} for #{form}"
-    response
-  rescue ActiveRecord::RecordNotFound => e
-    log "Skipping #{form.action}, presumed ok that record is missing"
-    Relevance::Tarantula::Response.new(:code => "404", :body => e.message, :content_type => "text/plain")
-  end
-
-  def crawl_queued_forms(number = 0)
-    while (form = @forms_to_crawl.pop)
-      response = crawl_form(form)
-      handle_form_results(form, response)
-      blip(number)
-    end
+  def follow(method, url, data=nil)
+    proxy.send(method, url, data)
   end
   
+  def submit(method, action, data)
+    proxy.send(method, action, data)
+  end
+
   def elasped_time_for_pass(num)
     Time.now - crawl_start_times[num]
   end
@@ -150,6 +133,14 @@ class Relevance::Tarantula::Crawler
   def grab_log!
     @log_grabber && @log_grabber.grab!
   end
+  
+  def make_result(options)
+    defaults = {
+      :log       => grab_log!,
+      :test_name => test_name      
+    }
+    Result.new(defaults.merge(options)).freeze
+  end
 
   def handle_form_results(form, response)
     handlers.each do |h|
@@ -193,23 +184,21 @@ class Relevance::Tarantula::Crawler
   end
 
   def queue_link(dest, referrer = nil)
-    dest = Link.new(dest)
-    dest.href = transform_url(dest.href)
+    dest = Link.new(dest, self, referrer)
     return if should_skip_link?(dest)
-    @referrers[dest] = referrer if referrer
-    @links_to_crawl << dest
+    @crawl_queue << dest
     @links_queued << dest
     dest
   end
 
   def queue_form(form, referrer = nil)
     fuzzers.each do |fuzzer|
-      fuzzer.mutate(Form.new(form)).each do |fs|
-        # fs = fuzzer.new(Form.new(form))
+      fuzzer.mutate(Form.new(form, self, referrer)).each do |fs|
+        # fs = fuzzer.new(Form.new(form, self, referrer))
         fs.action = transform_url(fs.action)
         return if should_skip_form_submission?(fs)
         @referrers[fs.action] = referrer if referrer
-        @forms_to_crawl << fs
+        @crawl_queue << fs
         @form_signatures_queued << fs.signature
       end
     end
@@ -234,6 +223,7 @@ class Relevance::Tarantula::Crawler
   end
 
   def report_results
+    puts "Crawled #{total_links_count} links and forms."
     generate_reports
   end
 
@@ -242,7 +232,7 @@ class Relevance::Tarantula::Crawler
   end
 
   def links_remaining_count
-    @links_to_crawl.size + @forms_to_crawl.size
+    @crawl_queue.size
   end
 
   def links_completed_count
@@ -251,7 +241,7 @@ class Relevance::Tarantula::Crawler
 
   def blip(number = 0)
     unless verbose
-      print "\r #{links_completed_count} of #{total_links_count} links completed               "
+      print "\r #{links_completed_count} of #{total_links_count} links completed               " if @stdout_tty
       timeout_if_too_long(number)
     end
   end

data/lib/relevance/tarantula/detail.html.erb

@@ -25,18 +25,18 @@
       </ul>
 
       <div id="report">
-        <h3>Detail of <%= short_description %> <em>Generated on <%= Time.now %></em></h3>
-        <p><b>Resource</b> <a href="<%= full_url %>"><%= full_url %></a></p>
-        <p><b>Response</b> <span class="r<%= code.first %>"><%= code %></span></p>
-        <p><b>Referrer</b> <%= referrer || "" %></p>
+        <h3>Detail of <%= result.short_description %> <em>Generated on <%= Time.now %></em></h3>
+        <p><b>Resource</b> <a href="<%= result.full_url %>"><%= result.full_url %></a></p>
+        <p><b>Response</b> <span class="r<%= result.code.first %>"><%= result.code %></span></p>
+        <p><b>Referrer</b> <%= result.referrer || "" %></p>
 
         <table class="output">
         <tbody>
           <tr>                
             <th colspan="2">#&nbsp;&nbsp;Data</th>           
           </tr>
-          <% if data %>
-            <%= wrap_in_line_number_table_row(data) %>
+          <% if result.data %>
+            <%= result.wrap_in_line_number_table_row(result.data) %>
           <% else %>
             <tr>
               <td colspan="2">No Data</td>
@@ -50,8 +50,8 @@
           <tr>
             <th colspan="2">#&nbsp;&nbsp;Body</th>           
           </tr>
-          <% if body %>
-            <%= wrap_in_line_number_table_row(body) %>
+          <% if result.body %>
+            <%= result.wrap_in_line_number_table_row(result.body) %>
           <% else %>
             <tr>
               <td colspan="2">No Body</td>
@@ -65,8 +65,8 @@
             <tr>
               <th colspan="2">#&nbsp;&nbsp;Log</th>           
             </tr>
-            <% if log %>
-              <%= wrap_in_line_number_table_row(log) {|line| wrap_stack_trace_line(line)} %>
+            <% if result.log %>
+              <%= result.wrap_in_line_number_table_row(result.log) {|line| wrap_stack_trace_line(line)} %>
             <% else %>
               <tr>
                 <td colspan="2">No Log</td>
@@ -78,4 +78,4 @@
     </div>
   </div>
 </body>
-</html>
+</html>

data/lib/relevance/tarantula/form.rb

@@ -2,8 +2,10 @@ class Relevance::Tarantula::Form
   extend Forwardable
   def_delegators("@tag", :search)
   
-  def initialize(tag)
-    @tag = tag
+  attr_accessor :crawler, :referrer
+  
+  def initialize(tag, crawler, referrer)
+    @tag, @crawler, @referrer = tag, crawler, referrer
   end
   
   def action

data/lib/relevance/tarantula/form_submission.rb

@@ -1,25 +1,58 @@
 class Relevance::Tarantula::FormSubmission
-  attr_accessor :method, :action, :data
-  def initialize(form)
+  include Relevance::Tarantula
+  attr_accessor :method, :action, :data, :attack, :form
+
+  class << self
+    def attacks
+      # normalize from hash input to Attack
+      @attacks = @attacks.map do |val|
+        Hash === val ? Relevance::Tarantula::Attack.new(val) : val
+      end
+      @attacks
+    end
+    def attacks=(atts)
+      # normalize from hash input to Attack
+      @attacks = atts.map do |val|
+        Hash === val ? Relevance::Tarantula::Attack.new(val) : val
+      end
+    end
+  end
+  @attacks = [Relevance::Tarantula::BasicAttack.new]
+
+  def initialize(form, attack = Relevance::Tarantula::BasicAttack.new)
+    @form = form
     @method = form.method
     @action = form.action
+    @attack = attack
     @data = mutate_selects(form).merge(mutate_text_areas(form)).merge(mutate_inputs(form))
   end
   
+  def crawl
+    begin
+      response = form.crawler.submit(method, action, data)
+      log "Response #{response.code} for #{self}"
+    rescue ActiveRecord::RecordNotFound => e
+      log "Skipping #{action}, presumed ok that record is missing"
+      response = Relevance::Tarantula::Response.new(:code => "404", :body => e.message, :content_type => "text/plain")
+    end
+    form.crawler.handle_form_results(self, response)
+    response
+  end
+
   def self.mutate(form)
-    [self.new(form)]
+    attacks.map{|attack| new(form, attack)} if attacks
   end
-  
+
   def to_s
-    "#{action} #{method} #{data.inspect}"
+    "#{action} #{method} #{data.inspect} #{attack.inspect}"
   end
-  
+
   # a form's signature is what makes it unique (e.g. action + fields)
   # used to keep track of which forms we have submitted already
   def signature
-    [action, data.keys.sort]
+    [action, data.keys.sort, attack.name]
   end
-  
+
   def create_random_data_for(form, tag_selector)
     form.search(tag_selector).inject({}) do |form_args, input|
       # TODO: test
@@ -35,36 +68,21 @@ class Relevance::Tarantula::FormSubmission
   def mutate_text_areas(form)
     create_random_data_for(form, 'textarea')
   end
-  
+
   def mutate_selects(form)
     form.search('select').inject({}) do |form_args, select|
       options = select.search('option')
       option = options.rand
-      form_args[select['name']] = option['value'] 
+      form_args[select['name']] = option['value']
       form_args
     end
   end
-  
+
   def random_data(input)
     case input['name']
-      when /amount/         : random_int
-      when /_id$/           : random_whole_number
-      when /uploaded_data/  : nil
-      when /^_method$/      : input['value']
-      when nil              : input['value']
-      else                    random_int
+    when /^_method$/      then input['value']
+    else
+      attack.input(input)
     end
   end
-  
-  def big_number
-    10000   # arbitrary
-  end
-  
-  def random_int
-    rand(big_number) - (big_number/2)
-  end
-  
-  def random_whole_number
-    rand(big_number)
-  end
 end

data/lib/relevance/tarantula/link.rb

@@ -1,4 +1,5 @@
 class Relevance::Tarantula::Link
+  include Relevance::Tarantula
   
   class << self
     include ActionView::Helpers::UrlHelper
@@ -17,18 +18,33 @@ class Relevance::Tarantula::Link
     METHOD_REGEXPS[m] = /#{s}/
   end
   
-  attr_accessor :href
+  attr_accessor :href, :crawler, :referrer
   
-  def initialize(link)
+  def initialize(link, crawler, referrer)
+    @crawler, @referrer = crawler, referrer
+    
     if String === link || link.nil?
-      @href = link
+      @href = transform_url(link)
       @method = :get
     else # should be a tag
-      @href = link['href'] ? link['href'].downcase : nil
+      @href = link['href'] ? transform_url(link['href'].downcase) : nil
       @tag = link
     end
   end
   
+  def crawl
+    response = crawler.follow(method, href)
+    log "Response #{response.code} for #{self}"
+    crawler.handle_link_results(self, make_result(response))
+  end
+  
+  def make_result(response)
+    crawler.make_result(:method    => method,
+                        :url       => href,
+                        :response  => response,
+                        :referrer  => referrer)
+  end
+  
   def method
     @method ||= begin
       (@tag &&
@@ -39,6 +55,10 @@ class Relevance::Tarantula::Link
     end
   end
   
+  def transform_url(link)
+    crawler.transform_url(link)
+  end
+  
   def ==(obj)
     obj.respond_to?(:href) && obj.respond_to?(:method) &&
       self.href.to_s == obj.href.to_s && self.method.to_s == obj.method.to_s

data/lib/relevance/tarantula/rails_integration_proxy.rb

@@ -41,7 +41,7 @@ class Relevance::Tarantula::RailsIntegrationProxy
     if response.code == '404'
       if File.exist?(static_content_path(url))
         case ext = File.extension(url)
-        when /html|te?xt|css|js|jpe?g|gif|psd|png|eps|pdf/
+        when /html|te?xt|css|js|jpe?g|gif|psd|png|eps|pdf|ico/
           response.body = static_content_file(url)
           response.headers["type"] = "text/#{ext}"  # readable as response.content_type
           response.meta.attr_accessor :code

data/lib/relevance/tarantula/result.rb

@@ -11,33 +11,43 @@ class Relevance::Tarantula::Result
       self.instance_variable_set("@#{k}", v)
     end
   end
+
   def short_description
     [method,url].join(" ")
   end
+
   def sequence_number
     @sequence_number ||= (self.class.next_number += 1)
   end
+
   def file_name
     "#{sequence_number}.html"
   end
+
   def code
     response && response.code
   end
+
   def body
     response && response.body
   end
+
   def full_url
     "#{DEFAULT_LOCALHOST}#{url}"
   end
+
   ALLOW_NNN_FOR = /^allow_(\d\d\d)_for$/
+
   class << self
     attr_accessor :next_number
+    
     def handle(result)
       retval = result.dup
       retval.success = successful?(result.response) || can_skip_error?(result)
       retval.description = "Bad HTTP Response" unless retval.success
       retval
     end
+
     def success_codes 
       %w{200 201 302 401}
     end
@@ -51,16 +61,17 @@ class Relevance::Tarantula::Result
       return false unless coll
       coll.any? {|item| item === result.url}
     end
+
     def successful?(response)
       success_codes.member?(response.code)
     end
+
     def method_missing(meth, *args)  
       super unless ALLOW_NNN_FOR =~ meth.to_s
       (allow_errors_for[$1] ||= []).push(*args)
     end
   end
+
   self.allow_errors_for = {}
   self.next_number = 0
-  
-  
-end
+end

data/lib/relevance/tarantula.rb

@@ -49,10 +49,10 @@ require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "log_gra
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "invalid_html_handler"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "transform"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "crawler"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "basic_attack"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form_submission"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack"))
-require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_form_submission"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_handler"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "link"))