romanbsd-tarantula 0.1.8

data/lib/relevance/tarantula/crawler.rb

@@ -0,0 +1,264 @@
+require 'active_record'
+require 'active_record/base'
+require File.expand_path(File.join(File.dirname(__FILE__), "rails_integration_proxy"))
+require File.expand_path(File.join(File.dirname(__FILE__), "html_document_handler.rb"))
+
+class Relevance::Tarantula::Crawler
+  extend Forwardable
+  include Relevance::Tarantula
+
+  class CrawlTimeout < RuntimeError; end
+
+  attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
+                :reporters, :links_to_crawl, :links_queued, :forms_to_crawl,
+                :form_signatures_queued, :max_url_length, :response_code_handler,
+                :times_to_crawl, :fuzzers, :test_name, :crawl_timeout
+  attr_reader   :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
+
+  def initialize
+    @max_url_length = 1024
+    @successes = []
+    @failures = []
+    @handlers = [@response_code_handler = Result]
+    @links_queued = Set.new
+    @form_signatures_queued = Set.new
+    @links_to_crawl = []
+    @forms_to_crawl = []
+    @crawl_start_times, @crawl_end_times = [], []
+    @crawl_timeout = 20.minutes
+    @referrers = {}
+    @skip_uri_patterns = [
+      /^javascript/,
+      /^mailto/,
+      /^http/,
+    ]
+    self.transform_url_patterns = [
+      [/#.*$/, '']
+    ]
+    @reporters = [Relevance::Tarantula::IOReporter.new($stderr)]
+    @decoder = HTMLEntities.new
+    @times_to_crawl = 1
+    @fuzzers = [Relevance::Tarantula::FormSubmission]
+  end
+
+  def method_missing(meth, *args)
+    super unless Result::ALLOW_NNN_FOR =~ meth.to_s
+    @response_code_handler.send(meth, *args)
+  end
+
+  def transform_url_patterns=(patterns)
+    @transform_url_patterns = patterns.map do |pattern|
+      Array === pattern ? Relevance::Tarantula::Transform.new(*pattern) : pattern
+    end
+  end
+
+  def crawl(url = "/")
+    orig_links_queued = @links_queued.dup
+    orig_form_signatures_queued = @form_signatures_queued.dup
+    orig_links_to_crawl = @links_to_crawl.dup
+    orig_forms_to_crawl = @forms_to_crawl.dup
+    @times_to_crawl.times do |num|
+      queue_link url
+      
+      begin 
+        do_crawl num
+      rescue CrawlTimeout => e
+        puts e.message
+      end
+      
+      puts "#{(num+1).ordinalize} crawl" if @times_to_crawl > 1
+
+      if num + 1 < @times_to_crawl
+        @links_queued = orig_links_queued
+        @form_signatures_queued = orig_form_signatures_queued
+        @links_to_crawl = orig_links_to_crawl
+        @forms_to_crawl = orig_forms_to_crawl
+        @referrers = {}
+      end
+    end
+  rescue Interrupt
+    $stderr.puts "CTRL-C"
+  ensure
+    report_results
+  end
+
+  def finished?
+    @links_to_crawl.empty? && @forms_to_crawl.empty?
+  end
+
+  def do_crawl(number)
+    while (!finished?)
+      @crawl_start_times << Time.now
+      crawl_queued_links(number)
+      crawl_queued_forms(number)
+      @crawl_end_times << Time.now
+    end
+  end
+
+  def crawl_queued_links(number = 0)
+    while (link = @links_to_crawl.pop)
+      response = proxy.send(link.method, link.href)
+      log "Response #{response.code} for #{link}"
+      handle_link_results(link, response)
+      blip(number)
+    end
+  end
+
+  def save_result(result)
+    reporters.each do |reporter|
+      reporter.report(result)
+    end
+  end
+
+  def handle_link_results(link, response)
+    handlers.each do |h|
+      begin
+        save_result h.handle(Result.new(:method => link.method,
+                                       :url => link.href,
+                                       :response => response,
+                                       :log => grab_log!,
+                                       :referrer => referrers[link],
+                                       :test_name => test_name).freeze)
+      rescue Exception => e
+        log "error handling #{link} #{e.message}"
+        # TODO: pass to results
+      end
+    end
+  end
+
+  def crawl_form(form)
+    response = proxy.send(form.method, form.action, form.data)
+    log "Response #{response.code} for #{form}"
+    response
+  rescue ActiveRecord::RecordNotFound => e
+    log "Skipping #{form.action}, presumed ok that record is missing"
+    Relevance::Tarantula::Response.new(:code => "404", :body => e.message, :content_type => "text/plain")
+  end
+
+  def crawl_queued_forms(number = 0)
+    while (form = @forms_to_crawl.pop)
+      response = crawl_form(form)
+      handle_form_results(form, response)
+      blip(number)
+    end
+  end
+  
+  def elasped_time_for_pass(num)
+    Time.now - crawl_start_times[num]
+  end
+
+  def grab_log!
+    @log_grabber && @log_grabber.grab!
+  end
+
+  def handle_form_results(form, response)
+    handlers.each do |h|
+      save_result h.handle(Result.new(:method => form.method,
+                                     :url => form.action,
+                                     :response => response,
+                                     :log => grab_log!,
+                                     :referrer => form.action,
+                                     :data => form.data.inspect,
+                                     :test_name => test_name).freeze)
+    end
+  end
+
+  def should_skip_url?(url)
+    return true if url.blank?
+    if @skip_uri_patterns.any? {|pattern| pattern =~ url}
+      log "Skipping #{url}"
+      return true
+    end
+    if url.length > max_url_length
+      log "Skipping long url #{url}"
+      return true
+    end
+  end
+
+  def should_skip_link?(link)
+    should_skip_url?(link.href) || @links_queued.member?(link)
+  end
+
+  def should_skip_form_submission?(fs)
+    should_skip_url?(fs.action) || @form_signatures_queued.member?(fs.signature)
+  end
+
+  def transform_url(url)
+    return unless url
+    url = @decoder.decode(url)
+    @transform_url_patterns.each do |pattern|
+      url = pattern[url]
+    end
+    url
+  end
+
+  def queue_link(dest, referrer = nil)
+    dest = Link.new(dest)
+    dest.href = transform_url(dest.href)
+    return if should_skip_link?(dest)
+    @referrers[dest] = referrer if referrer
+    @links_to_crawl << dest
+    @links_queued << dest
+    dest
+  end
+
+  def queue_form(form, referrer = nil)
+    fuzzers.each do |fuzzer|
+      fuzzer.mutate(Form.new(form)).each do |fs|
+        # fs = fuzzer.new(Form.new(form))
+        fs.action = transform_url(fs.action)
+        return if should_skip_form_submission?(fs)
+        @referrers[fs.action] = referrer if referrer
+        @forms_to_crawl << fs
+        @form_signatures_queued << fs.signature
+      end
+    end
+  end
+
+  def report_dir
+    File.join(rails_root, "tmp", "tarantula")
+  end
+
+  def generate_reports
+    errors = []
+    reporters.each do |reporter|
+      begin
+        reporter.finish_report(test_name)
+      rescue RuntimeError => e
+        errors << e
+      end
+    end
+    unless errors.empty?
+      raise errors.map(&:message).join("\n")
+    end
+  end
+
+  def report_results
+    generate_reports
+  end
+
+  def total_links_count
+    @links_queued.size + @form_signatures_queued.size
+  end
+
+  def links_remaining_count
+    @links_to_crawl.size + @forms_to_crawl.size
+  end
+
+  def links_completed_count
+      total_links_count - links_remaining_count
+  end
+
+  def blip(number = 0)
+    unless verbose
+      print "\r #{links_completed_count} of #{total_links_count} links completed               "
+      timeout_if_too_long(number)
+    end
+  end
+  
+  def timeout_if_too_long(number = 0)
+    if elasped_time_for_pass(number) > crawl_timeout
+      raise CrawlTimeout, "Exceeded crawl timeout of #{crawl_timeout} seconds - skipping to the next crawl..."
+    end
+  end
+end

data/lib/relevance/tarantula/detail.html.erb

@@ -0,0 +1,82 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+	"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
+  <link rel="stylesheet" href="../stylesheets/tarantula.css" type="text/css" media="screen" title="no title" charset="utf-8" />
+	<title>Detail</title>
+</head>
+<body>
+  <div id="container">
+    <div id="header">
+      <h1>Tarantula by Relevance</h1>
+      <h2>Eight legs, two fangs ... and an attitude</h2>
+      <p>Tarantula is an open source tool for testing Rails web 
+      applications. Tarantula is developed by <a href="http://thinkrelevance.com">Relevance, Inc.</a>
+      and lives at <a href="http://github.com/relevance/tarantula">http://github.com/relevance/tarantula</a>.</p>
+      <hr/>
+    </div>
+    <div id="page">
+      <ul class="tabs">
+        <li><a href="../index.html">&laquo; Back</a></li>
+        <!-- TODO dynamically hide/show body/log sections based on selected "tab" - dynamically assign the "active" class to the selected tab -->
+        <li><a href="#fragment-1" class="active">Body</a></li>
+        <li><a href="#fragment-2" class="active">Log</a></li>
+      </ul>
+
+      <div id="report">
+        <h3>Detail of <%= short_description %> <em>Generated on <%= Time.now %></em></h3>
+        <p><b>Resource</b> <a href="<%= full_url %>"><%= full_url %></a></p>
+        <p><b>Response</b> <span class="r<%= code.first %>"><%= code %></span></p>
+        <p><b>Referrer</b> <%= referrer || "" %></p>
+
+        <table class="output">
+        <tbody>
+          <tr>                
+            <th colspan="2">#&nbsp;&nbsp;Data</th>           
+          </tr>
+          <% if data %>
+            <%= wrap_in_line_number_table_row(data) %>
+          <% else %>
+            <tr>
+              <td colspan="2">No Data</td>
+            </tr>
+          <% end %>
+          </tbody>
+        </table>                            
+
+        <table class="output" id="fragment-1">
+          <tbody>
+          <tr>
+            <th colspan="2">#&nbsp;&nbsp;Body</th>           
+          </tr>
+          <% if body %>
+            <%= wrap_in_line_number_table_row(body) %>
+          <% else %>
+            <tr>
+              <td colspan="2">No Body</td>
+            </tr>
+          <% end %>
+          </tbody>
+        </table>                             
+
+        <table class="output" id="fragment-2">
+          <tbody>
+            <tr>
+              <th colspan="2">#&nbsp;&nbsp;Log</th>           
+            </tr>
+            <% if log %>
+              <%= wrap_in_line_number_table_row(log) {|line| wrap_stack_trace_line(line)} %>
+            <% else %>
+              <tr>
+                <td colspan="2">No Log</td>
+              </tr>
+            <% end %>
+          </tbody>
+        </table>                            
+      </div>
+    </div>
+  </div>
+</body>
+</html>

data/lib/relevance/tarantula/form.rb

@@ -0,0 +1,21 @@
+class Relevance::Tarantula::Form
+  extend Forwardable
+  def_delegators("@tag", :search)
+  
+  def initialize(tag)
+    @tag = tag
+  end
+  
+  def action
+    @tag['action'].downcase
+  end
+  
+  def method
+    (rails_method_hack or @tag['method'] or 'get').downcase
+  end
+  
+  def rails_method_hack
+    (tag = @tag.at('input[@name="_method"]')) && tag["value"]
+  end
+
+end

data/lib/relevance/tarantula/form_submission.rb

@@ -0,0 +1,70 @@
+class Relevance::Tarantula::FormSubmission
+  attr_accessor :method, :action, :data
+  def initialize(form)
+    @method = form.method
+    @action = form.action
+    @data = mutate_selects(form).merge(mutate_text_areas(form)).merge(mutate_inputs(form))
+  end
+  
+  def self.mutate(form)
+    [self.new(form)]
+  end
+  
+  def to_s
+    "#{action} #{method} #{data.inspect}"
+  end
+  
+  # a form's signature is what makes it unique (e.g. action + fields)
+  # used to keep track of which forms we have submitted already
+  def signature
+    [action, data.keys.sort]
+  end
+  
+  def create_random_data_for(form, tag_selector)
+    form.search(tag_selector).inject({}) do |form_args, input|
+      # TODO: test
+      form_args[input['name']] = random_data(input) if input['name']
+      form_args
+    end
+  end
+
+  def mutate_inputs(form)
+    create_random_data_for(form, 'input')
+  end
+
+  def mutate_text_areas(form)
+    create_random_data_for(form, 'textarea')
+  end
+  
+  def mutate_selects(form)
+    form.search('select').inject({}) do |form_args, select|
+      options = select.search('option')
+      option = options.rand
+      form_args[select['name']] = option['value'] 
+      form_args
+    end
+  end
+  
+  def random_data(input)
+    case input['name']
+      when /amount/         : random_int
+      when /_id$/           : random_whole_number
+      when /uploaded_data/  : nil
+      when /^_method$/      : input['value']
+      when nil              : input['value']
+      else                    random_int
+    end
+  end
+  
+  def big_number
+    10000   # arbitrary
+  end
+  
+  def random_int
+    rand(big_number) - (big_number/2)
+  end
+  
+  def random_whole_number
+    rand(big_number)
+  end
+end

data/lib/relevance/tarantula/html_document_handler.rb

@@ -0,0 +1,36 @@
+require 'hpricot'
+
+class Relevance::Tarantula::HtmlDocumentHandler 
+  extend Forwardable
+  def_delegators("@crawler", :queue_link, :queue_form)
+  
+  def initialize(crawler)
+    @crawler = crawler
+  end              
+  # HTML::Document shouts to stderr when it sees ugly HTML
+  # We don't want this -- the InvalidHtmlHandler will deal with it
+  def html_doc_without_stderr_noise(html)  
+    body = nil
+    Recording.stderr do
+      body = Hpricot html
+    end       
+    body
+  end
+  def handle(result)
+    response = result.response
+    url = result.url
+    return unless response.html?
+    body = html_doc_without_stderr_noise(response.body)
+    body.search('a').each do |tag|
+      queue_link(tag, url)
+    end
+    body.search('link').each do |tag|
+      queue_link(tag, url)
+    end
+    body.search('form').each do |form|
+      form['action'] = url unless form['action']
+      queue_form(form, url)
+    end
+    nil
+  end
+end

data/lib/relevance/tarantula/html_report_helper.rb

@@ -0,0 +1,39 @@
+require "erb"
+module Relevance::Tarantula::HtmlReportHelper 
+  include ERB::Util
+  include Relevance::Tarantula
+  def wrap_in_line_number_table_row(text, &blk)
+    x = Builder::XmlMarkup.new
+
+    x.tr do
+      lines = text.split("\n")
+      x.td(:class => "numbers") do
+        lines.size.times do |index|
+          x.span(index+1, :class => "line number")
+        end
+      end
+      x.td(:class => "lines") do
+        lines.each do |line|
+          x.span(line, :class => "line")
+        end
+      end
+    end
+
+    x.target!
+  end 
+                                                                            
+  def textmate_url(file, line_no)
+    "txmt://open?url=file://#{File.expand_path(File.join(rails_root,file))}&line_no=#{line_no}"
+  end
+  
+  def wrap_stack_trace_line(text)
+    if text =~ %r{^\s*(/[^:]+):(\d+):([^:]+)$}
+      file = h($1) # .to_s_xss_protected
+      line_number = $2
+      message = h($3) # .to_s_xss_protected
+      "<a href='#{textmate_url(file, line_number)}'>#{file}:#{line_number}</a>:#{message}" # .mark_as_xss_protected
+    else
+      h(text) # .to_s_xss_protected
+    end
+  end
+end

data/lib/relevance/tarantula/html_reporter.rb

@@ -0,0 +1,105 @@
+class Relevance::Tarantula::HtmlReporter
+  
+  include Relevance::Tarantula
+  attr_accessor :basedir, :results
+  delegate :successes, :failures, :to => :results
+  
+  HtmlResultOverview = Struct.new(:code, :url, :description, :method, :referrer, :file_name)
+  
+  def initialize(basedir)
+    @basedir = basedir    
+    @results = Struct.new(:successes, :failures).new([], [])
+    FileUtils.mkdir_p(@basedir)
+  end
+  
+  def report(result)
+    return if result.nil?
+    
+    create_detail_report(result)
+    
+    collection = result.success ? results.successes : results.failures
+    collection << HtmlResultOverview.new(
+      result.code, result.url, result.description, result.method, result.referrer, result.file_name
+    )
+  end
+
+  def finish_report(test_name)
+    puts "Writing results to #{basedir}"
+    copy_styles  unless styles_exist?
+    create_index unless index_exists?
+    update_index(test_name)
+  end
+  
+  def create_detail_report(result)
+    template = ERB.new(template("detail.html.erb"))
+    output(result.file_name, template.result(result.send(:binding)), result.test_name)
+  end 
+   
+  def copy_styles
+    # not using cp_r because it picks up .svn crap
+    FileUtils.mkdir_p(File.join(basedir, "stylesheets"))
+    Dir.glob("#{tarantula_home}/laf/stylesheets/*.css").each do |file|
+      FileUtils.cp(file, File.join(basedir, "stylesheets")) 
+    end
+    FileUtils.mkdir_p(File.join(basedir, "images"))
+    Dir.glob("#{tarantula_home}/laf/images/*.{jpg,gif,png}").each do |file|
+      FileUtils.cp(file, File.join(basedir, "images")) 
+    end
+    FileUtils.mkdir_p(File.join(basedir, "javascripts"))
+    Dir.glob("#{tarantula_home}/laf/javascripts/*.js").each do |file|
+      FileUtils.cp(file, File.join(basedir, "javascripts")) 
+    end
+  end
+  
+  def create_index
+    template = ERB.new(template("index.html.erb"))
+    output("index.html", template.result(binding))
+  end
+  
+  def update_index(test_name)    
+    File.open(File.join(basedir, "index.html"), "r+") do |file|
+      doc = Hpricot file.read
+      tabs_container = doc.search "#tabs-container ul"
+      results_container = doc.search "#results-container"
+      tabs_container.append tab_html(test_name)
+      results_container.append results_html(test_name)
+      file.rewind
+      file.write doc.to_s
+    end
+  end
+  
+  def index_exists?
+    File.exists?(File.join(basedir, "index.html"))
+  end
+  
+  def styles_exist?
+    File.exists?(File.join(basedir, "stylesheets", "tarantula.css"))
+  end
+
+  def tab_html(test_name)
+    "<li><a href='##{test_name}'><span>#{test_name}</span></a></li>"
+  end
+
+  def results_html(test_name)
+    template = ERB.new(template("test_report.html.erb"))
+    template.result(binding)
+  end
+  
+  def template(name)
+    File.read(File.join(File.dirname(__FILE__), name))
+  end
+  
+  def output(name, body, subdir = '')
+    FileUtils.mkdir_p(File.join(basedir, subdir)) unless subdir.empty?
+    File.open(File.join(basedir, subdir, name), "w") do |file|
+      file.write body
+    end
+  end      
+  
+  # CSS class for HTML status codes
+  def class_for_code(code)
+    "r#{Integer(code)/100}" 
+  end
+  
+  
+end

data/lib/relevance/tarantula/index.html.erb

@@ -0,0 +1,37 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+	"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+<head>
+	<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>                
+  <link rel="stylesheet" href="stylesheets/tarantula.css" type="text/css" media="screen" title="no title" charset="utf-8">   
+	<script type="text/javascript" src="javascripts/jquery-1.2.3.js"></script>
+  <script type="text/javascript" src="javascripts/jquery.tablesorter.js"></script>
+  <script type="text/javascript" src="javascripts/jquery-ui-tabs.js"></script>                                                                                
+  <script type="text/javascript" src="javascripts/tarantula.js"></script>
+
+	<title>Tarantula</title>
+</head>
+
+<body>       
+  <div id="container">
+    <div id="header">
+      <h1>Tarantula by Relevance</h1>
+      <h2>Eight legs, two fangs ... and an attitude</h2>
+      <p>Tarantula is an open source tool for testing Rails web 
+      applications. Tarantula is developed by <a href="http://thinkrelevance.com">Relevance, Inc.</a>
+      and lives at <a href="http://github.com/relevance/tarantula">http://github.com/relevance/tarantula</a>.</p>
+      <hr/>
+    </div>
+    <div id="page">
+      <div id="tabs-container">
+        <ul class="tabs"> </ul>
+    	</div>    
+    
+      <div id="results-container">
+
+      </div>
+    </div>   
+  </div>
+</body>
+</html>

data/lib/relevance/tarantula/invalid_html_handler.rb

@@ -0,0 +1,18 @@
+class Relevance::Tarantula::InvalidHtmlHandler
+  include Relevance::Tarantula
+  def handle(result)
+    response = result.response
+    return unless response.html?
+    begin
+      body = HTML::Document.new(response.body, true)
+    rescue Exception => e
+      error_result = result.dup
+      error_result.success = false
+      error_result.description = "Bad HTML (Scanner)"
+      error_result.data = e.message
+      error_result
+    else
+      nil
+    end
+  end
+end