tarantula 0.0.5

data/laf/stylesheets/ui.tabs.css

@@ -0,0 +1,113 @@
+/* Caution! Ensure accessibility in print and other media types... */
+@media projection, screen { /* Use class for showing/hiding tab content, so that visibility can be better controlled in different media types... */
+    .ui-tabs-hide {
+        display: none;
+    }
+}
+
+/* Hide useless elements in print layouts... */
+@media print {
+    .ui-tabs-nav {
+        display: none;
+    }
+}
+
+/* Skin */
+.ui-tabs-nav, .ui-tabs-panel {
+    font-family: "Trebuchet MS", Trebuchet, Verdana, Helvetica, Arial, sans-serif;
+    font-size: 12px;
+}
+.ui-tabs-nav {
+    list-style: none;
+    margin: 0;
+    padding: 0 0 0 4px;
+}
+.ui-tabs-nav:after { /* clearing without presentational markup, IE gets extra treatment */
+    display: block;
+    clear: both;
+    content: " ";
+}
+.ui-tabs-nav li {
+    list-style: none;
+    float: left;
+    margin: 0 0 0 1px;
+    min-width: 84px; /* be nice to Opera */
+}
+.ui-tabs-nav a, .ui-tabs-nav a span {
+    display: block;
+    padding: 0 10px;
+    background: url(../images/tab.png) no-repeat;
+}
+.ui-tabs-nav a {
+    margin: 1px 0 0; /* position: relative makes opacity fail for disabled tab in IE */
+    padding-left: 0;
+    color: #27537a;
+    font-weight: bold;
+    line-height: 1.2;
+    text-align: center;
+    text-decoration: none;
+    white-space: nowrap; /* required in IE 6 */    
+    outline: 0; /* prevent dotted border in Firefox */
+}
+.ui-tabs-nav .ui-tabs-selected a {
+    position: relative;
+    top: 1px;
+    z-index: 2;
+    margin-top: 0;
+    color: #000;
+}
+.ui-tabs-nav a span {
+    width: 64px; /* IE 6 treats width as min-width */
+    min-width: 64px;
+    height: 18px; /* IE 6 treats height as min-height */
+    min-height: 18px;
+    padding-top: 6px;
+    padding-right: 0;
+}
+*>.ui-tabs-nav a span { /* hide from IE 6 */
+    width: auto;
+    height: auto;
+}
+.ui-tabs-nav .ui-tabs-selected a span {
+    padding-bottom: 1px;
+}
+.ui-tabs-nav .ui-tabs-selected a, .ui-tabs-nav a:hover, .ui-tabs-nav a:focus, .ui-tabs-nav a:active {
+    background-position: 100% -150px;
+}
+.ui-tabs-nav a, .ui-tabs-nav .ui-tabs-disabled a:hover, .ui-tabs-nav .ui-tabs-disabled a:focus, .ui-tabs-nav .ui-tabs-disabled a:active {
+    background-position: 100% -100px;
+}
+.ui-tabs-nav .ui-tabs-selected a span, .ui-tabs-nav a:hover span, .ui-tabs-nav a:focus span, .ui-tabs-nav a:active span {
+    background-position: 0 -50px;
+}
+.ui-tabs-nav a span, .ui-tabs-nav .ui-tabs-disabled a:hover span, .ui-tabs-nav .ui-tabs-disabled a:focus span, .ui-tabs-nav .ui-tabs-disabled a:active span {
+    background-position: 0 0;
+}
+.ui-tabs-nav .ui-tabs-selected a:link, .ui-tabs-nav .ui-tabs-selected a:visited, .ui-tabs-nav .ui-tabs-disabled a:link, .ui-tabs-nav .ui-tabs-disabled a:visited { /* @ Opera, use pseudo classes otherwise it confuses cursor... */
+    cursor: text;
+}
+.ui-tabs-nav a:hover, .ui-tabs-nav a:focus, .ui-tabs-nav a:active,
+.ui-tabs-nav .ui-tabs-unselect a:hover, .ui-tabs-nav .ui-tabs-unselect a:focus, .ui-tabs-nav .ui-tabs-unselect a:active { /* @ Opera, we need to be explicit again here now... */
+    cursor: pointer;
+}
+.ui-tabs-disabled {
+    opacity: .4;
+    filter: alpha(opacity=40);
+}
+.ui-tabs-panel {
+    border-top: 1px solid #97a5b0;
+    padding: 1em 8px;    
+}
+.ui-tabs-loading em {
+    padding: 0 0 0 20px;
+    background: url(loading.gif) no-repeat 0 50%;
+}
+
+/* Additional IE specific bug fixes... */
+* html .ui-tabs-nav { /* auto clear, @ IE 6 & IE 7 Quirks Mode */
+    display: inline-block;
+}
+*:first-child+html .ui-tabs-nav  { /* @ IE 7 Standards Mode - do not group selectors, otherwise IE 6 will ignore complete rule (because of the unknown + combinator)... */
+    display: inline-block;
+}
+

data/lib/relevance/core_extensions/ellipsize.rb

@@ -0,0 +1,34 @@
+module Relevance::CoreExtensions::Nil
+  def ellipsize(cutoff = 20)
+    ""
+  end
+end
+
+module Relevance::CoreExtensions::String
+  def ellipsize(cutoff = 20)
+    if length > cutoff
+      "#{self[0...cutoff]}..."
+    else
+      self
+    end
+  end
+end
+ 
+module Relevance::CoreExtensions::Object
+  def ellipsize(cutoff = 20)
+    inspect.ellipsize(cutoff)
+  end
+end
+
+class Object
+  include Relevance::CoreExtensions::Object
+end
+class String
+  include Relevance::CoreExtensions::String
+end
+class NilClass
+  include Relevance::CoreExtensions::Nil
+end
+  
+  
+

data/lib/relevance/core_extensions/file.rb

@@ -0,0 +1,9 @@
+module Relevance::CoreExtensions::File 
+  def extension(path)
+    extname(path)[1..-1]
+  end
+end
+
+class File
+  extend Relevance::CoreExtensions::File
+end

data/lib/relevance/core_extensions/response.rb

@@ -0,0 +1,9 @@
+# dynamically mixed in to response objects
+module Relevance::CoreExtensions::Response 
+  def html?                                           
+    # some versions of Rails integration tests don't set content type
+    # so we are treating nil as html. A better fix would be welcome here.
+    ((content_type =~ %r{^text/html}) != nil)  || content_type == nil
+  end
+end
+

data/lib/relevance/core_extensions/test_case.rb

@@ -0,0 +1,12 @@
+class Test::Unit::TestCase
+  def tarantula_crawl(integration_test, options = {})
+    url = options[:url] || "/"
+    t = tarantula_crawler(integration_test, options)
+    t.crawl url
+  end
+  
+  def tarantula_crawler(integration_test, options = {})
+    Relevance::Tarantula::RailsIntegrationProxy.rails_integration_test(integration_test, options)
+  end
+end
+

data/lib/relevance/tarantula.rb

@@ -0,0 +1,63 @@
+require 'forwardable'
+
+TARANTULA_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "../.."))
+
+require 'erb'
+gem 'actionpack'
+gem 'activesupport'
+require 'active_support'
+require 'action_controller'
+# bringing in xss-shield requires a bunch of other dependencies
+# still not certain about this, if it ruins your world please let me know
+#xss_shield_path = File.join(TARANTULA_ROOT, %w{vendor xss-shield})
+#$: << File.join(xss_shield_path, "lib")
+#require File.join(xss_shield_path, "init")
+
+require 'htmlentities'
+require 'facets/kernel/meta'
+require 'facets/metaid'
+
+module Relevance; end
+module Relevance; module CoreExtensions; end; end
+module Relevance
+  module Tarantula
+    VERSION = "0.0.5"
+
+    def tarantula_home
+      File.expand_path(File.join(File.dirname(__FILE__), "../.."))
+    end
+    def log(msg)
+      puts msg if verbose
+    end
+    def rails_root
+      ::RAILS_ROOT
+    end
+    def verbose
+      ENV["VERBOSE"]
+    end
+  end
+end
+
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "test_case"))
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "ellipsize"))
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "file"))
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "response"))
+
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_reporter"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_report_helper"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "io_reporter"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "recording"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "response"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "result"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "log_grabber"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "invalid_html_handler"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "transform"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "crawler"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form_submission"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_form_submission"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_handler"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "link"))
+
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "tidy_handler")) if ENV['TIDY_PATH']

data/lib/relevance/tarantula/attack.rb

@@ -0,0 +1,15 @@
+class Relevance::Tarantula::Attack
+  HASHABLE_ATTRS = [:name, :input, :output, :description]
+  attr_accessor *HASHABLE_ATTRS
+  def initialize(hash)
+    hash.each do |k,v|
+      raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
+      self.instance_variable_set("@#{k}", v)
+    end
+  end
+  def ==(other)
+    Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
+  end
+end
+
+

data/lib/relevance/tarantula/attack_form_submission.rb

@@ -0,0 +1,75 @@
+class Relevance::Tarantula::AttackFormSubmission
+  attr_accessor :method, :action, :data, :attack
+
+  class << self
+    def attacks
+      # normalize from hash input to Attack
+      @attacks = @attacks.map do |val|
+        Hash === val ? Relevance::Tarantula::Attack.new(val) : val
+      end
+      @attacks
+    end
+    def attacks=(atts)
+      # normalize from hash input to Attack
+      @attacks = atts.map do |val|
+        Hash === val ? Relevance::Tarantula::Attack.new(val) : val
+      end
+    end
+  end
+  @attacks = []
+
+  def initialize(form, attack = nil)
+    @method = form.method
+    @action = form.action
+    @attack = attack
+    @data = mutate_selects(form).merge(mutate_text_areas(form)).merge(mutate_inputs(form))
+  end
+
+  def self.mutate(form)
+    attacks and attacks.map do |attack|
+      self.new(form, attack)
+    end
+  end
+
+  def to_s
+    "#{action} #{method} #{data.inspect} #{attack.inspect}"
+  end
+
+  # a form's signature is what makes it unique (e.g. action + fields)
+  # used to keep track of which forms we have submitted already
+  def signature
+    [action, data.keys.sort, attack.name]
+  end
+
+  def create_random_data_for(form, tag_selector)
+    form.search(tag_selector).inject({}) do |form_args, input|
+      # TODO: test
+      form_args[input['name']] = random_data(input) if input['name']
+      form_args
+    end
+  end
+
+  def mutate_inputs(form)
+    create_random_data_for(form, 'input')
+  end
+
+  def mutate_text_areas(form)
+    create_random_data_for(form, 'textarea')
+  end
+
+  def mutate_selects(form)
+    form.search('select').inject({}) do |form_args, select|
+      options = select.search('option')
+      option = options.rand
+      form_args[select['name']] = option['value']
+      form_args
+    end
+  end
+
+  def random_data(input)
+    case input['name']
+      when /^_method$/      : input['value']
+      else                    attack.input
+    end
+  end
+end

data/lib/relevance/tarantula/attack_handler.rb

@@ -0,0 +1,37 @@
+require 'hpricot'
+
+class Relevance::Tarantula::AttackHandler 
+  include ERB::Util
+  
+  def attacks
+    Relevance::Tarantula::AttackFormSubmission.attacks.select(&:output)
+  end
+  
+  def handle(result)
+    return unless attacks.size > 0
+    regexp = '(' + attacks.map {|a| Regexp.escape a.output}.join('|') + ')'
+    response = result.response
+    return unless response.html?
+    if n = (response.body =~ /#{regexp}/)
+      error_result = result.dup
+      error_result.success = false
+      error_result.description = "XSS error found, match was: #{h($1)}"
+      error_result.data = <<-STR
+        ########################################################################
+        # Text around unescaped string: #{$1}
+        ########################################################################
+        #{response.body[[0, n - 200].max , 400]}
+        
+        
+        
+        
+        
+        ########################################################################
+        # Attack information:
+        ########################################################################
+        #{attacks.select {|a| a.output == $1}[0].to_yaml}
+      STR
+      error_result
+    end
+  end
+end

data/lib/relevance/tarantula/crawler.rb

@@ -0,0 +1,240 @@
+require File.expand_path(File.join(File.dirname(__FILE__), "rails_integration_proxy"))
+require File.expand_path(File.join(File.dirname(__FILE__), "html_document_handler.rb"))
+
+class Relevance::Tarantula::Crawler
+  extend Forwardable
+  include Relevance::Tarantula
+
+  attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
+                :reporters, :links_to_crawl, :links_queued, :forms_to_crawl,
+                :form_signatures_queued, :max_url_length, :response_code_handler,
+                :times_to_crawl, :fuzzers, :test_name
+  attr_reader   :transform_url_patterns, :referrers, :failures, :successes
+
+  def initialize
+    @max_url_length = 1024
+    @successes = []
+    @failures = []
+    @handlers = [@response_code_handler = Result]
+    @links_queued = Set.new
+    @form_signatures_queued = Set.new
+    @links_to_crawl = []
+    @forms_to_crawl = []
+    @referrers = {}
+    @skip_uri_patterns = [
+      /^javascript/,
+      /^mailto/,
+      /^http/,
+    ]
+    self.transform_url_patterns = [
+      [/#.*$/, '']
+    ]
+    @reporters = [Relevance::Tarantula::IOReporter.new($stderr)]
+    @decoder = HTMLEntities.new
+    @times_to_crawl = 1
+    @fuzzers = [Relevance::Tarantula::FormSubmission]
+  end
+
+  def method_missing(meth, *args)
+    super unless Result::ALLOW_NNN_FOR =~ meth.to_s
+    @response_code_handler.send(meth, *args)
+  end
+
+  def transform_url_patterns=(patterns)
+    @transform_url_patterns = patterns.map do |pattern|
+      Array === pattern ? Relevance::Tarantula::Transform.new(*pattern) : pattern
+    end
+  end
+
+  def crawl(url = "/")
+    orig_links_queued = @links_queued.dup
+    orig_form_signatures_queued = @form_signatures_queued.dup
+    orig_links_to_crawl = @links_to_crawl.dup
+    orig_forms_to_crawl = @forms_to_crawl.dup
+    @times_to_crawl.times do |i|
+      queue_link url
+      do_crawl
+
+      puts "#{(i+1).ordinalize} crawl" if @times_to_crawl > 1
+
+      if i + 1 < @times_to_crawl
+        @links_queued = orig_links_queued
+        @form_signatures_queued = orig_form_signatures_queued
+        @links_to_crawl = orig_links_to_crawl
+        @forms_to_crawl = orig_forms_to_crawl
+        @referrers = {}
+      end
+    end
+  rescue Interrupt
+    $stderr.puts "CTRL-C"
+  ensure
+    report_results
+  end
+
+  def finished?
+    @links_to_crawl.empty? && @forms_to_crawl.empty?
+  end
+
+  def do_crawl
+    while (!finished?)
+      crawl_queued_links
+      crawl_queued_forms
+    end
+  end
+
+  def crawl_queued_links
+    while (link = @links_to_crawl.pop)
+      response = proxy.send(link.method, link.href)
+      log "Response #{response.code} for #{link}"
+      handle_link_results(link, response)
+      blip
+    end
+  end
+
+  def save_result(result)
+    reporters.each do |reporter|
+      reporter.report(result)
+    end
+  end
+
+  def handle_link_results(link, response)
+    handlers.each do |h|
+      begin
+        save_result h.handle(Result.new(:method => link.method,
+                                       :url => link.href,
+                                       :response => response,
+                                       :log => grab_log!,
+                                       :referrer => referrers[link],
+                                       :test_name => test_name).freeze)
+      rescue Exception => e
+        log "error handling #{link} #{e.message}"
+        # TODO: pass to results
+      end
+    end
+  end
+
+  def crawl_form(form)
+    response = proxy.send(form.method, form.action, form.data)
+    log "Response #{response.code} for #{form}"
+    response
+  rescue ActiveRecord::RecordNotFound => e
+    log "Skipping #{form.action}, presumed ok that record is missing"
+    Relevance::Tarantula::Response.new(:code => "404", :body => e.message, :content_type => "text/plain")
+  end
+
+  def crawl_queued_forms
+    while (form = @forms_to_crawl.pop)
+      response = crawl_form(form)
+      handle_form_results(form, response)
+      blip
+    end
+  end
+
+  def grab_log!
+    @log_grabber && @log_grabber.grab!
+  end
+
+  def handle_form_results(form, response)
+    handlers.each do |h|
+      save_result h.handle(Result.new(:method => form.method,
+                                     :url => form.action,
+                                     :response => response,
+                                     :log => grab_log!,
+                                     :referrer => form.action,
+                                     :data => form.data.inspect,
+                                     :test_name => test_name).freeze)
+    end
+  end
+
+  def should_skip_url?(url)
+    return true if url.blank?
+    if @skip_uri_patterns.any? {|pattern| pattern =~ url}
+      log "Skipping #{url}"
+      return true
+    end
+    if url.length > max_url_length
+      log "Skipping long url #{url}"
+      return true
+    end
+  end
+
+  def should_skip_link?(link)
+    should_skip_url?(link.href) || @links_queued.member?(link)
+  end
+
+  def should_skip_form_submission?(fs)
+    should_skip_url?(fs.action) || @form_signatures_queued.member?(fs.signature)
+  end
+
+  def transform_url(url)
+    return unless url
+    url = @decoder.decode(url)
+    @transform_url_patterns.each do |pattern|
+      url = pattern[url]
+    end
+    url
+  end
+
+  def queue_link(dest, referrer = nil)
+    dest = Link.new(dest)
+    dest.href = transform_url(dest.href)
+    return if should_skip_link?(dest)
+    @referrers[dest] = referrer if referrer
+    @links_to_crawl << dest
+    @links_queued << dest
+    dest
+  end
+
+  def queue_form(form, referrer = nil)
+    fuzzers.each do |fuzzer|
+      fuzzer.mutate(Form.new(form)).each do |fs|
+        # fs = fuzzer.new(Form.new(form))
+        fs.action = transform_url(fs.action)
+        return if should_skip_form_submission?(fs)
+        @referrers[fs.action] = referrer if referrer
+        @forms_to_crawl << fs
+        @form_signatures_queued << fs.signature
+      end
+    end
+  end
+
+  def report_dir
+    File.join(rails_root, "tmp", "tarantula")
+  end
+
+  def generate_reports
+    errors = []
+    reporters.each do |reporter|
+      begin
+        reporter.finish_report(test_name)
+      rescue RuntimeError => e
+        errors << e
+      end
+    end
+    unless errors.empty?
+      raise errors.map(&:message).join("\n")
+    end
+  end
+
+  def report_results
+    generate_reports
+  end
+
+  def total_links_count
+    @links_queued.size + @form_signatures_queued.size
+  end
+
+  def links_remaining_count
+    @links_to_crawl.size + @forms_to_crawl.size
+  end
+
+  def links_completed_count
+      total_links_count - links_remaining_count
+  end
+
+  def blip
+    unless verbose
+      print "\r #{links_completed_count} of #{total_links_count} links completed               "
+    end
+  end
+end