tarantula-rails3 0.3.3

data/lib/relevance/core_extensions/response.rb

@@ -0,0 +1,9 @@
+# dynamically mixed in to response objects
+module Relevance::CoreExtensions::Response 
+  def html?                                           
+    # some versions of Rails integration tests don't set content type
+    # so we are treating nil as html. A better fix would be welcome here.
+    ((content_type =~ %r{^text/html}) != nil)  || content_type == nil
+  end
+end
+

data/lib/relevance/core_extensions/string_chars_fix.rb

@@ -0,0 +1,11 @@
+if RUBY_VERSION == "1.8.7" # fix interaction between Ruby 187 and Rails 202, so we can at least run the test suite on that combination
+  unless '1.9'.respond_to?(:force_encoding)
+    String.class_eval do
+      begin
+        remove_method :chars
+      rescue NameError
+        # OK
+      end
+    end
+  end
+end

data/lib/relevance/core_extensions/test_case.rb

@@ -0,0 +1,19 @@
+require 'action_dispatch/testing/integration'
+
+module Relevance::CoreExtensions::TestCaseExtensions
+
+  def tarantula_crawl(integration_test, options = {})
+    url = options[:url] || "/"
+    t = tarantula_crawler(integration_test, options)
+    t.crawl url
+  end
+  
+  def tarantula_crawler(integration_test, options = {})
+    Relevance::Tarantula::RailsIntegrationProxy.rails_integration_test(integration_test, options)
+  end
+  
+end
+
+if defined? ActionController::IntegrationTest
+  ActionController::IntegrationTest.class_eval { include Relevance::CoreExtensions::TestCaseExtensions }
+end

data/lib/relevance/tarantula.rb

@@ -0,0 +1,58 @@
+TARANTULA_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "../.."))
+
+require 'forwardable'
+require 'erb'
+require 'active_support'
+require 'action_controller'
+# bringing in xss-shield requires a bunch of other dependencies
+# still not certain about this, if it ruins your world please let me know
+#xss_shield_path = File.join(TARANTULA_ROOT, %w{vendor xss-shield})
+#$: << File.join(xss_shield_path, "lib")
+#require File.join(xss_shield_path, "init")
+
+require 'htmlentities'
+
+module Relevance; end
+module Relevance; module CoreExtensions; end; end
+module Relevance
+  module Tarantula
+    def tarantula_home
+      File.expand_path(File.join(File.dirname(__FILE__), "../.."))
+    end
+    def log(msg)
+      puts msg if verbose
+    end
+    def rails_root
+      ::Rails.root.to_s
+    end
+    def verbose
+      ENV["VERBOSE"]
+    end
+  end
+end
+
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "test_case"))
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "ellipsize"))
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "file"))
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "response"))
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "metaclass"))
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "string_chars_fix"))
+
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_reporter"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_report_helper"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "io_reporter"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "recording"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "response"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "result"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "log_grabber"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "invalid_html_handler"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "transform"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "crawler"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "basic_attack"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "form_submission"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "attack_handler"))
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "link"))
+
+require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "tidy_handler")) if ENV['TIDY_PATH']

data/lib/relevance/tarantula/attack.rb

@@ -0,0 +1,18 @@
+class Relevance::Tarantula::Attack
+  HASHABLE_ATTRS = [:name, :input, :output, :description]
+  attr_accessor *HASHABLE_ATTRS
+  def initialize(hash)
+    hash.each do |k,v|
+      raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
+      self.instance_variable_set("@#{k}", v)
+    end
+  end
+  def ==(other)
+    Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
+  end
+  def input(input_field=nil)
+    @input
+  end
+end
+
+

data/lib/relevance/tarantula/attack_handler.rb

@@ -0,0 +1,37 @@
+require 'hpricot'
+
+class Relevance::Tarantula::AttackHandler 
+  include ERB::Util
+  
+  def attacks
+    Relevance::Tarantula::FormSubmission.attacks.select(&:output)
+  end
+  
+  def handle(result)
+    return unless attacks.size > 0
+    regexp = '(' + attacks.map {|a| Regexp.escape a.output}.join('|') + ')'
+    response = result.response
+    return unless response.html?
+    if n = (response.body =~ /#{regexp}/)
+      error_result = result.dup
+      error_result.success = false
+      error_result.description = "XSS error found, match was: #{h($1)}"
+      error_result.data = <<-STR
+        ########################################################################
+        # Text around unescaped string: #{$1}
+        ########################################################################
+        #{response.body[[0, n - 200].max , 400]}
+        
+        
+        
+        
+        
+        ########################################################################
+        # Attack information:
+        ########################################################################
+        #{attacks.select {|a| a.output == $1}[0].to_yaml}
+      STR
+      error_result
+    end
+  end
+end

data/lib/relevance/tarantula/basic_attack.rb

@@ -0,0 +1,40 @@
+class Relevance::Tarantula::BasicAttack
+  ATTRS = [:name, :output, :description]
+
+  attr_reader *ATTRS
+
+  def initialize
+    @name = "Tarantula Basic Fuzzer"
+    @output = nil
+    @description = "Supplies purely random but simplistically generated form input."
+  end
+
+  def ==(other)
+    Relevance::Tarantula::BasicAttack === other && ATTRS.all? { |attr| send(attr) == other.send(attr)}
+  end
+
+  def input(input_field)
+    case input_field['name']
+      when /amount/         then random_int
+      when /_id$/           then random_whole_number
+      when /uploaded_data/  then nil
+      when nil              then input['value']
+      else
+        random_int
+    end
+  end
+
+  def big_number
+    10000   # arbitrary
+  end
+  
+  def random_int
+    rand(big_number) - (big_number/2)
+  end
+  
+  def random_whole_number
+    rand(big_number)
+  end
+end
+
+

data/lib/relevance/tarantula/crawler.rb

@@ -0,0 +1,254 @@
+require 'active_record'
+require 'active_record/base'
+require File.expand_path(File.join(File.dirname(__FILE__), "rails_integration_proxy"))
+require File.expand_path(File.join(File.dirname(__FILE__), "html_document_handler.rb"))
+
+class Relevance::Tarantula::Crawler
+  extend Forwardable
+  include Relevance::Tarantula
+
+  class CrawlTimeout < RuntimeError; end
+
+  attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
+                :reporters, :crawl_queue, :links_queued,
+                :form_signatures_queued, :max_url_length, :response_code_handler,
+                :times_to_crawl, :fuzzers, :test_name, :crawl_timeout
+  attr_reader   :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
+
+  def initialize
+    @max_url_length = 1024
+    @successes = []
+    @failures = []
+    @handlers = [@response_code_handler = Result]
+    @links_queued = Set.new
+    @form_signatures_queued = Set.new
+    @crawl_queue = []
+    @crawl_start_times, @crawl_end_times = [], []
+    @crawl_timeout = 20.minutes
+    @referrers = {}
+    @skip_uri_patterns = [
+      /^javascript/,
+      /^mailto/,
+      /^http/,
+    ]
+    self.transform_url_patterns = [
+      [/#.*$/, '']
+    ]
+    @reporters = [Relevance::Tarantula::IOReporter.new($stderr)]
+    @decoder = HTMLEntities.new
+    @times_to_crawl = 1
+    @fuzzers = [Relevance::Tarantula::FormSubmission]
+    
+    @stdout_tty = $stdout.tty?
+  end
+
+  def method_missing(meth, *args)
+    super unless Result::ALLOW_NNN_FOR =~ meth.to_s
+    @response_code_handler.send(meth, *args)
+  end
+
+  def transform_url_patterns=(patterns)
+    @transform_url_patterns = patterns.map do |pattern|
+      Array === pattern ? Relevance::Tarantula::Transform.new(*pattern) : pattern
+    end
+  end
+
+  def crawl(url = "/")
+    orig_links_queued = @links_queued.dup
+    orig_form_signatures_queued = @form_signatures_queued.dup
+    orig_crawl_queue = @crawl_queue.dup
+    @times_to_crawl.times do |num|
+      queue_link url
+      
+      begin 
+        do_crawl num
+      rescue CrawlTimeout => e
+        puts
+        puts e.message
+      end
+      
+      puts "#{(num+1).ordinalize} crawl" if @times_to_crawl > 1
+
+      if num + 1 < @times_to_crawl
+        @links_queued = orig_links_queued
+        @form_signatures_queued = orig_form_signatures_queued
+        @crawl_queue = orig_crawl_queue
+        @referrers = {}
+      end
+    end
+  rescue Interrupt
+    $stderr.puts "CTRL-C"
+  ensure
+    report_results
+  end
+
+  def finished?
+    @crawl_queue.empty?
+  end
+
+  def do_crawl(number)
+    while (!finished?)
+      @crawl_start_times << Time.now
+      crawl_the_queue(number)
+      @crawl_end_times << Time.now
+    end
+  end
+
+  def crawl_the_queue(number = 0)
+    while (request = @crawl_queue.pop)
+      request.crawl
+      blip(number)
+    end
+  end
+
+  def save_result(result)
+    reporters.each do |reporter|
+      reporter.report(result)
+    end
+  end
+
+  def handle_link_results(link, result)
+    handlers.each do |h|
+      begin
+        save_result h.handle(result)
+      rescue Exception => e
+        log "error handling #{link} #{e.message}"
+        # TODO: pass to results
+      end
+    end
+  end
+
+  def follow(method, url, data=nil)
+    proxy.send(method, url, data)
+  end
+  
+  def submit(method, action, data)
+    proxy.send(method, action, data)
+  end
+
+  def elasped_time_for_pass(num)
+    Time.now - crawl_start_times[num]
+  end
+
+  def grab_log!
+    @log_grabber && @log_grabber.grab!
+  end
+  
+  def make_result(options)
+    defaults = {
+      :log       => grab_log!,
+      :test_name => test_name      
+    }
+    Result.new(defaults.merge(options)).freeze
+  end
+
+  def handle_form_results(form, response)
+    handlers.each do |h|
+      save_result h.handle(Result.new(:method => form.method,
+                                     :url => form.action,
+                                     :response => response,
+                                     :log => grab_log!,
+                                     :referrer => form.action,
+                                     :data => form.data.inspect,
+                                     :test_name => test_name).freeze)
+    end
+  end
+
+  def should_skip_url?(url)
+    return true if url.blank?
+    if @skip_uri_patterns.any? {|pattern| pattern =~ url}
+      log "Skipping #{url}"
+      return true
+    end
+    if url.length > max_url_length
+      log "Skipping long url #{url}"
+      return true
+    end
+  end
+
+  def should_skip_link?(link)
+    should_skip_url?(link.href) || @links_queued.member?(link)
+  end
+
+  def should_skip_form_submission?(fs)
+    should_skip_url?(fs.action) || @form_signatures_queued.member?(fs.signature)
+  end
+
+  def transform_url(url)
+    return unless url
+    url = @decoder.decode(url)
+    @transform_url_patterns.each do |pattern|
+      url = pattern[url]
+    end
+    url
+  end
+
+  def queue_link(dest, referrer = nil)
+    dest = Link.new(dest, self, referrer)
+    return if should_skip_link?(dest)
+    @crawl_queue << dest
+    @links_queued << dest
+    dest
+  end
+
+  def queue_form(form, referrer = nil)
+    fuzzers.each do |fuzzer|
+      fuzzer.mutate(Form.new(form, self, referrer)).each do |fs|
+        # fs = fuzzer.new(Form.new(form, self, referrer))
+        fs.action = transform_url(fs.action)
+        return if should_skip_form_submission?(fs)
+        @referrers[fs.action] = referrer if referrer
+        @crawl_queue << fs
+        @form_signatures_queued << fs.signature
+      end
+    end
+  end
+
+  def report_dir
+    File.join(rails_root, "tmp", "tarantula")
+  end
+
+  def generate_reports
+    errors = []
+    reporters.each do |reporter|
+      begin
+        reporter.finish_report(test_name)
+      rescue RuntimeError => e
+        errors << e
+      end
+    end
+    unless errors.empty?
+      raise errors.map(&:message).join("\n")
+    end
+  end
+
+  def report_results
+    puts "Crawled #{total_links_count} links and forms."
+    generate_reports
+  end
+
+  def total_links_count
+    @links_queued.size + @form_signatures_queued.size
+  end
+
+  def links_remaining_count
+    @crawl_queue.size
+  end
+
+  def links_completed_count
+      total_links_count - links_remaining_count
+  end
+
+  def blip(number = 0)
+    unless verbose
+      print "\r #{links_completed_count} of #{total_links_count} links completed               " if @stdout_tty
+      timeout_if_too_long(number)
+    end
+  end
+  
+  def timeout_if_too_long(number = 0)
+    if elasped_time_for_pass(number) > crawl_timeout
+      raise CrawlTimeout, "Exceeded crawl timeout of #{crawl_timeout} seconds - skipping to the next crawl..."
+    end
+  end
+end