codez-tarantula 0.5.0

data/lib/relevance/core_extensions/response.rb

@@ -0,0 +1,14 @@
+# dynamically mixed in to response objects
+module Relevance
+  module CoreExtensions
+    module Response
+      def html?
+        # some versions of Rails integration tests don't set content type
+        # so we are treating nil as html. A better fix would be welcome here.
+        (content_type.respond_to?(:html?) ?
+          content_type.html? : content_type =~ %r{^text/html}) ||
+          content_type.nil?
+      end
+    end
+  end
+end

data/lib/relevance/core_extensions/test_case.rb

@@ -0,0 +1,21 @@
+module Relevance
+  module CoreExtensions
+
+    module TestCaseExtensions
+      def tarantula_crawl(integration_test, options = {})
+        url = options[:url] || "/"
+        t = tarantula_crawler(integration_test, options)
+        t.crawl url
+      end
+
+      def tarantula_crawler(integration_test, options = {})
+        Relevance::Tarantula::RailsIntegrationProxy.rails_integration_test(integration_test, options)
+      end
+    end
+
+  end
+end
+
+if defined? ActionController::IntegrationTest
+  ActionController::IntegrationTest.class_eval { include Relevance::CoreExtensions::TestCaseExtensions }
+end

data/lib/relevance/tarantula.rb

@@ -0,0 +1,55 @@
+require 'forwardable'
+require 'erb'
+require 'active_support'
+require 'action_controller'
+require 'htmlentities'
+
+if RUBY_VERSION < '1.9.1'
+  warn "***************************************************"
+  warn "tarantula will stop supporting ruby 1.8.x in 0.6.0."
+  warn "***************************************************"
+end
+
+module Relevance; end
+module Relevance; module CoreExtensions; end; end
+module Relevance
+  module Tarantula
+    def tarantula_home
+      File.expand_path(File.join(File.dirname(__FILE__), "../.."))
+    end
+    def log(msg)
+      puts msg if verbose
+    end
+    def rails_root
+      ::Rails.root.to_s
+    end
+    def verbose
+      ENV["VERBOSE"]
+    end
+  end
+end
+
+require "relevance/core_extensions/test_case"
+require "relevance/core_extensions/ellipsize"
+require "relevance/core_extensions/file"
+require "relevance/core_extensions/response"
+require "relevance/core_extensions/metaclass"
+
+require "relevance/tarantula/html_reporter"
+require "relevance/tarantula/html_report_helper"
+require "relevance/tarantula/io_reporter"
+require "relevance/tarantula/recording"
+require "relevance/tarantula/response"
+require "relevance/tarantula/result"
+require "relevance/tarantula/log_grabber"
+require "relevance/tarantula/invalid_html_handler"
+require "relevance/tarantula/transform"
+require "relevance/tarantula/crawler"
+require "relevance/tarantula/basic_attack"
+require "relevance/tarantula/form"
+require "relevance/tarantula/form_submission"
+require "relevance/tarantula/attack"
+require "relevance/tarantula/attack_handler"
+require "relevance/tarantula/link"
+
+require "relevance/tarantula/tidy_handler" if ENV['TIDY_PATH']

data/lib/relevance/tarantula/attack.rb

@@ -0,0 +1,22 @@
+module Relevance
+  module Tarantula
+
+    class Attack
+      HASHABLE_ATTRS = [:name, :input, :output, :description]
+      attr_accessor *HASHABLE_ATTRS
+      def initialize(hash)
+        hash.each do |k,v|
+          raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
+          self.instance_variable_set("@#{k}", v)
+        end
+      end
+      def ==(other)
+        Relevance::Tarantula::Attack === other && HASHABLE_ATTRS.all? { |attr| send(attr) == other.send(attr)}
+      end
+      def input(input_field=nil)
+        @input
+      end
+    end
+
+  end
+end

data/lib/relevance/tarantula/attack_handler.rb

@@ -0,0 +1,43 @@
+require 'hpricot'
+
+module Relevance
+  module Tarantula
+
+    class AttackHandler 
+      include ERB::Util
+
+      def attacks
+        Relevance::Tarantula::FormSubmission.attacks.select(&:output)
+      end
+
+      def handle(result)
+        return unless attacks.size > 0
+        regexp = '(' + attacks.map {|a| Regexp.escape a.output}.join('|') + ')'
+        response = result.response
+        return unless response.html?
+        if n = (response.body =~ /#{regexp}/)
+          error_result = result.dup
+          error_result.success = false
+          error_result.description = "XSS error found, match was: #{h($1)}"
+          error_result.data = <<-STR
+        ########################################################################
+        # Text around unescaped string: #{$1}
+        ########################################################################
+          #{response.body[[0, n - 200].max , 400]}
+
+
+
+
+
+        ########################################################################
+        # Attack information:
+        ########################################################################
+          #{attacks.select {|a| a.output == $1}[0].to_yaml}
+          STR
+          error_result
+        end
+      end
+    end
+
+  end
+end

data/lib/relevance/tarantula/basic_attack.rb

@@ -0,0 +1,44 @@
+module Relevance
+  module Tarantula
+
+    class BasicAttack
+      ATTRS = [:name, :output, :description]
+
+      attr_reader *ATTRS
+
+      def initialize
+        @name = "Tarantula Basic Fuzzer"
+        @output = nil
+        @description = "Supplies purely random but simplistically generated form input."
+      end
+
+      def ==(other)
+        Relevance::Tarantula::BasicAttack === other && ATTRS.all? { |attr| send(attr) == other.send(attr)}
+      end
+
+      def input(input_field)
+        case input_field['name']
+        when /amount/         then random_int
+        when /_id$/           then random_whole_number
+        when /uploaded_data/  then nil
+        when nil              then input['value']
+        else
+          random_int
+        end
+      end
+
+      def big_number
+        10000   # arbitrary
+      end
+
+      def random_int
+        rand(big_number) - (big_number/2)
+      end
+
+      def random_whole_number
+        rand(big_number)
+      end
+    end
+
+  end
+end

data/lib/relevance/tarantula/crawler.rb

@@ -0,0 +1,271 @@
+require 'active_record'
+require 'active_record/base'
+require File.expand_path(File.join(File.dirname(__FILE__), "rails_integration_proxy"))
+require File.expand_path(File.join(File.dirname(__FILE__), "html_document_handler.rb"))
+
+module Relevance
+  module Tarantula
+
+    class Crawler
+      extend Forwardable
+      include Relevance::Tarantula
+
+      class CrawlTimeout < RuntimeError; end
+
+      attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
+        :reporters, :crawl_queue, :links_queued,
+        :form_signatures_queued, :max_url_length, :response_code_handler,
+        :times_to_crawl, :fuzzers, :test_name, :crawl_timeout
+      attr_reader   :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
+
+      def initialize
+        @max_url_length = 1024
+        @successes = []
+        @failures = []
+        @handlers = [@response_code_handler = Result]
+        @links_queued = Set.new
+        @form_signatures_queued = Set.new
+        @crawl_queue = []
+        @crawl_start_times, @crawl_end_times = [], []
+        @crawl_timeout = 20.minutes
+        @referrers = {}
+        @skip_uri_patterns = [
+          /^javascript/,
+          /^mailto/,
+          /^http/,
+        ]
+        self.transform_url_patterns = [
+          [/#.*$/, '']
+        ]
+        @reporters = [Relevance::Tarantula::IOReporter.new($stderr)]
+        @decoder = HTMLEntities.new
+        @times_to_crawl = 1
+        @fuzzers = [Relevance::Tarantula::FormSubmission]
+
+        @stdout_tty = $stdout.tty?
+      end
+
+      def method_missing(meth, *args)
+        super unless Result::ALLOW_NNN_FOR =~ meth.to_s
+        @response_code_handler.send(meth, *args)
+      end
+
+      def transform_url_patterns=(patterns)
+        @transform_url_patterns = patterns.map do |pattern|
+          Array === pattern ? Relevance::Tarantula::Transform.new(*pattern) : pattern
+        end
+      end
+
+      def crawl(url = "/")
+        orig_links_queued = @links_queued.dup
+        orig_form_signatures_queued = @form_signatures_queued.dup
+        orig_crawl_queue = @crawl_queue.dup
+        @times_to_crawl.times do |num|
+          queue_link url
+
+          begin
+            do_crawl num
+          rescue CrawlTimeout => e
+            puts
+            puts e.message
+          end
+
+          puts "#{ActiveSupport::Inflector.ordinalize((num+1))} crawl" if @times_to_crawl > 1
+
+          if num + 1 < @times_to_crawl
+            @links_queued = orig_links_queued
+            @form_signatures_queued = orig_form_signatures_queued
+            @crawl_queue = orig_crawl_queue
+            @referrers = {}
+          end
+        end
+      rescue Interrupt
+        $stderr.puts "CTRL-C"
+      ensure
+        report_results
+      end
+
+      def finished?
+        @crawl_queue.empty?
+      end
+
+      def do_crawl(number)
+        while (!finished?)
+          @crawl_start_times << Time.now
+          crawl_the_queue(number)
+          @crawl_end_times << Time.now
+        end
+      end
+
+      def crawl_the_queue(number = 0)
+        while (request = @crawl_queue.shift)
+          request.crawl
+          blip(number)
+        end
+      end
+
+      def save_result(result)
+        reporters.each do |reporter|
+          reporter.report(result)
+        end
+      end
+
+      def handle_link_results(link, result)
+        handlers.each do |h|
+          begin
+            save_result h.handle(result)
+          rescue Exception => e
+            log "error handling #{link} #{e.message}"
+            # TODO: pass to results
+          end
+        end
+      end
+
+      def follow(method, url, data=nil)
+        proxy.send(method, url, data)
+      end
+
+      def submit(method, action, data)
+        proxy.send(method, action, data)
+      end
+
+      def elasped_time_for_pass(num)
+        Time.now - crawl_start_times[num]
+      end
+
+      def grab_log!
+        @log_grabber && @log_grabber.grab!
+      end
+
+      def make_result(options)
+        defaults = {
+          :log       => grab_log!,
+          :test_name => test_name
+        }
+        Result.new(defaults.merge(options)).freeze
+      end
+
+      def handle_form_results(form, response)
+        handlers.each do |h|
+          save_result h.handle(Result.new(:method => form.meth,
+                                          :url => form.action,
+                                          :response => response,
+                                          :log => grab_log!,
+                                          :referrer => form.action,
+                                          :data => form.data.inspect,
+                                          :test_name => test_name).freeze)
+        end
+      end
+
+      def should_skip_url?(url)
+        return true if url.blank?
+        if @skip_uri_patterns.any? {|pattern| pattern =~ url}
+          log "Skipping #{url}"
+          return true
+        end
+        if url.length > max_url_length
+          log "Skipping long url #{url}"
+          return true
+        end
+      end
+
+      def should_skip_link?(link)
+        should_skip_url?(link.href) || @links_queued.member?(link)
+      end
+
+      def should_skip_form_submission?(fs)
+        should_skip_url?(fs.action) || @form_signatures_queued.member?(fs.signature)
+      end
+
+      def transform_url(url)
+        return unless url
+        url = @decoder.decode(url)
+        @transform_url_patterns.each do |pattern|
+          url = pattern[url]
+        end
+        url
+      end
+
+      def queue_link(dest, referrer = nil)
+        dest = Link.new(dest, self, referrer)
+        return if should_skip_link?(dest)
+        append_to_queue(dest)
+        @links_queued << dest
+        dest
+      end
+
+      def queue_form(form, referrer = nil)
+        fuzzers.each do |fuzzer|
+          fuzzer.mutate(Form.new(form, self, referrer)).each do |fs|
+            # fs = fuzzer.new(Form.new(form, self, referrer))
+            fs.action = transform_url(fs.action)
+            return if should_skip_form_submission?(fs)
+            @referrers[fs.action] = referrer if referrer
+            append_to_queue(fs)
+            @form_signatures_queued << fs.signature
+          end
+        end
+      end
+
+      # append delete requests to the end of the queue, all others just before the first delete request
+      def append_to_queue(request)
+        if request.meth != 'delete' && index = @crawl_queue.index {|r| r.meth == 'delete' }
+          @crawl_queue.insert(index, request)
+        elsif request.meth == 'delete' && parent_index = @crawl_queue.index {|r| r.meth == 'delete' && request.url.start_with?(r.url) }
+          @crawl_queue.insert(parent_index, request)
+        else
+          @crawl_queue << request
+        end
+      end
+
+      def report_dir
+        File.join(rails_root, "tmp", "tarantula")
+      end
+
+      def generate_reports
+        errors = []
+        reporters.each do |reporter|
+          begin
+            reporter.finish_report(test_name)
+          rescue RuntimeError => e
+            errors << e
+          end
+        end
+        unless errors.empty?
+          raise errors.map(&:message).join("\n")
+        end
+      end
+
+      def report_results
+        puts "Crawled #{total_links_count} links and forms."
+        generate_reports
+      end
+
+      def total_links_count
+        @links_queued.size + @form_signatures_queued.size
+      end
+
+      def links_remaining_count
+        @crawl_queue.size
+      end
+
+      def links_completed_count
+        total_links_count - links_remaining_count
+      end
+
+      def blip(number = 0)
+        unless verbose
+          print "\r #{links_completed_count} of #{total_links_count} links completed               " if @stdout_tty
+          timeout_if_too_long(number)
+        end
+      end
+
+      def timeout_if_too_long(number = 0)
+        if elasped_time_for_pass(number) > crawl_timeout
+          raise CrawlTimeout, "Exceeded crawl timeout of #{crawl_timeout} seconds - skipping to the next crawl..."
+        end
+      end
+    end
+
+  end
+end