panmind-tarantula 0.3.3

data/lib/relevance/tarantula/link.rb

@@ -0,0 +1,76 @@
+class Relevance::Tarantula::Link
+  include Relevance::Tarantula
+  
+  class << self
+    include ActionView::Helpers::UrlHelper
+    # method_javascript_function needs this method
+    def protect_against_forgery?
+      false
+    end
+  end
+  
+  METHOD_REGEXPS = {}
+  [:put, :delete, :post].each do |m|
+    # remove submit from the end so we'll match with or without forgery protection
+    s = method_javascript_function(m).gsub( /f.submit();/, "" )
+    # don't just match this.href in case a different url was passed originally
+    s = Regexp.escape(s).gsub( /this.href/, ".*" )
+    METHOD_REGEXPS[m] = /#{s}/
+  end
+  
+  attr_accessor :href, :crawler, :referrer
+  
+  def initialize(link, crawler, referrer)
+    @crawler, @referrer = crawler, referrer
+    
+    if String === link || link.nil?
+      @href = transform_url(link)
+      @method = :get
+    else # should be a tag
+      @href = link['href'] ? transform_url(link['href'].downcase) : nil
+      @tag = link
+    end
+  end
+  
+  def crawl
+    response = crawler.follow(method, href)
+    log "Response #{response.code} for #{self}"
+    crawler.handle_link_results(self, make_result(response))
+  end
+  
+  def make_result(response)
+    crawler.make_result(:method    => method,
+                        :url       => href,
+                        :response  => response,
+                        :referrer  => referrer)
+  end
+  
+  def method
+    @method ||= begin
+      (@tag &&
+       [:put, :delete, :post].detect do |m| # post should be last since it's least specific
+         @tag['onclick'] =~ METHOD_REGEXPS[m]
+       end) ||
+      :get
+    end
+  end
+  
+  def transform_url(link)
+    crawler.transform_url(link)
+  end
+  
+  def ==(obj)
+    obj.respond_to?(:href) && obj.respond_to?(:method) &&
+      self.href.to_s == obj.href.to_s && self.method.to_s == obj.method.to_s
+  end
+  alias :eql? :==
+  
+  def hash
+    to_s.hash
+  end
+  
+  def to_s
+    "<Relevance::Tarantula::Link href=#{href}, method=#{method}>"
+  end
+  
+end

data/lib/relevance/tarantula/log_grabber.rb

@@ -0,0 +1,16 @@
+class Relevance::Tarantula::LogGrabber
+  attr_accessor :path
+  def initialize(path)
+    @path = path
+  end
+  
+  def clear!
+    File.open(@path, "w")
+  end
+  
+  def grab!
+    File.read(@path)
+  ensure
+    clear!
+  end
+end

data/lib/relevance/tarantula/rails_integration_proxy.rb

@@ -0,0 +1,87 @@
+require 'test/unit'
+
+class Relevance::Tarantula::RailsIntegrationProxy
+  include Relevance::Tarantula
+  extend Relevance::Tarantula
+  extend Forwardable
+  attr_accessor :integration_test
+
+  def self.rails_integration_test(integration_test, options = {})
+    t = Crawler.new
+    t.max_url_length = options[:max_url_length] if options[:max_url_length] 
+    t.proxy = RailsIntegrationProxy.new(integration_test)    
+    t.handlers << HtmlDocumentHandler.new(t)
+    t.handlers << InvalidHtmlHandler.new
+    t.log_grabber = Relevance::Tarantula::LogGrabber.new(File.join(rails_root, "log/test.log"))
+    t.skip_uri_patterns << /logout$/
+    t.skip_uri_patterns.delete /^http/
+    t.skip_uri_patterns.push /^https?:\/\/(?!#{integration_test.host})/
+
+    t.transform_url_patterns += [
+      [/\?\d+$/, ''],                               # strip trailing numbers for assets
+      [/^https?:\/\/#{integration_test.host}/, '']  # strip full path down to relative
+    ]
+    t.test_name = t.proxy.integration_test.method_name
+    t.reporters << Relevance::Tarantula::HtmlReporter.new(t.report_dir)
+    t
+  end
+
+  def initialize(integration_test)
+    @integration_test = integration_test
+    @integration_test.meta.attr_accessor :response
+  end
+  
+  [:get, :post, :put, :delete].each do |verb|
+    define_method(verb) do |url, *args|
+      integration_test.send(verb, url, *args)
+      process integration_test.response, verb, url, *args
+    end
+  end
+
+  def xhr(verb, url, *args)
+    integration_test.send(:xhr, verb, url, *args)
+    process integration_test.response, verb, url, *args
+  end
+
+  def process(response, verb, url, *args)
+    if verb == :get && response.code == '302'
+      location = response.headers['Location']
+
+      if location =~ /#{url}$/ # Schema change
+        integration_test.https!(!!(location =~ /^https/))
+        response = send(verb, url, *args)
+      elsif location =~ /#!?/  # AJAX load
+        response = send(:xhr, verb, url, *args)
+      end
+
+    elsif response.code == '404'
+      if File.exist?(static_content_path(url))
+        case ext = File.extension(url)
+        when /html|te?xt|css|js|jpe?g|gif|psd|png|eps|pdf|ico/
+          response.body = static_content_file(url)
+          response.headers["type"] = "text/#{ext}"  # readable as response.content_type
+          response.meta.attr_accessor :code
+          response.code = "200"
+        else
+          log "Skipping unknown type #{url}"
+        end
+      end
+    end
+
+    # don't count on metaclass taking block, e.g.
+    # http://relevancellc.com/2008/2/12/how-should-metaclass-work
+    response.metaclass.class_eval do
+      include Relevance::CoreExtensions::Response
+    end
+
+    return response
+  end
+  
+  def static_content_file(url)
+    File.read(static_content_path(url))
+  end
+  
+  def static_content_path(url)
+    File.expand_path(File.join(rails_root, "public", url))
+  end
+end

data/lib/relevance/tarantula/recording.rb

@@ -0,0 +1,12 @@
+module Recording
+  def self.stderr
+    $stderr = recorder = StringIO.new
+    begin
+      yield
+    ensure
+      $stderr = STDERR
+    end
+    recorder.rewind
+    recorder.read
+  end
+end

data/lib/relevance/tarantula/response.rb

@@ -0,0 +1,13 @@
+# Used to create a stub response when we didn't get back a real response
+class Relevance::Tarantula::Response
+  HASHABLE_ATTRS = [:code, :body, :content_type]
+  attr_accessor *HASHABLE_ATTRS
+
+  def initialize(hash)
+    hash.each do |k,v|
+      raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
+      self.instance_variable_set("@#{k}", v)
+    end
+  end
+  
+end

data/lib/relevance/tarantula/result.rb

@@ -0,0 +1,77 @@
+class Relevance::Tarantula::Result
+  HASHABLE_ATTRS = [:success, :method, :url, :response, :referrer, :data, :description, :log, :test_name]
+  DEFAULT_LOCALHOST = "http://localhost:3000"
+  attr_accessor *HASHABLE_ATTRS
+  include Relevance::Tarantula
+  include Relevance::Tarantula::HtmlReportHelper
+
+  def initialize(hash)
+    hash.each do |k,v|
+      raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
+      self.instance_variable_set("@#{k}", v)
+    end
+  end
+
+  def short_description
+    [method,url].join(" ")
+  end
+
+  def sequence_number
+    @sequence_number ||= (self.class.next_number += 1)
+  end
+
+  def file_name
+    "#{sequence_number}.html"
+  end
+
+  def code
+    response && response.code
+  end
+
+  def body
+    response && response.body
+  end
+
+  def full_url
+    "#{DEFAULT_LOCALHOST}#{url}"
+  end
+
+  ALLOW_NNN_FOR = /^allow_(\d\d\d)_for$/
+
+  class << self
+    attr_accessor :next_number
+    
+    def handle(result)
+      retval = result.dup
+      retval.success = successful?(result.response) || can_skip_error?(result)
+      retval.description = "Bad HTTP Response" unless retval.success
+      retval
+    end
+
+    def success_codes 
+      %w{200 201 302 401}
+    end
+    
+    # allow_errors_for is a hash 
+    #  k=error code,
+    #  v=array of matchers for urls that can skip said error
+    attr_accessor :allow_errors_for
+    def can_skip_error?(result)
+      coll = allow_errors_for[result.code]
+      return false unless coll
+      coll.any? {|item| item === result.url}
+    end
+
+    def successful?(response)
+      success_codes.member?(response.code)
+    end
+
+    def method_missing(meth, *args)  
+      super unless ALLOW_NNN_FOR =~ meth.to_s
+      (allow_errors_for[$1] ||= []).push(*args)
+    end
+  end
+
+  self.allow_errors_for = {}
+  self.next_number = 0
+end

data/lib/relevance/tarantula/test_report.html.erb

@@ -0,0 +1,32 @@
+<div id="<%= test_name %>">
+  <% %w{failures successes}.each do |result_type| %>
+    <table class="list tablesorter" cellspacing="0">
+      <caption><%= send(result_type).size %> <%= result_type.capitalize %></caption>
+      <thead>
+        <tr>
+          <th class="sort asc"><span>URL</span><span class="sort">&nbsp;</span></th>
+          <th><span>Action</span><span class="sort">&nbsp;</span></th>
+          <th><span>Response</span><span class="sort">&nbsp;</span></th>
+          <th class="left"><span>Description</span><span class="sort">&nbsp;</span></th>
+          <th><span>Referrer</span><span class="sort">&nbsp;</span></th>
+        </tr>
+      </thead>
+      <tfoot>
+         <tr><td colspan="5">&nbsp;</td></tr>
+      </tfoot>
+      
+      <tbody>
+        <% send(result_type).sort{|x,y| y.code.to_s <=> x.code.to_s}.each_with_index do |result,i| %>
+          <tr class="<%= (i%2 == 0) ? 'even' : 'odd' %>">
+            <td class="left"><a href="<%= "#{test_name}/#{result.file_name}" %>"><%= result.url.ellipsize(50) %></a></td>
+            <td class="method"><%= result.method.to_s.upcase %></td> <!-- TODO Clean up demeter violation -->
+            <td><span class="<%= class_for_code(result.code) %>"><%= result.code %></span></td>
+            <td class="left"><%= result.description %></td>
+            <td class="left"><%= result.referrer.ellipsize(30) %></td>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>       
+    <br/>
+  <% end %>
+</div>

data/lib/relevance/tarantula/tidy_handler.rb

@@ -0,0 +1,32 @@
+require 'rubygems'
+begin
+  gem 'tidy'
+  require 'tidy'
+rescue Gem::LoadError
+  puts "Tidy gem not available -- 'gem install tidy' to get it."
+end
+
+if defined? Tidy
+  Tidy.path = ENV['TIDY_PATH'] if ENV['TIDY_PATH']
+
+  class Relevance::Tarantula::TidyHandler 
+    include Relevance::Tarantula
+    def initialize(options = {})
+      @options = {:show_warnings=>true}.merge(options)
+    end
+    def handle(result)
+      response = result.response
+      return unless response.html?
+      tidy = Tidy.open(@options) do |tidy|
+        xml = tidy.clean(response.body)
+        tidy
+      end
+      unless tidy.errors.blank?
+        error_result = result.dup
+        error_result.description = "Bad HTML (Tidy)"
+        error_result.data = tidy.errors.inspect
+        error_result
+      end
+    end
+  end
+end

data/lib/relevance/tarantula/transform.rb

@@ -0,0 +1,17 @@
+class Relevance::Tarantula::Transform 
+  attr_accessor :from, :to
+  def initialize(from, to)
+    @from = from
+    @to = to
+  end
+  def [](string)
+    case to
+    when Proc
+      string.gsub(from, &to)
+    else
+      string.gsub(from, to)
+    end
+  end
+end
+
+

data/tasks/tarantula_tasks.rake

@@ -0,0 +1,42 @@
+require 'rake'
+
+namespace :tarantula do
+
+  desc 'Run tarantula tests.'
+  task :test do
+    rm_rf "tmp/tarantula"
+    Rake::TestTask.new(:tarantula_test) do |t|
+      t.libs << 'test'
+      t.pattern = 'test/tarantula/**/*_test.rb'
+      t.verbose = true
+    end
+    
+    Rake::Task[:tarantula_test].invoke
+  end
+
+  desc 'Run tarantula tests and open results in your browser.'
+  task :report do
+    begin
+      Rake::Task['tarantula:test'].invoke
+    rescue RuntimeError => e
+      puts e.message
+    end    
+    
+    Dir.glob("tmp/tarantula/**/index.html") do |file|
+      if PLATFORM['darwin']
+        system("open #{file}")
+      elsif PLATFORM[/linux/]
+        system("firefox #{file}")
+      else
+        puts "You can view tarantula results at #{file}"
+      end
+    end
+  end
+
+  desc 'Generate a default tarantula test'
+  task :setup do
+    mkdir_p "test/tarantula"
+    template_path = File.expand_path(File.join(File.dirname(__FILE__), "..", "template", "tarantula_test.rb"))
+    cp template_path, "test/tarantula/"
+  end
+end

data/template/tarantula_test.rb

@@ -0,0 +1,22 @@
+require "#{File.dirname(__FILE__)}/../test_helper"
+require "relevance/tarantula"
+
+class TarantulaTest < ActionController::IntegrationTest
+  # Load enough test data to ensure that there's a link to every page in your
+  # application. Doing so allows Tarantula to follow those links and crawl 
+  # every page.  For many applications, you can load a decent data set by
+  # loading all fixtures.
+  fixtures :all
+
+  def test_tarantula
+    # If your application requires users to log in before accessing certain 
+    # pages, uncomment the lines below and update them to allow this test to
+    # log in to your application.  Doing so allows Tarantula to crawl the 
+    # pages that are only accessible to logged-in users.
+    # 
+    #   post '/session', :login => 'quentin', :password => 'monkey'
+    #   follow_redirect!
+    
+    tarantula_crawl(self)
+  end
+end

metadata

@@ -0,0 +1,213 @@
+--- !ruby/object:Gem::Specification 
+name: panmind-tarantula
+version: !ruby/object:Gem::Version 
+  hash: 21
+  prerelease: false
+  segments: 
+  - 0
+  - 3
+  - 3
+  version: 0.3.3
+platform: ruby
+authors: 
+- Relevance, Inc.
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-10-18 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: htmlentities
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 55
+        segments: 
+        - 4
+        - 2
+        - 0
+        version: 4.2.0
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 61
+        segments: 
+        - 0
+        - 8
+        - 1
+        version: 0.8.1
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: micronaut
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: log_buddy
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+description: A big hairy fuzzy spider that crawls your site, wreaking havoc
+email: opensource@thinkrelevance.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- CHANGELOG
+- README.rdoc
+- Rakefile
+- VERSION.yml
+- examples/example_helper.rb
+- examples/relevance/core_extensions/ellipsize_example.rb
+- examples/relevance/core_extensions/file_example.rb
+- examples/relevance/core_extensions/response_example.rb
+- examples/relevance/core_extensions/test_case_example.rb
+- examples/relevance/tarantula/attack_handler_example.rb
+- examples/relevance/tarantula/basic_attack_example.rb
+- examples/relevance/tarantula/crawler_example.rb
+- examples/relevance/tarantula/form_example.rb
+- examples/relevance/tarantula/form_submission_example.rb
+- examples/relevance/tarantula/html_document_handler_example.rb
+- examples/relevance/tarantula/html_report_helper_example.rb
+- examples/relevance/tarantula/html_reporter_example.rb
+- examples/relevance/tarantula/invalid_html_handler_example.rb
+- examples/relevance/tarantula/io_reporter_example.rb
+- examples/relevance/tarantula/link_example.rb
+- examples/relevance/tarantula/log_grabber_example.rb
+- examples/relevance/tarantula/rails_integration_proxy_example.rb
+- examples/relevance/tarantula/result_example.rb
+- examples/relevance/tarantula/tidy_handler_example.rb
+- examples/relevance/tarantula/transform_example.rb
+- examples/relevance/tarantula_example.rb
+- laf/images/header_bg.jpg
+- laf/images/logo.png
+- laf/images/tagline.png
+- laf/javascripts/jquery-1.2.3.js
+- laf/javascripts/jquery-ui-tabs.js
+- laf/javascripts/jquery.tablesorter.js
+- laf/javascripts/tarantula.js
+- laf/stylesheets/tarantula.css
+- lib/relevance/core_extensions/ellipsize.rb
+- lib/relevance/core_extensions/file.rb
+- lib/relevance/core_extensions/metaclass.rb
+- lib/relevance/core_extensions/response.rb
+- lib/relevance/core_extensions/string_chars_fix.rb
+- lib/relevance/core_extensions/test_case.rb
+- lib/relevance/tarantula.rb
+- lib/relevance/tarantula/attack.rb
+- lib/relevance/tarantula/attack_handler.rb
+- lib/relevance/tarantula/basic_attack.rb
+- lib/relevance/tarantula/crawler.rb
+- lib/relevance/tarantula/detail.html.erb
+- lib/relevance/tarantula/form.rb
+- lib/relevance/tarantula/form_submission.rb
+- lib/relevance/tarantula/html_document_handler.rb
+- lib/relevance/tarantula/html_report_helper.rb
+- lib/relevance/tarantula/html_reporter.rb
+- lib/relevance/tarantula/index.html.erb
+- lib/relevance/tarantula/invalid_html_handler.rb
+- lib/relevance/tarantula/io_reporter.rb
+- lib/relevance/tarantula/link.rb
+- lib/relevance/tarantula/log_grabber.rb
+- lib/relevance/tarantula/rails_integration_proxy.rb
+- lib/relevance/tarantula/recording.rb
+- lib/relevance/tarantula/response.rb
+- lib/relevance/tarantula/result.rb
+- lib/relevance/tarantula/test_report.html.erb
+- lib/relevance/tarantula/tidy_handler.rb
+- lib/relevance/tarantula/transform.rb
+- tasks/tarantula_tasks.rake
+- template/tarantula_test.rb
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/Panmind/tarantula
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A big hairy fuzzy spider that crawls your site, wreaking havoc
+test_files: 
+- examples/example_helper.rb
+- examples/relevance/core_extensions/ellipsize_example.rb
+- examples/relevance/core_extensions/file_example.rb
+- examples/relevance/core_extensions/response_example.rb
+- examples/relevance/core_extensions/test_case_example.rb
+- examples/relevance/tarantula/attack_handler_example.rb
+- examples/relevance/tarantula/basic_attack_example.rb
+- examples/relevance/tarantula/crawler_example.rb
+- examples/relevance/tarantula/form_example.rb
+- examples/relevance/tarantula/form_submission_example.rb
+- examples/relevance/tarantula/html_document_handler_example.rb
+- examples/relevance/tarantula/html_report_helper_example.rb
+- examples/relevance/tarantula/html_reporter_example.rb
+- examples/relevance/tarantula/invalid_html_handler_example.rb
+- examples/relevance/tarantula/io_reporter_example.rb
+- examples/relevance/tarantula/link_example.rb
+- examples/relevance/tarantula/log_grabber_example.rb
+- examples/relevance/tarantula/rails_integration_proxy_example.rb
+- examples/relevance/tarantula/result_example.rb
+- examples/relevance/tarantula/tidy_handler_example.rb
+- examples/relevance/tarantula/transform_example.rb
+- examples/relevance/tarantula_example.rb