tarantula 0.1.5 → 0.1.8

data/examples/relevance/tarantula/form_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe "Relevance::Tarantula::Form large example" do
   before do

data/examples/relevance/tarantula/form_submission_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe "Relevance::Tarantula::FormSubmission" do
   

data/examples/relevance/tarantula/html_document_handler_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe "Relevance::Tarantula::HtmlDocumentHandler" do
   

data/examples/relevance/tarantula/html_report_helper_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 module HtmlReportHelperSpec
   # Is there an idiom for this?

data/examples/relevance/tarantula/html_reporter_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe "Relevance::Tarantula::HtmlReporter file output" do
 

data/examples/relevance/tarantula/invalid_html_handler_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe "Relevance::Tarantula::InvalidHtmlHandler" do
   before do

data/examples/relevance/tarantula/io_reporter_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe 'Relevance::Tarantula::IOReporter' do
   it "reports errors to stderr and then raises" do

data/examples/relevance/tarantula/link_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe "Relevance::Tarantula::Link" do
   include ActionView::Helpers::UrlHelper

data/examples/relevance/tarantula/log_grabber_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe 'Relevance::Tarantula::LogGrabber' do
   before do

data/examples/relevance/tarantula/rails_integration_proxy_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe "Relevance::Tarantula::RailsIntegrationProxy rails_integration_test" do
   before {

data/examples/relevance/tarantula/result_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 describe "Relevance::Tarantula::Result" do
   before do

data/examples/relevance/tarantula/tidy_handler_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "..", "example_helper.rb"))
 
 if defined?(Tidy) && ENV['TIDY_PATH']
   describe "Relevance::Tarantula::TidyHandler default" do

data/examples/relevance/tarantula/transform_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "../..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "../..", "example_helper.rb"))
 
 describe "Relevance::Tarantula::Transform" do
   it "can do a simple replace" do

data/examples/relevance/tarantula_example.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__), "..", "example_helper.rb")
+require File.expand_path(File.join(File.dirname(__FILE__), "..", "example_helper.rb"))
 
 describe Relevance::Tarantula do
   include Relevance::Tarantula

data/lib/relevance/core_extensions/string_chars_fix.rb

@@ -0,0 +1,11 @@
+if RUBY_VERSION == "1.8.7" # fix interaction between Ruby 187 and Rails 202, so we can at least run the test suite on that combination
+  unless '1.9'.respond_to?(:force_encoding)
+    String.class_eval do
+      begin
+        remove_method :chars
+      rescue NameError
+        # OK
+      end
+    end
+  end
+end

data/lib/relevance/core_extensions/test_case.rb

@@ -1,4 +1,7 @@
-class Test::Unit::TestCase
+require 'action_controller/integration'
+
+module Relevance::CoreExtensions::TestCaseExtensions
+
   def tarantula_crawl(integration_test, options = {})
     url = options[:url] || "/"
     t = tarantula_crawler(integration_test, options)
@@ -8,5 +11,9 @@ class Test::Unit::TestCase
   def tarantula_crawler(integration_test, options = {})
     Relevance::Tarantula::RailsIntegrationProxy.rails_integration_test(integration_test, options)
   end
+  
 end
 
+if defined? ActionController::IntegrationTest
+  ActionController::IntegrationTest.class_eval { include Relevance::CoreExtensions::TestCaseExtensions }
+end

data/lib/relevance/tarantula.rb

@@ -2,7 +2,6 @@ TARANTULA_ROOT = File.expand_path(File.join(File.dirname(__FILE__), "../.."))
 
 require 'forwardable'
 require 'erb'
-require 'rubygems'
 require 'active_support'
 require 'action_controller'
 
@@ -38,6 +37,7 @@ require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "e
 require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "file"))
 require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "response"))
 require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "metaclass"))
+require File.expand_path(File.join(File.dirname(__FILE__), "core_extensions", "string_chars_fix"))
 
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_reporter"))
 require File.expand_path(File.join(File.dirname(__FILE__), "tarantula", "html_report_helper"))

data/lib/relevance/tarantula/crawler.rb

@@ -1,3 +1,5 @@
+require 'active_record'
+require 'active_record/base'
 require File.expand_path(File.join(File.dirname(__FILE__), "rails_integration_proxy"))
 require File.expand_path(File.join(File.dirname(__FILE__), "html_document_handler.rb"))
 
@@ -5,11 +7,13 @@ class Relevance::Tarantula::Crawler
   extend Forwardable
   include Relevance::Tarantula
 
+  class CrawlTimeout < RuntimeError; end
+
   attr_accessor :proxy, :handlers, :skip_uri_patterns, :log_grabber,
                 :reporters, :links_to_crawl, :links_queued, :forms_to_crawl,
                 :form_signatures_queued, :max_url_length, :response_code_handler,
-                :times_to_crawl, :fuzzers, :test_name
-  attr_reader   :transform_url_patterns, :referrers, :failures, :successes
+                :times_to_crawl, :fuzzers, :test_name, :crawl_timeout
+  attr_reader   :transform_url_patterns, :referrers, :failures, :successes, :crawl_start_times, :crawl_end_times
 
   def initialize
     @max_url_length = 1024
@@ -20,6 +24,8 @@ class Relevance::Tarantula::Crawler
     @form_signatures_queued = Set.new
     @links_to_crawl = []
     @forms_to_crawl = []
+    @crawl_start_times, @crawl_end_times = [], []
+    @crawl_timeout = 20.minutes
     @referrers = {}
     @skip_uri_patterns = [
       /^javascript/,
@@ -51,13 +57,18 @@ class Relevance::Tarantula::Crawler
     orig_form_signatures_queued = @form_signatures_queued.dup
     orig_links_to_crawl = @links_to_crawl.dup
     orig_forms_to_crawl = @forms_to_crawl.dup
-    @times_to_crawl.times do |i|
+    @times_to_crawl.times do |num|
       queue_link url
-      do_crawl
-
-      puts "#{(i+1).ordinalize} crawl" if @times_to_crawl > 1
+      
+      begin 
+        do_crawl num
+      rescue CrawlTimeout => e
+        puts e.message
+      end
+      
+      puts "#{(num+1).ordinalize} crawl" if @times_to_crawl > 1
 
-      if i + 1 < @times_to_crawl
+      if num + 1 < @times_to_crawl
         @links_queued = orig_links_queued
         @form_signatures_queued = orig_form_signatures_queued
         @links_to_crawl = orig_links_to_crawl
@@ -75,19 +86,21 @@ class Relevance::Tarantula::Crawler
     @links_to_crawl.empty? && @forms_to_crawl.empty?
   end
 
-  def do_crawl
+  def do_crawl(number)
     while (!finished?)
-      crawl_queued_links
-      crawl_queued_forms
+      @crawl_start_times << Time.now
+      crawl_queued_links(number)
+      crawl_queued_forms(number)
+      @crawl_end_times << Time.now
     end
   end
 
-  def crawl_queued_links
+  def crawl_queued_links(number = 0)
     while (link = @links_to_crawl.pop)
       response = proxy.send(link.method, link.href)
       log "Response #{response.code} for #{link}"
       handle_link_results(link, response)
-      blip
+      blip(number)
     end
   end
 
@@ -122,13 +135,17 @@ class Relevance::Tarantula::Crawler
     Relevance::Tarantula::Response.new(:code => "404", :body => e.message, :content_type => "text/plain")
   end
 
-  def crawl_queued_forms
+  def crawl_queued_forms(number = 0)
     while (form = @forms_to_crawl.pop)
       response = crawl_form(form)
       handle_form_results(form, response)
-      blip
+      blip(number)
     end
   end
+  
+  def elasped_time_for_pass(num)
+    Time.now - crawl_start_times[num]
+  end
 
   def grab_log!
     @log_grabber && @log_grabber.grab!
@@ -232,9 +249,16 @@ class Relevance::Tarantula::Crawler
       total_links_count - links_remaining_count
   end
 
-  def blip
+  def blip(number = 0)
     unless verbose
       print "\r #{links_completed_count} of #{total_links_count} links completed               "
+      timeout_if_too_long(number)
+    end
+  end
+  
+  def timeout_if_too_long(number = 0)
+    if elasped_time_for_pass(number) > crawl_timeout
+      raise CrawlTimeout, "Exceeded crawl timeout of #{crawl_timeout} seconds - skipping to the next crawl..."
     end
   end
 end

data/lib/relevance/tarantula/index.html.erb

@@ -23,9 +23,9 @@
       and lives at <a href="http://github.com/relevance/tarantula">http://github.com/relevance/tarantula</a>.</p>
       <hr/>
     </div>
-    <div id="page">  
+    <div id="page">
       <div id="tabs-container">
-        <ul class="tabs"></ul>
+        <ul class="tabs"> </ul>
     	</div>    
     
       <div id="results-container">

data/lib/relevance/tarantula/test_report.html.erb

@@ -18,7 +18,7 @@
       <tbody>
         <% send(result_type).sort{|x,y| y.code.to_s <=> x.code.to_s}.each_with_index do |result,i| %>
           <tr class="<%= (i%2 == 0) ? 'even' : 'odd' %>">
-            <td class="left"><a href="<%= "#{test_name}/#{result.file_name}" %>"><%= result.url.ellipsize(30) %></a></td>
+            <td class="left"><a href="<%= "#{test_name}/#{result.file_name}" %>"><%= result.url.ellipsize(50) %></a></td>
             <td class="method"><%= result.method.to_s.upcase %></td> <!-- TODO Clean up demeter violation -->
             <td><span class="<%= class_for_code(result.code) %>"><%= result.code %></span></td>
             <td class="left"><%= result.description %></td>

data/lib/relevance/tarantula/tidy_handler.rb

@@ -3,7 +3,7 @@ begin
   gem 'tidy'
   require 'tidy'
 rescue Gem::LoadError
-  # tidy not available
+  puts "Tidy gem not available -- 'gem install tidy' to get it."
 end
 
 if defined? Tidy

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: tarantula
 version: !ruby/object:Gem::Version 
-  version: 0.1.5
+  version: 0.1.8
 platform: ruby
 authors: 
 - Relevance, Inc.
@@ -9,32 +9,48 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-02-20 00:00:00 -05:00
+date: 2009-04-07 00:00:00 -04:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: htmlentities
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hpricot
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
 description: A big hairy fuzzy spider that crawls your site, wreaking havoc
 email: opensource@thinkrelevance.com
 executables: []
 
 extensions: []
 
-extra_rdoc_files: []
-
+extra_rdoc_files: 
+- README.rdoc
 files: 
 - CHANGELOG
 - MIT-LICENSE
-- Rakefile
 - README.rdoc
+- Rakefile
 - VERSION.yml
 - examples/example_helper.rb
-- examples/relevance
-- examples/relevance/core_extensions
 - examples/relevance/core_extensions/ellipsize_example.rb
 - examples/relevance/core_extensions/file_example.rb
 - examples/relevance/core_extensions/response_example.rb
 - examples/relevance/core_extensions/test_case_example.rb
-- examples/relevance/tarantula
 - examples/relevance/tarantula/attack_form_submission_example.rb
 - examples/relevance/tarantula/attack_handler_example.rb
 - examples/relevance/tarantula/crawler_example.rb
@@ -47,35 +63,26 @@ files:
 - examples/relevance/tarantula/io_reporter_example.rb
 - examples/relevance/tarantula/link_example.rb
 - examples/relevance/tarantula/log_grabber_example.rb
-- examples/relevance/tarantula/rails_init_example.rb
 - examples/relevance/tarantula/rails_integration_proxy_example.rb
 - examples/relevance/tarantula/result_example.rb
-- examples/relevance/tarantula/STUB_RAILS_ROOT
-- examples/relevance/tarantula/STUB_RAILS_ROOT/tmp
-- examples/relevance/tarantula/STUB_RAILS_ROOT/tmp/tarantula
 - examples/relevance/tarantula/tidy_handler_example.rb
 - examples/relevance/tarantula/transform_example.rb
 - examples/relevance/tarantula_example.rb
-- laf/images
 - laf/images/button_active.png
 - laf/images/button_hover.png
 - laf/images/button_inactive.png
 - laf/images/header_bg.jpg
 - laf/images/logo.png
 - laf/images/tagline.png
-- laf/javascripts
 - laf/javascripts/jquery-1.2.3.js
 - laf/javascripts/jquery-ui-tabs.js
 - laf/javascripts/jquery.tablesorter.js
+- laf/javascripts/niftyLayout.js
 - laf/javascripts/niftycube-details.js
 - laf/javascripts/niftycube.js
-- laf/javascripts/niftyLayout.js
 - laf/javascripts/tarantula.js
-- laf/stylesheets
 - laf/stylesheets/tarantula.css
-- laf/v2
 - laf/v2/detail.html
-- laf/v2/images
 - laf/v2/images/button_active.png
 - laf/v2/images/button_hover.png
 - laf/v2/images/button_inactive.png
@@ -83,16 +90,14 @@ files:
 - laf/v2/images/logo.png
 - laf/v2/images/tagline.png
 - laf/v2/index.html
-- laf/v2/stylesheets
 - laf/v2/stylesheets/tarantula.v2.css
-- lib/relevance
-- lib/relevance/core_extensions
 - lib/relevance/core_extensions/ellipsize.rb
 - lib/relevance/core_extensions/file.rb
 - lib/relevance/core_extensions/metaclass.rb
 - lib/relevance/core_extensions/response.rb
+- lib/relevance/core_extensions/string_chars_fix.rb
 - lib/relevance/core_extensions/test_case.rb
-- lib/relevance/tarantula
+- lib/relevance/tarantula.rb
 - lib/relevance/tarantula/attack.rb
 - lib/relevance/tarantula/attack_form_submission.rb
 - lib/relevance/tarantula/attack_handler.rb
@@ -115,14 +120,12 @@ files:
 - lib/relevance/tarantula/test_report.html.erb
 - lib/relevance/tarantula/tidy_handler.rb
 - lib/relevance/tarantula/transform.rb
-- lib/relevance/tarantula.rb
 - tasks/tarantula_tasks.rake
 - template/tarantula_test.rb
 has_rdoc: true
 homepage: http://github.com/relevance/tarantula
 post_install_message: 
 rdoc_options: 
-- --inline-source
 - --charset=UTF-8
 require_paths: 
 - lib
@@ -140,10 +143,31 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   version: 
 requirements: []
 
-rubyforge_project: 
+rubyforge_project: thinkrelevance
 rubygems_version: 1.3.1
 signing_key: 
 specification_version: 2
 summary: A big hairy fuzzy spider that crawls your site, wreaking havoc
-test_files: []
-
+test_files: 
+- examples/example_helper.rb
+- examples/relevance/core_extensions/ellipsize_example.rb
+- examples/relevance/core_extensions/file_example.rb
+- examples/relevance/core_extensions/response_example.rb
+- examples/relevance/core_extensions/test_case_example.rb
+- examples/relevance/tarantula/attack_form_submission_example.rb
+- examples/relevance/tarantula/attack_handler_example.rb
+- examples/relevance/tarantula/crawler_example.rb
+- examples/relevance/tarantula/form_example.rb
+- examples/relevance/tarantula/form_submission_example.rb
+- examples/relevance/tarantula/html_document_handler_example.rb
+- examples/relevance/tarantula/html_report_helper_example.rb
+- examples/relevance/tarantula/html_reporter_example.rb
+- examples/relevance/tarantula/invalid_html_handler_example.rb
+- examples/relevance/tarantula/io_reporter_example.rb
+- examples/relevance/tarantula/link_example.rb
+- examples/relevance/tarantula/log_grabber_example.rb
+- examples/relevance/tarantula/rails_integration_proxy_example.rb
+- examples/relevance/tarantula/result_example.rb
+- examples/relevance/tarantula/tidy_handler_example.rb
+- examples/relevance/tarantula/transform_example.rb
+- examples/relevance/tarantula_example.rb