relevance-tarantula 0.0.1

data/test/relevance/tarantula/link_test.rb

@@ -0,0 +1,49 @@
+require File.join(File.dirname(__FILE__), "..", "..", "test_helper.rb")
+include Relevance::Tarantula
+
+describe "Relevance::Tarantula::Link" do
+  include ActionView::Helpers::UrlHelper
+  
+  it "parses anchor tags" do
+    link = Relevance::Tarantula::Link.new(Hpricot('<a href="/foo">foo</a>').at('a'))
+    assert_equal "/foo", link.href
+    assert_equal :get, link.method
+  end
+
+  it "parses anchor tags with POST 'method'" do
+    link = Relevance::Tarantula::Link.new(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:post)}">foo</a>}).at('a'))
+    assert_equal "/foo", link.href
+    assert_equal :post, link.method
+  end
+
+  it "parses anchor tags with PUT 'method'" do
+    link = Relevance::Tarantula::Link.new(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:put)}">foo</a>}).at('a'))
+    assert_equal "/foo", link.href
+    assert_equal :put, link.method
+  end
+
+  it "parses anchor tags with DELETE 'method'" do
+    link = Relevance::Tarantula::Link.new(Hpricot(%Q{<a href="/foo" onclick="#{method_javascript_function(:delete)}">foo</a>}).at('a'))
+    assert_equal "/foo", link.href
+    assert_equal :delete, link.method
+  end
+
+  it "parses link tags with text" do
+    link = Relevance::Tarantula::Link.new(Hpricot('<link href="/bar">bar</a>').at('link'))
+    assert_equal "/bar", link.href
+    assert_equal :get, link.method
+  end
+  
+  it "parses link tags without text" do
+    link = Relevance::Tarantula::Link.new(Hpricot('<link href="/bar" />').at('link'))
+    assert_equal "/bar", link.href
+    assert_equal :get, link.method
+  end
+  
+  # method_javascript_function needs this method
+  def protect_against_forgery?
+    false
+  end
+  
+end
+

data/test/relevance/tarantula/log_grabber_test.rb

@@ -0,0 +1,27 @@
+require File.join(File.dirname(__FILE__), "..", "..", "test_helper.rb")
+include Relevance::Tarantula
+
+describe 'Relevance::Tarantula::LogGrabber' do
+  before do
+    @grabber = LogGrabber.new(log_file)
+    FileUtils.mkdir_p(test_output_dir)
+  end
+
+  def log_file
+    File.join(File.join(test_output_dir, "example.log"))
+  end
+
+  it "can clear the log file" do
+    File.open(log_file, "w") {|f| f.print "sample log"}
+    File.size(log_file).should == 10
+    @grabber.clear!
+    File.size(log_file).should == 0
+  end
+
+  it "can grab the log file" do
+    File.open(log_file, "w") {|f| f.print "sample log"}
+    @grabber.grab!.should == "sample log"
+    File.size(log_file).should == 0
+  end
+
+end

data/test/relevance/tarantula/rails_integration_proxy_test.rb

@@ -0,0 +1,95 @@
+require File.join(File.dirname(__FILE__), "..", "..", "test_helper.rb")
+include Relevance::Tarantula
+
+describe "Relevance::Tarantula::RailsIntegrationProxy rails_integration_test" do
+  before {
+    Crawler.any_instance.stubs(:crawl)
+    Crawler.any_instance.stubs(:rails_root).returns("STUB_RAILS_ROOT")
+    RailsIntegrationProxy.stubs(:rails_root).returns("STUB_RAILS_ROOT")
+    RailsIntegrationProxy.stubs(:new).returns(stub(:integration_test => stub(:method_name => @test_name)))
+    @test_name = "test_user_pages"
+  }
+
+  it "strips leading hostname from link urls" do    
+    crawler = RailsIntegrationProxy.rails_integration_test(stub(:host => "foo.com"))
+    crawler.transform_url("http://foo.com/path").should == "/path"
+    crawler.transform_url("http://bar.com/path").should == "http://bar.com/path"
+  end
+  
+  it "allows override of max_url_length" do
+    crawler = RailsIntegrationProxy.rails_integration_test(stub(:host => "foo.com"), 
+                                             :max_url_length => 16)
+    crawler.max_url_length.should == 16
+  end
+
+  it "has some useful defaults" do
+    crawler = RailsIntegrationProxy.rails_integration_test(stub(:host => "foo.com")) 
+    crawler.log_grabber.should.not.be nil
+  end
+end
+
+
+describe "Relevance::Tarantula::RailsIntegrationProxy" do
+  %w{get post}.each do |http_method|
+    it "can #{http_method}" do
+      @rip = Relevance::Tarantula::RailsIntegrationProxy.new(stub)
+      @response = stub({:code => :foo})
+      @rip.integration_test = stub_everything(:response => @response)
+      @rip.send(http_method, "/url").should.be @response
+    end
+  end
+  
+  it "adds a response accessor to its delegate rails integration test" do
+    o = Object.new
+    Relevance::Tarantula::RailsIntegrationProxy.new(o)
+    o.methods(false).sort.should == %w{response response=}
+  end
+
+end
+
+describe "Relevance::Tarantula::RailsIntegrationProxy patching" do
+  before do
+    @rip = Relevance::Tarantula::RailsIntegrationProxy.new(stub)
+    @rip.stubs(:rails_root).returns("faux_rails_root")
+    @response = stub_everything({:code => "404", :headers => {}})
+    File.stubs(:exist?).returns(true)
+  end
+  
+  it "patches in Relevance::CoreExtensions::Response" do
+    @rip = Relevance::Tarantula::RailsIntegrationProxy.new(stub)
+    @rip.stubs(:rails_root).returns("faux_rails_root")
+    @response = stub_everything({:code => "404", :headers => {}, :content_type => "text/html"})
+    @response.meta.ancestors.should.not.include Relevance::CoreExtensions::Response
+    @rip.patch_response("/url", @response)
+    @response.meta.ancestors.should.include Relevance::CoreExtensions::Response
+    @response.html?.should == true
+  end
+  
+  it "ignores 404s for known static binary types" do
+    File.expects(:extension).returns("pdf")
+    @rip.expects(:log).with("Skipping /url (for now)")
+    @rip.patch_response("/url", @response)
+  end
+  
+  it "replaces 404s with 200s, pulling content from public, for known text types" do
+    File.expects(:extension).returns("html")
+    @rip.expects(:static_content_file).with("/url").returns("File body")
+    @rip.patch_response("/url", @response)
+    @response.headers.should == {"type" => "text/html"}
+  end
+  
+  it "logs and skips types we haven't dealt with yet" do
+    File.expects(:extension).returns("whizzy")
+    @rip.expects(:log).with("Skipping unknown type /url")
+    @rip.patch_response("/url", @response)
+  end
+  
+  it "can find static content relative to rails root" do
+    @rip.static_content_path("foo").should == File.expand_path("faux_rails_root/public/foo")
+  end
+  
+  it "can read static content relative to rails root" do
+    File.expects(:read).with(@rip.static_content_path("foo"))
+    @rip.static_content_file("foo")
+  end
+end

data/test/relevance/tarantula/result_test.rb

@@ -0,0 +1,86 @@
+require File.join(File.dirname(__FILE__), "..", "..", "test_helper.rb")
+include Relevance::Tarantula
+
+describe "Relevance::Tarantula::Result" do
+  before do
+    @result = Relevance::Tarantula::Result.new(
+        :success => true, 
+        :method => "get", 
+        :url => "/some/url?arg1=foo&arg2=bar"
+    )
+  end
+  
+  it "has a short description" do
+    @result.short_description.should == "get /some/url?arg1=foo&arg2=bar"
+  end
+  
+  it "has a sequence number" do
+    @result.class.next_number = 0
+    @result.sequence_number.should == 1
+    @result.class.next_number.should == 1
+  end
+  
+  it "has link to the url at localhost" do
+    @result.full_url.should == "http://localhost:3000/some/url?arg1=foo&arg2=bar"
+  end
+  
+end
+
+describe "Relevance::Tarantula::Result class methods" do
+  before do
+    @rh = Relevance::Tarantula::Result
+  end
+  
+  it "defines HTTP responses that are considered 'successful' when spidering" do
+    %w{200 201 302 401}.each do |code|
+      @rh.successful?(stub(:code => code)).should == true
+    end
+  end
+  
+  it "adds successful responses to success collection" do
+    stub = stub_everything(:code => "200")
+    @rh.handle(Result.new(:response => stub)).success.should == true
+  end
+
+  it "adds failed responses to failure collection" do
+    stub = stub_everything(:code => "500")
+    result = @rh.handle(Result.new(:response => stub))
+    result.success.should == false
+    result.description.should == "Bad HTTP Response"
+  end
+  
+end
+
+describe "Relevance::Tarantula::Result allowed errors" do
+  before do
+    Result.allow_errors_for = {}
+  end
+  
+  it "defaults to *not* skip errors" do
+    Result.can_skip_error?(stub(:code => "404")).should == false
+  end
+
+  it "can skip errors matching code and url" do
+    Result.allow_errors_for = {"404" => [/some_url/]}
+    Result.can_skip_error?(stub(:code => "404", :url => "this_is_some_url")).should == true
+  end
+
+  it "does not skip errors matching code only" do
+    Result.allow_errors_for = {"404" => [/some_other_url/]}
+    Result.can_skip_error?(stub(:code => "404", :url => "this_is_some_url")).should == false
+  end
+  
+  it "users allow_nnn_for syntax to specify allowed errors" do
+    Result.allow_404_for(/this_url/)
+    Result.allow_errors_for.should == {"404" => [/this_url/]}
+    Result.allow_404_for(/another_url/)
+    Result.allow_errors_for.should == {"404" => [/this_url/, /another_url/]}
+  end
+  
+  it "chains to super method missing" do
+    lambda{Result.allow_xxx_for}.should.raise(NoMethodError)
+  end
+  
+end
+
+

data/test/relevance/tarantula/tidy_handler_test.rb

@@ -0,0 +1,59 @@
+require File.join(File.dirname(__FILE__), "..", "..", "test_helper.rb")
+include Relevance::Tarantula
+
+if defined?(Tidy) && ENV['TIDY_PATH']
+  describe "Relevance::Tarantula::TidyHandler default" do
+    before do
+      @handler = Relevance::Tarantula::TidyHandler.new
+    end
+  
+    it "likes a good document" do
+      response = stub(:html? => true, :body => <<-BODY, :code => 200)
+<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 3.2//EN\">
+<html>
+  <title></title>
+  <body></body>
+</html>
+BODY
+      @handler.handle(Result.new(:response => response)).should == nil
+    end
+
+    it "rejects a document with errors" do
+      response = stub(:html? => true, :body => "<hotml>", :code => 200)
+      result = @handler.handle(Result.new(:response => response))
+      result.should.not.be nil
+      result.data.should =~ /Error: <hotml> is not recognized!/
+      result.description.should == "Bad HTML (Tidy)"
+    end
+
+    it "rejects a document with warnings" do
+      response = stub(:html? => true, :body => <<-BODY, :code => 200)
+<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 3.2//EN\">
+<html>
+</html>
+BODY
+      result = @handler.handle(Result.new(:response => response))
+      result.should.not.be nil
+      result.data.should =~ /Warning: inserting missing 'title' element/
+    end
+    
+  end
+  
+  describe "Relevance::Tarantula::TidyHandler with :show_warnings => false" do
+    before do
+      @handler = Relevance::Tarantula::TidyHandler.new(:show_warnings => false)
+    end
+
+    it "permits a document with warnings" do
+      response = stub(:html? => true, :body => <<-BODY, :code => 200)
+<!DOCTYPE html PUBLIC \"-//W3C//DTD HTML 3.2//EN\">
+<html>
+</html>
+BODY
+      result = @handler.handle(Result.new(:response => response))
+      result.should.be nil
+    end
+  end
+else
+  puts "TIDY_PATH not set. Tidy test will not run"
+end

data/test/relevance/tarantula/transform_test.rb

@@ -0,0 +1,21 @@
+require File.join(File.dirname(__FILE__), "../..", "test_helper.rb")
+
+describe "Relevance::Tarantula::Transform" do
+  it "can do a simple replace" do
+    t = Relevance::Tarantula::Transform.new(/\w/, ".")
+    t["hello world"].should == "..... ....."
+  end
+  
+  it "can do a replace with a block" do
+    t = Relevance::Tarantula::Transform.new(/([aeiou])/, Proc.new {|value| value.upcase})
+    t["hello world"].should == "hEllO wOrld"
+  end
+
+  # this is broken in Ruby?
+  it "cannot access groups from a block, despite Ruby docs" do
+    p = Proc.new {|value| $1.upcase}
+    t = Relevance::Tarantula::Transform.new(/([aeiou])/, p)
+    lambda {t["hello world"]}.should.raise(NoMethodError).message.should ==
+                              "undefined method `upcase' for nil:NilClass"
+  end
+end

data/test/relevance/tarantula_test.rb

@@ -0,0 +1,23 @@
+require File.join(File.dirname(__FILE__), "..", "test_helper.rb")
+
+describe "Relevance::Tarantula" do
+  include Relevance::Tarantula
+  attr_accessor :verbose
+  
+  it "writes to stdout if verbose" do
+    self.verbose = true
+    expects(:puts).with("foo")
+    log("foo")
+  end
+
+  it "swallows output if !verbose" do
+    self.verbose = false
+    expects(:puts).never
+    log("foo")
+  end
+  
+  it "puts RAILS_ROOT behind a method call" do
+    lambda{rails_root}.should.raise(NameError).message.should == "uninitialized constant RAILS_ROOT"
+  end
+end
+

data/test/test_helper.rb

@@ -0,0 +1,32 @@
+basedir = File.dirname(__FILE__)
+$:.unshift "#{basedir}/../lib"
+require 'rubygems' 
+gem 'ruby-debug'
+gem 'test-spec'
+gem 'activesupport'
+gem 'actionpack'
+gem 'activerecord'
+
+require 'test/spec'
+require 'mocha'    
+require 'ostruct'
+require 'ruby-debug'
+require 'activerecord'
+require 'relevance/tarantula'
+
+# needed for html-scanner, grr
+require 'active_support'
+require 'action_controller'
+
+class Test::Unit::TestCase 
+  def test_output_dir
+    File.join(File.dirname(__FILE__), "..", "tmp", "test_output")
+  end
+
+  # TODO change puts/print to use a single method for logging, which will then make the stubbing cleaner
+  def stub_puts_and_print(obj)
+    obj.stubs(:puts)
+    obj.stubs(:print)
+  end
+  
+end

data/uninstall.rb

@@ -0,0 +1 @@
+# Uninstall hook code here

data/vendor/xss-shield/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2007 Trampoline Systems
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/vendor/xss-shield/README

@@ -0,0 +1,76 @@
+FIXME: THIS README IS NOT UP-TO-DATE.
+
+This plugin provides XSS protection for views coded in HAML and RHTML.
+
+ERB templates are sometimes used for HTML, and sometimes for
+other kinds of languages (SQL, email templates, YAML etc.).
+XSS Shield protects only those templates with .rhtml extension,
+leaving templates with .erb extension unprotected.
+
+=== Quick start ===
+
+Assuming you're using HAML for all your templates.
+
+* Install plugin.
+* Edit all your layout files and change:
+    = @content_for_layout
+    = yield(:foo) # Foo being usually :js or :css
+  to:
+    = @content_for_layout.mark_as_xss_protected
+    = yield(:foo).mark_as_xss_protected
+* By this point your application should be runnanble,
+  but might need some tweaking here and there to avoid potential
+  double-escaping.
+
+=== How it works ===
+
+It works by subclassing String into SafeString.
+When HAML engine seems a "= foo" fragment it check if result of executing "foo"
+is a SafeString. If it is - it copies it to the output, if it's anything else
+(String, Integer, nil and so on) it HTML-escapes it first.
+
+To avoid double-escaping output of h is a SafeString, as is everything you
+mark as XSS-protected.
+  = h(@foo)
+  = @foo # fully equivalent to h(@foo)
+  = "X <br /> Y".mark_as_xss_protected
+
+It would be cumbersome to require mark_as_xss_protected every time you use
+some helper like render :partial or link_to, so some helpers are modified
+to return SafeString.
+
+  = render :partial => "foo"
+  = link_to "Bar", :action => :bar
+
+If you trust your helpers, make them as XSS-protected:
+
+  module Some::Module
+    mark_helpers_as_xss_protected :text_field, :check_box
+  end
+
+Because it is not possible to alter syntactic keywords like yield
+or instance variables like @content_for_layout to mark them automatically
+as secure, layout files need some manual tweaking.
+
+=== Other template engines ===
+
+If a templates uses some templating engine other than HAML or ERB,
+or it uses ERB but has extension .erb not .rhtml, XSS Shield does not protect it.
+
+However some helpers like link_to and button_to are patched by XSS Shield to
+make them more secure, and this extra security will be there even when used
+in an otherwise unprotected context.
+
+For example with XSS shield
+  link_to "A & B", "/foo"
+will return (marked as safe):
+  '<a href="/foo">A &amp; B</a>'
+not (plain String):
+  '<a href="/foo">A & B</a>'
+
+Also - RHTML protection only works with default ERB engine (erb.rb from Ruby base).
+If you use some alternative ERB engine it probably won't work.
+
+Adding support for alternative templating engine should be relatively straightforward.
+It's mostly a matter of changing to_s to to_s_xss_protected in a few places
+in their source.

data/vendor/xss-shield/init.rb

@@ -0,0 +1,16 @@
+unless ENV['DISABLE_XSS_SHIELD']
+  puts "Loading XSS Shield"
+  require 'xss_shield'
+else
+  class ::String
+    def mark_as_xss_protected
+      self
+    end
+  end
+
+  class ::NilClass
+    def mark_as_xss_protected
+      self
+    end
+  end
+end

data/vendor/xss-shield/lib/xss_shield/erb_hacks.rb

@@ -0,0 +1,111 @@
+class XSSProtectedERB < ERB
+  class Compiler < ::ERB::Compiler
+    def compile(s)
+      out = Buffer.new(self)
+
+      content = ''
+      scanner = make_scanner(s)
+      scanner.scan do |token|
+  if scanner.stag.nil?
+    case token
+          when PercentLine
+      out.push("#{@put_cmd} #{content.dump}") if content.size > 0
+      content = ''
+            out.push(token.to_s)
+            out.cr
+    when :cr
+      out.cr
+    when '<%', '<%=', '<%#'
+      scanner.stag = token
+      out.push("#{@put_cmd} #{content.dump}") if content.size > 0
+      content = ''
+    when "\n"
+      content << "\n"
+      out.push("#{@put_cmd} #{content.dump}")
+      out.cr
+      content = ''
+    when '<%%'
+      content << '<%'
+    else
+      content << token
+    end
+  else
+    case token
+    when '%>'
+      case scanner.stag
+      when '<%'
+        if content[-1] == ?\n
+    content.chop!
+    out.push(content)
+    out.cr
+        else
+    out.push(content)
+        end
+      when '<%='
+              # NOTE: Changed lines
+        out.push("#{@insert_cmd}((#{content}).to_s_xss_protected)")
+              # NOTE: End changed lines
+      when '<%#'
+        # out.push("# #{content.dump}")
+      end
+      scanner.stag = nil
+      content = ''
+    when '%%>'
+      content << '%>'
+    else
+      content << token
+    end
+  end
+      end
+      out.push("#{@put_cmd} #{content.dump}") if content.size > 0
+      out.close
+      out.script
+    end
+  end
+
+  def initialize(str, safe_level=nil, trim_mode=nil, eoutvar='_erbout')
+    @safe_level = safe_level
+    compiler = XSSProtectedERB::Compiler.new(trim_mode)
+    set_eoutvar(compiler, eoutvar)
+    @src = compiler.compile(str)
+    @filename = nil
+  end
+end
+
+module ActionView
+  class Base
+    private
+      def create_template_source(extension, template, render_symbol, locals)
+        if template_requires_setup?(extension)
+          body = case extension.to_sym
+            when :rxml, :builder
+              content_type_handler = (controller.respond_to?(:response) ? "controller.response" : "controller")
+              "#{content_type_handler}.content_type ||= Mime::XML\n" +
+              "xml = Builder::XmlMarkup.new(:indent => 2)\n" +
+              template +
+              "\nxml.target!\n"
+            when :rjs
+              "controller.response.content_type ||= Mime::JS\n" +
+              "update_page do |page|\n#{template}\nend"
+          end
+        # NOTE: Changed lines
+        elsif extension.to_sym == :rhtml
+          body = XSSProtectedERB.new(template, nil, @@erb_trim_mode).src
+        # NOTE: End changed lines
+        else
+          body = ERB.new(template, nil, @@erb_trim_mode).src
+        end
+
+        @@template_args[render_symbol] ||= {}
+        locals_keys = @@template_args[render_symbol].keys | locals
+        @@template_args[render_symbol] = locals_keys.inject({}) { |h, k| h[k] = true; h }
+
+        locals_code = ""
+        locals_keys.each do |key|
+          locals_code << "#{key} = local_assigns[:#{key}]\n"
+        end
+
+        "def #{render_symbol}(local_assigns)\n#{locals_code}#{body}\nend"
+      end
+  end
+end

data/vendor/xss-shield/lib/xss_shield/haml_hacks.rb

@@ -0,0 +1,42 @@
+raise "Haml not loaded" unless Haml::Engine.instance_method(:push_script)
+
+module Haml
+  class Engine
+    def push_script(text, flattened)
+      unless options[:suppress_eval]
+        push_silent("haml_temp = #{text}", true)
+        push_silent("haml_temp = haml_temp.to_s_xss_protected", true)
+        out = "haml_temp = _hamlout.push_script(haml_temp, #{@output_tabs}, #{flattened})\n"
+        if @block_opened
+          push_and_tabulate([:loud, out])
+        else
+          @precompiled << out
+        end
+      end
+    end
+
+    def build_attributes(attributes = {})
+      # We ignore @options[:attr_wrapper] because ERB::Util.h does not espace ' to &apos;
+      # making ' as attribute quote not workable
+      result = attributes.map do |a,v|
+        v = v.to_s_xss_protected
+        unless v.blank?
+          " #{a}=\"#{v}\""
+        end
+      end
+      result.sort.join
+    end
+  end
+
+  class Buffer
+    def build_attributes(attributes = {})
+      result = attributes.map do |a,v|
+        v = v.to_s_xss_protected
+        unless v.blank?
+          " #{a}=\"#{v}\""
+        end
+      end
+      result.sort.join
+    end
+  end
+end