tarantula-rails3 0.3.3
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGELOG +49 -0
- data/LICENSE +20 -0
- data/README.rdoc +161 -0
- data/Rakefile +83 -0
- data/VERSION.yml +4 -0
- data/examples/example_helper.rb +57 -0
- data/examples/relevance/core_extensions/ellipsize_example.rb +19 -0
- data/examples/relevance/core_extensions/file_example.rb +8 -0
- data/examples/relevance/core_extensions/response_example.rb +29 -0
- data/examples/relevance/core_extensions/test_case_example.rb +20 -0
- data/examples/relevance/tarantula/attack_handler_example.rb +29 -0
- data/examples/relevance/tarantula/basic_attack_example.rb +12 -0
- data/examples/relevance/tarantula/crawler_example.rb +375 -0
- data/examples/relevance/tarantula/form_example.rb +50 -0
- data/examples/relevance/tarantula/form_submission_example.rb +171 -0
- data/examples/relevance/tarantula/html_document_handler_example.rb +43 -0
- data/examples/relevance/tarantula/html_report_helper_example.rb +46 -0
- data/examples/relevance/tarantula/html_reporter_example.rb +82 -0
- data/examples/relevance/tarantula/invalid_html_handler_example.rb +33 -0
- data/examples/relevance/tarantula/io_reporter_example.rb +11 -0
- data/examples/relevance/tarantula/link_example.rb +84 -0
- data/examples/relevance/tarantula/log_grabber_example.rb +26 -0
- data/examples/relevance/tarantula/rails_integration_proxy_example.rb +88 -0
- data/examples/relevance/tarantula/result_example.rb +85 -0
- data/examples/relevance/tarantula/tidy_handler_example.rb +58 -0
- data/examples/relevance/tarantula/transform_example.rb +20 -0
- data/examples/relevance/tarantula_example.rb +23 -0
- data/laf/images/header_bg.jpg +0 -0
- data/laf/images/logo.png +0 -0
- data/laf/images/tagline.png +0 -0
- data/laf/javascripts/jquery-1.2.3.js +3408 -0
- data/laf/javascripts/jquery-ui-tabs.js +890 -0
- data/laf/javascripts/jquery.tablesorter.js +861 -0
- data/laf/javascripts/tarantula.js +10 -0
- data/laf/stylesheets/tarantula.css +346 -0
- data/lib/relevance/core_extensions/ellipsize.rb +34 -0
- data/lib/relevance/core_extensions/file.rb +9 -0
- data/lib/relevance/core_extensions/metaclass.rb +78 -0
- data/lib/relevance/core_extensions/response.rb +9 -0
- data/lib/relevance/core_extensions/string_chars_fix.rb +11 -0
- data/lib/relevance/core_extensions/test_case.rb +19 -0
- data/lib/relevance/tarantula.rb +58 -0
- data/lib/relevance/tarantula/attack.rb +18 -0
- data/lib/relevance/tarantula/attack_handler.rb +37 -0
- data/lib/relevance/tarantula/basic_attack.rb +40 -0
- data/lib/relevance/tarantula/crawler.rb +254 -0
- data/lib/relevance/tarantula/detail.html.erb +81 -0
- data/lib/relevance/tarantula/form.rb +23 -0
- data/lib/relevance/tarantula/form_submission.rb +88 -0
- data/lib/relevance/tarantula/html_document_handler.rb +36 -0
- data/lib/relevance/tarantula/html_report_helper.rb +39 -0
- data/lib/relevance/tarantula/html_reporter.rb +105 -0
- data/lib/relevance/tarantula/index.html.erb +37 -0
- data/lib/relevance/tarantula/invalid_html_handler.rb +18 -0
- data/lib/relevance/tarantula/io_reporter.rb +34 -0
- data/lib/relevance/tarantula/link.rb +94 -0
- data/lib/relevance/tarantula/log_grabber.rb +16 -0
- data/lib/relevance/tarantula/rails_integration_proxy.rb +68 -0
- data/lib/relevance/tarantula/recording.rb +12 -0
- data/lib/relevance/tarantula/response.rb +13 -0
- data/lib/relevance/tarantula/result.rb +77 -0
- data/lib/relevance/tarantula/test_report.html.erb +32 -0
- data/lib/relevance/tarantula/tidy_handler.rb +32 -0
- data/lib/relevance/tarantula/transform.rb +17 -0
- data/lib/relevance/tasks/tarantula_tasks.rake +42 -0
- data/lib/tarantula-rails3.rb +9 -0
- data/template/tarantula_test.rb +22 -0
- metadata +164 -0
@@ -0,0 +1,94 @@
|
|
1
|
+
class Relevance::Tarantula::Link
|
2
|
+
include Relevance::Tarantula
|
3
|
+
|
4
|
+
class << self
|
5
|
+
include ActionView::Helpers::UrlHelper
|
6
|
+
# method_javascript_function needs this method
|
7
|
+
def protect_against_forgery?
|
8
|
+
false
|
9
|
+
end
|
10
|
+
#fast fix for rails3
|
11
|
+
def method_javascript_function(method, url = '', href = nil)
|
12
|
+
action = (href && url.size > 0) ? "'#{url}'" : 'this.href'
|
13
|
+
submit_function =
|
14
|
+
"var f = document.createElement('form'); f.style.display = 'none'; " +
|
15
|
+
"this.parentNode.appendChild(f); f.method = 'POST'; f.action = #{action};"
|
16
|
+
|
17
|
+
unless method == :post
|
18
|
+
submit_function << "var m = document.createElement('input'); m.setAttribute('type', 'hidden'); "
|
19
|
+
submit_function << "m.setAttribute('name', '_method'); m.setAttribute('value', '#{method}'); f.appendChild(m);"
|
20
|
+
end
|
21
|
+
|
22
|
+
if protect_against_forgery?
|
23
|
+
submit_function << "var s = document.createElement('input'); s.setAttribute('type', 'hidden'); "
|
24
|
+
submit_function << "s.setAttribute('name', '#{request_forgery_protection_token}'); s.setAttribute('value', '#{escape_javascript form_authenticity_token}'); f.appendChild(s);"
|
25
|
+
end
|
26
|
+
submit_function << "f.submit();"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
METHOD_REGEXPS = {}
|
31
|
+
[:put, :delete, :post].each do |m|
|
32
|
+
# remove submit from the end so we'll match with or without forgery protection
|
33
|
+
s = method_javascript_function(m).gsub( /f.submit();/, "" )
|
34
|
+
# don't just match this.href in case a different url was passed originally
|
35
|
+
s = Regexp.escape(s).gsub( /this.href/, ".*" )
|
36
|
+
METHOD_REGEXPS[m] = /#{s}/
|
37
|
+
end
|
38
|
+
|
39
|
+
attr_accessor :href, :crawler, :referrer
|
40
|
+
|
41
|
+
def initialize(link, crawler, referrer)
|
42
|
+
@crawler, @referrer = crawler, referrer
|
43
|
+
|
44
|
+
if String === link || link.nil?
|
45
|
+
@href = transform_url(link)
|
46
|
+
@method = :get
|
47
|
+
else # should be a tag
|
48
|
+
@href = link['href'] ? transform_url(link['href'].downcase) : nil
|
49
|
+
@tag = link
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def crawl
|
54
|
+
response = crawler.follow(method, href)
|
55
|
+
log "Response #{response.code} for #{self}"
|
56
|
+
crawler.handle_link_results(self, make_result(response))
|
57
|
+
end
|
58
|
+
|
59
|
+
def make_result(response)
|
60
|
+
crawler.make_result(:method => method,
|
61
|
+
:url => href,
|
62
|
+
:response => response,
|
63
|
+
:referrer => referrer)
|
64
|
+
end
|
65
|
+
|
66
|
+
def method
|
67
|
+
@method ||= begin
|
68
|
+
(@tag &&
|
69
|
+
[:put, :delete, :post].detect do |m| # post should be last since it's least specific
|
70
|
+
@tag['onclick'] =~ METHOD_REGEXPS[m]
|
71
|
+
end) ||
|
72
|
+
:get
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
def transform_url(link)
|
77
|
+
crawler.transform_url(link)
|
78
|
+
end
|
79
|
+
|
80
|
+
def ==(obj)
|
81
|
+
obj.respond_to?(:href) && obj.respond_to?(:method) &&
|
82
|
+
self.href.to_s == obj.href.to_s && self.method.to_s == obj.method.to_s
|
83
|
+
end
|
84
|
+
alias :eql? :==
|
85
|
+
|
86
|
+
def hash
|
87
|
+
to_s.hash
|
88
|
+
end
|
89
|
+
|
90
|
+
def to_s
|
91
|
+
"<Relevance::Tarantula::Link href=#{href}, method=#{method}>"
|
92
|
+
end
|
93
|
+
|
94
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
|
3
|
+
class Relevance::Tarantula::RailsIntegrationProxy
|
4
|
+
include Relevance::Tarantula
|
5
|
+
extend Relevance::Tarantula
|
6
|
+
extend Forwardable
|
7
|
+
attr_accessor :integration_test
|
8
|
+
|
9
|
+
def self.rails_integration_test(integration_test, options = {})
|
10
|
+
t = Crawler.new
|
11
|
+
t.max_url_length = options[:max_url_length] if options[:max_url_length]
|
12
|
+
t.proxy = RailsIntegrationProxy.new(integration_test)
|
13
|
+
t.handlers << HtmlDocumentHandler.new(t)
|
14
|
+
t.handlers << InvalidHtmlHandler.new
|
15
|
+
t.log_grabber = Relevance::Tarantula::LogGrabber.new(File.join(rails_root, "log/test.log"))
|
16
|
+
t.skip_uri_patterns << /logout$/
|
17
|
+
t.transform_url_patterns += [
|
18
|
+
[/\?\d+$/, ''], # strip trailing numbers for assets
|
19
|
+
[/^http:\/\/#{integration_test.host}/, ''] # strip full path down to relative
|
20
|
+
]
|
21
|
+
t.test_name = t.proxy.integration_test.method_name
|
22
|
+
t.reporters << Relevance::Tarantula::HtmlReporter.new(t.report_dir)
|
23
|
+
t
|
24
|
+
end
|
25
|
+
|
26
|
+
def initialize(integration_test)
|
27
|
+
@integration_test = integration_test
|
28
|
+
@integration_test.meta.attr_accessor :response
|
29
|
+
end
|
30
|
+
|
31
|
+
[:get, :post, :put, :delete].each do |verb|
|
32
|
+
define_method(verb) do |url, *args|
|
33
|
+
integration_test.send(verb, url, *args)
|
34
|
+
response = integration_test.response
|
35
|
+
patch_response(url, response)
|
36
|
+
response
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def patch_response(url, response)
|
41
|
+
if response.code == '404'
|
42
|
+
if File.exist?(static_content_path(url))
|
43
|
+
case ext = File.extension(url)
|
44
|
+
when /html|te?xt|css|js|jpe?g|gif|psd|png|eps|pdf|ico/
|
45
|
+
response.body = static_content_file(url)
|
46
|
+
response.headers["type"] = "text/#{ext}" # readable as response.content_type
|
47
|
+
response.meta.attr_accessor :code
|
48
|
+
response.code = "200"
|
49
|
+
else
|
50
|
+
log "Skipping unknown type #{url}"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
# don't count on metaclass taking block, e.g.
|
55
|
+
# http://relevancellc.com/2008/2/12/how-should-metaclass-work
|
56
|
+
response.metaclass.class_eval do
|
57
|
+
include Relevance::CoreExtensions::Response
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def static_content_file(url)
|
62
|
+
File.read(static_content_path(url))
|
63
|
+
end
|
64
|
+
|
65
|
+
def static_content_path(url)
|
66
|
+
File.expand_path(File.join(rails_root, "public", url))
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
# Used to create a stub response when we didn't get back a real response
|
2
|
+
class Relevance::Tarantula::Response
|
3
|
+
HASHABLE_ATTRS = [:code, :body, :content_type]
|
4
|
+
attr_accessor *HASHABLE_ATTRS
|
5
|
+
|
6
|
+
def initialize(hash)
|
7
|
+
hash.each do |k,v|
|
8
|
+
raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
|
9
|
+
self.instance_variable_set("@#{k}", v)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
class Relevance::Tarantula::Result
|
2
|
+
HASHABLE_ATTRS = [:success, :method, :url, :response, :referrer, :data, :description, :log, :test_name]
|
3
|
+
DEFAULT_LOCALHOST = "http://localhost:3000"
|
4
|
+
attr_accessor *HASHABLE_ATTRS
|
5
|
+
include Relevance::Tarantula
|
6
|
+
include Relevance::Tarantula::HtmlReportHelper
|
7
|
+
|
8
|
+
def initialize(hash)
|
9
|
+
hash.each do |k,v|
|
10
|
+
raise ArgumentError, k unless HASHABLE_ATTRS.member?(k)
|
11
|
+
self.instance_variable_set("@#{k}", v)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def short_description
|
16
|
+
[method,url].join(" ")
|
17
|
+
end
|
18
|
+
|
19
|
+
def sequence_number
|
20
|
+
@sequence_number ||= (self.class.next_number += 1)
|
21
|
+
end
|
22
|
+
|
23
|
+
def file_name
|
24
|
+
"#{sequence_number}.html"
|
25
|
+
end
|
26
|
+
|
27
|
+
def code
|
28
|
+
response && response.code
|
29
|
+
end
|
30
|
+
|
31
|
+
def body
|
32
|
+
response && response.body
|
33
|
+
end
|
34
|
+
|
35
|
+
def full_url
|
36
|
+
"#{DEFAULT_LOCALHOST}#{url}"
|
37
|
+
end
|
38
|
+
|
39
|
+
ALLOW_NNN_FOR = /^allow_(\d\d\d)_for$/
|
40
|
+
|
41
|
+
class << self
|
42
|
+
attr_accessor :next_number
|
43
|
+
|
44
|
+
def handle(result)
|
45
|
+
retval = result.dup
|
46
|
+
retval.success = successful?(result.response) || can_skip_error?(result)
|
47
|
+
retval.description = "Bad HTTP Response" unless retval.success
|
48
|
+
retval
|
49
|
+
end
|
50
|
+
|
51
|
+
def success_codes
|
52
|
+
%w{200 201 302 401}
|
53
|
+
end
|
54
|
+
|
55
|
+
# allow_errors_for is a hash
|
56
|
+
# k=error code,
|
57
|
+
# v=array of matchers for urls that can skip said error
|
58
|
+
attr_accessor :allow_errors_for
|
59
|
+
def can_skip_error?(result)
|
60
|
+
coll = allow_errors_for[result.code]
|
61
|
+
return false unless coll
|
62
|
+
coll.any? {|item| item === result.url}
|
63
|
+
end
|
64
|
+
|
65
|
+
def successful?(response)
|
66
|
+
success_codes.member?(response.code)
|
67
|
+
end
|
68
|
+
|
69
|
+
def method_missing(meth, *args)
|
70
|
+
super unless ALLOW_NNN_FOR =~ meth.to_s
|
71
|
+
(allow_errors_for[$1] ||= []).push(*args)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
self.allow_errors_for = {}
|
76
|
+
self.next_number = 0
|
77
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
<div id="<%= test_name %>">
|
2
|
+
<% %w{failures successes}.each do |result_type| %>
|
3
|
+
<table class="list tablesorter" cellspacing="0">
|
4
|
+
<caption><%= send(result_type).size %> <%= result_type.capitalize %></caption>
|
5
|
+
<thead>
|
6
|
+
<tr>
|
7
|
+
<th class="sort asc"><span>URL</span><span class="sort"> </span></th>
|
8
|
+
<th><span>Action</span><span class="sort"> </span></th>
|
9
|
+
<th><span>Response</span><span class="sort"> </span></th>
|
10
|
+
<th class="left"><span>Description</span><span class="sort"> </span></th>
|
11
|
+
<th><span>Referrer</span><span class="sort"> </span></th>
|
12
|
+
</tr>
|
13
|
+
</thead>
|
14
|
+
<tfoot>
|
15
|
+
<tr><td colspan="5"> </td></tr>
|
16
|
+
</tfoot>
|
17
|
+
|
18
|
+
<tbody>
|
19
|
+
<% send(result_type).sort{|x,y| y.code.to_s <=> x.code.to_s}.each_with_index do |result,i| %>
|
20
|
+
<tr class="<%= (i%2 == 0) ? 'even' : 'odd' %>">
|
21
|
+
<td class="left"><a href="<%= "#{test_name}/#{result.file_name}" %>"><%= result.url.ellipsize(50) %></a></td>
|
22
|
+
<td class="method"><%= result.method.to_s.upcase %></td> <!-- TODO Clean up demeter violation -->
|
23
|
+
<td><span class="<%= class_for_code(result.code) %>"><%= result.code %></span></td>
|
24
|
+
<td class="left"><%= result.description %></td>
|
25
|
+
<td class="left"><%= result.referrer.ellipsize(30) %></td>
|
26
|
+
</tr>
|
27
|
+
<% end %>
|
28
|
+
</tbody>
|
29
|
+
</table>
|
30
|
+
<br/>
|
31
|
+
<% end %>
|
32
|
+
</div>
|
@@ -0,0 +1,32 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
begin
|
3
|
+
gem 'tidy'
|
4
|
+
require 'tidy'
|
5
|
+
rescue Gem::LoadError
|
6
|
+
puts "Tidy gem not available -- 'gem install tidy' to get it."
|
7
|
+
end
|
8
|
+
|
9
|
+
if defined? Tidy
|
10
|
+
Tidy.path = ENV['TIDY_PATH'] if ENV['TIDY_PATH']
|
11
|
+
|
12
|
+
class Relevance::Tarantula::TidyHandler
|
13
|
+
include Relevance::Tarantula
|
14
|
+
def initialize(options = {})
|
15
|
+
@options = {:show_warnings=>true}.merge(options)
|
16
|
+
end
|
17
|
+
def handle(result)
|
18
|
+
response = result.response
|
19
|
+
return unless response.html?
|
20
|
+
tidy = Tidy.open(@options) do |tidy|
|
21
|
+
xml = tidy.clean(response.body)
|
22
|
+
tidy
|
23
|
+
end
|
24
|
+
unless tidy.errors.blank?
|
25
|
+
error_result = result.dup
|
26
|
+
error_result.description = "Bad HTML (Tidy)"
|
27
|
+
error_result.data = tidy.errors.inspect
|
28
|
+
error_result
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class Relevance::Tarantula::Transform
|
2
|
+
attr_accessor :from, :to
|
3
|
+
def initialize(from, to)
|
4
|
+
@from = from
|
5
|
+
@to = to
|
6
|
+
end
|
7
|
+
def [](string)
|
8
|
+
case to
|
9
|
+
when Proc
|
10
|
+
string.gsub(from, &to)
|
11
|
+
else
|
12
|
+
string.gsub(from, to)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
|
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'rake'
|
2
|
+
|
3
|
+
namespace :tarantula do
|
4
|
+
|
5
|
+
desc 'Run tarantula tests.'
|
6
|
+
task :test do
|
7
|
+
rm_rf "tmp/tarantula"
|
8
|
+
Rake::TestTask.new(:tarantula_test) do |t|
|
9
|
+
t.libs << 'test'
|
10
|
+
t.pattern = 'test/tarantula/**/*_test.rb'
|
11
|
+
t.verbose = true
|
12
|
+
end
|
13
|
+
|
14
|
+
Rake::Task[:tarantula_test].invoke
|
15
|
+
end
|
16
|
+
|
17
|
+
desc 'Run tarantula tests and open results in your browser.'
|
18
|
+
task :report do
|
19
|
+
begin
|
20
|
+
Rake::Task['tarantula:test'].invoke
|
21
|
+
rescue RuntimeError => e
|
22
|
+
puts e.message
|
23
|
+
end
|
24
|
+
|
25
|
+
Dir.glob("tmp/tarantula/**/index.html") do |file|
|
26
|
+
if PLATFORM['darwin']
|
27
|
+
system("open #{file}")
|
28
|
+
elsif PLATFORM[/linux/]
|
29
|
+
system("firefox #{file}")
|
30
|
+
else
|
31
|
+
puts "You can view tarantula results at #{file}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
desc 'Generate a default tarantula test'
|
37
|
+
task :setup do
|
38
|
+
mkdir_p "test/tarantula"
|
39
|
+
template_path = File.expand_path(File.join(File.dirname(__FILE__), "../../..", "template", "tarantula_test.rb"))
|
40
|
+
cp template_path, "test/tarantula/"
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require "test_helper"
|
2
|
+
require "relevance/tarantula"
|
3
|
+
|
4
|
+
class TarantulaTest < ActionController::IntegrationTest
|
5
|
+
# Load enough test data to ensure that there's a link to every page in your
|
6
|
+
# application. Doing so allows Tarantula to follow those links and crawl
|
7
|
+
# every page. For many applications, you can load a decent data set by
|
8
|
+
# loading all fixtures.
|
9
|
+
fixtures :all
|
10
|
+
|
11
|
+
def test_tarantula
|
12
|
+
# If your application requires users to log in before accessing certain
|
13
|
+
# pages, uncomment the lines below and update them to allow this test to
|
14
|
+
# log in to your application. Doing so allows Tarantula to crawl the
|
15
|
+
# pages that are only accessible to logged-in users.
|
16
|
+
#
|
17
|
+
# post '/session', :login => 'quentin', :password => 'monkey'
|
18
|
+
# follow_redirect!
|
19
|
+
|
20
|
+
tarantula_crawl(self)
|
21
|
+
end
|
22
|
+
end
|