arachni 1.4 → 1.5

data/lib/arachni/element/ui_form.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -16,7 +16,7 @@ class UIForm < Base
 
     include Arachni::Element::Capabilities::DOMOnly
 
-    SUPPORTED_TYPES = Set.new([:input, :button])
+    SUPPORTED_TYPES = %w(input button)
 
     attr_accessor :opening_tags
 
@@ -46,8 +46,7 @@ class UIForm < Base
         return ui_forms if inputs.empty?
 
         # Looks like we have input groups, get buttons with events.
-        browser.elements_with_events.each do |locator, events|
-            next if !SUPPORTED_TYPES.include?( locator.tag_name )
+        browser.each_element_with_events SUPPORTED_TYPES do |locator, events|
             next if locator.tag_name == :input &&
                 locator.attributes['type'] != 'button' &&
                 locator.attributes['type'] != 'submit'
@@ -76,7 +75,7 @@ class UIForm < Base
         inputs       = {}
 
         if UIInput.with_textarea_in_html?( page.body )
-            page.document.css( 'textarea' ).each do |textarea|
+            page.document.nodes_by_name( :textarea ).each do |textarea|
                 name = node_to_name( textarea )
 
                 inputs[name]       = textarea.text
@@ -86,7 +85,7 @@ class UIForm < Base
         end
 
         if UIInput.with_input_in_html?( page.body )
-            page.document.css( 'input' ).each do |input|
+            page.document.nodes_by_name( :input ).each do |input|
                 next if input['type'] && input['type'] != 'text'
 
                 name = node_to_name( input )

data/lib/arachni/element/ui_form/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/ui_input.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -16,7 +16,7 @@ class UIInput < Base
 
     include Arachni::Element::Capabilities::DOMOnly
 
-    SUPPORTED_TYPES = Set.new([:input, :textarea])
+    SUPPORTED_TYPES = %w(input textarea)
 
     def self.type
         :ui_input
@@ -27,10 +27,8 @@ class UIInput < Base
 
         return inputs if !browser.javascript.supported? || !in_html?( page.body )
 
-        browser.elements_with_events.each do |locator, events|
-            next if !SUPPORTED_TYPES.include?( locator.tag_name )
-            next if locator.attributes['type'] &&
-                locator.attributes['type'] != 'text'
+        browser.each_element_with_events SUPPORTED_TYPES do |locator, events|
+            next if locator.attributes['type'] && locator.attributes['type'] != 'text'
 
             events.each do |event, _|
                 name = locator.attributes['name'] || locator.attributes['id'] ||

data/lib/arachni/element/ui_input/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/xml.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -19,6 +19,8 @@ class XML < Base
     Dir.glob( lib ).each { |f| require f }
 
     include Arachni::Element::Capabilities::Auditable
+    include Arachni::Element::Capabilities::Auditable::Buffered
+    include Arachni::Element::Capabilities::Auditable::LineBuffered
     include Arachni::Element::Capabilities::Submittable
     include Arachni::Element::Capabilities::Analyzable
     include Arachni::Element::Capabilities::WithSource
@@ -87,17 +89,11 @@ class XML < Base
         { self.action => self.inputs }
     end
 
-    # @param   (see .encode)
-    # @return  (see .encode)
-    #
     # @see .encode
     def encode( *args )
         self.class.encode( *args )
     end
 
-    # @param   (see .decode)
-    # @return  (see .decode)
-    #
     # @see .decode
     def decode( *args )
         self.class.decode( *args )

data/lib/arachni/element/xml/capabilities/inputtable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/xml/capabilities/mutable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element_filter.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/error.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/ethon/easy.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/audit.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -188,6 +188,11 @@ module Audit
                 pending_jobs = browser_cluster.pending_job_counter
                 if pending_jobs != last_pending_jobs
                     browser_cluster.print_info "Pending jobs: #{pending_jobs}"
+
+                    browser_cluster.print_debug 'Current jobs:'
+                    browser_cluster.workers.each do |worker|
+                        browser_cluster.print_debug worker.job.to_s
+                    end
                 end
                 last_pending_jobs = pending_jobs
 

data/lib/arachni/framework/parts/browser.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -76,17 +76,14 @@ module Browser
         @browser_job     = nil
     end
 
-    def browser_sitemap
-        return {} if !@browser_cluster
-        browser_cluster.sitemap
-    end
-
     def browser_job_update_skip_states( states )
         return if states.empty?
         browser_cluster.update_skip_states browser_job.id, states
     end
 
-    def handle_browser_page( page )
+    def handle_browser_page( result, * )
+        page = result.is_a?( Page ) ? result : result.page
+
         synchronize do
             return if !push_to_page_queue page
 
@@ -114,9 +111,11 @@ module Browser
         # needs to have a clean state.
         schedule_dom_metadata_application( page )
 
-        browser_cluster.queue( browser_job.forward( resource: page.dom.state ) ) do |result|
-            handle_browser_page result.page
-        end
+        @perform_browser_analysis_cb ||= method(:handle_browser_page)
+        browser_cluster.queue(
+            browser_job.forward( resource: page.dom.state ),
+            @perform_browser_analysis_cb
+        )
 
         true
     end
@@ -132,16 +131,17 @@ module Browser
             find { |c| c.check? page, [Element::Form::DOM, Element::Cookie::DOM], true }
 
         dom = page.dom.state
-        browser_cluster.with_browser do |browser|
-            apply_dom_metadata( browser, dom )
-        end
+        dom.page = nil # Help out the GC.
+
+        @dom_metadata_application_cb ||= method(:apply_dom_metadata)
+        browser_cluster.with_browser dom, @dom_metadata_application_cb
     end
 
     def apply_dom_metadata( browser, dom )
         bp = nil
 
         begin
-            bp = browser.load( dom ).to_page
+            bp = browser.load( dom, take_snapshot: false ).to_page
         rescue Selenium::WebDriver::Error::WebDriverError,
             Watir::Exception::Error => e
             print_debug "Could not apply metadata to '#{dom.url}'" <<

data/lib/arachni/framework/parts/check.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/data.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/platform.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/plugin.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/report.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -84,7 +84,7 @@ module Report
 
             IO.binread( outfile )
         ensure
-            File.delete( outfile ) if outfile
+            File.delete( outfile ) if outfile && File.exists?( outfile )
             @reporters.clear
             @reporters.load loaded
         end

data/lib/arachni/framework/parts/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/state.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/http.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/http/client.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -85,6 +85,8 @@ class Client
     # Default 1 minute timeout for HTTP requests.
     HTTP_TIMEOUT = 60_000
 
+    SEED_HEADER_NAME = 'X-Arachni-Scan-Seed'
+
     # @return   [String]
     #   Framework target URL, used as reference.
     attr_reader :url
@@ -129,12 +131,13 @@ class Client
 
         headers.clear
         headers.merge!(
-            'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-            'User-Agent'      => Options.http.user_agent,
-            'Accept-Language' => 'en-US,en;q=0.8,he;q=0.6'
+            'Accept'              => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'User-Agent'          => Options.http.user_agent,
+            'Accept-Language'     => 'en-US,en;q=0.8,he;q=0.6',
+            SEED_HEADER_NAME      => Arachni::Utilities.random_seed
         )
         headers['From'] = Options.authorized_by if Options.authorized_by
-        headers.merge!( Options.http.request_headers )
+        headers.merge!( Options.http.request_headers, false )
     end
 
     # @return   [Arachni::HTTP]
@@ -374,13 +377,34 @@ class Client
                 end
             end
 
+            on_headers    = options.delete(:on_headers)
+            on_body       = options.delete(:on_body)
+            on_body_line  = options.delete(:on_body_line)
+            on_body_lines = options.delete(:on_body_lines)
+
             request = Request.new( options.merge(
                 url:         url,
-                headers:     headers.merge( options.delete( :headers ) || {} ),
+                headers:     headers.merge( options.delete( :headers ) || {}, false ),
                 cookies:     cookies,
                 raw_cookies: raw_cookies
             ))
 
+            if on_headers
+                request.on_headers( &on_headers )
+            end
+
+            if on_body
+                request.on_body( &on_body )
+            end
+
+            if on_body_line
+                request.on_body_line( &on_body_line )
+            end
+
+            if on_body_lines
+                request.on_body_lines( &on_body_lines )
+            end
+
             if block_given?
                 request.on_complete( &block )
             end
@@ -531,7 +555,8 @@ class Client
         end
 
         if add_callbacks
-            request.on_complete( &method(:global_on_complete) )
+            @global_on_complete ||= method(:global_on_complete)
+            request.on_complete( &@global_on_complete )
         end
 
         synchronize { @request_count += 1 }

data/lib/arachni/http/client/dynamic_404_handler.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -35,7 +35,9 @@ class Dynamic404Handler
     # @param  [Block]   block
     #   To be passed `true` or `false` depending on the result of the analysis.
     def _404?( response, &block )
-        url = response.url
+        # This matters, the request URL may differ from the response one due to
+        # redirections and we need to test the original.
+        url = response.request.url
 
         # Easy pickins, well-behaved static 404 handler and a URL that doesn't
         # need advanced analysis.
@@ -253,7 +255,7 @@ class Dynamic404Handler
                         # Both attempts yielded in the same result, the webapp
                         # was stable during the process and the signature can be
                         # considered accurate.
-                        if control_data[:rdiff] == signature_data[:rdiff]
+                        if control_data[:rdiff].similar? signature_data[:rdiff]
                             block.call response, :done
 
                         # Coo-coo for cocoa puffs, can't work with it.
@@ -329,7 +331,8 @@ class Dynamic404Handler
         !!(
             !resource_name.empty? ||
             uri.resource_extension ||
-            uri.resource_name.to_s.include?( '~' )
+            uri.resource_name.to_s.include?( '~' ) ||
+            uri.resource_name.to_s.include?( '-' )
         )
     end
 
@@ -371,15 +374,22 @@ class Dynamic404Handler
         up_to_path = uri.up_to_path
 
         trv_back = File.dirname( Arachni::URI( up_to_path ).path )
-        trv_back += '/' if trv_back[-1] != '/'
+        trv_back << '/' if trv_back[-1] != '/'
 
         parsed = uri.dup
-        parsed.path  = trv_back
-        trv_back_url = parsed.to_s
+        parsed.path   = trv_back
+        parsed.query  = ''
+        trv_back_url  = parsed.to_s
 
-        [
+        g = [
             # Get a random path with an extension.
-            proc { up_to_path + random_string + '.' + random_string[0..precision] },
+            proc {
+                s = up_to_path.dup
+                s << random_string
+                s << '.'
+                s << random_string[0..precision]
+                s
+            },
 
             # Get a random path without an extension.
             proc { up_to_path + random_string },
@@ -395,11 +405,32 @@ class Dynamic404Handler
             proc { trv_back_url + random_string_alpha_capital },
 
             # Move up a dir and get a random file with an extension.
-            proc { trv_back_url + random_string + '.' + random_string[0..precision] },
+            proc {
+                s = trv_back_url.dup
+                s << random_string
+                s << '.'
+                s << random_string[0..precision]
+                s
+            },
 
             # Get a random directory.
-            proc { up_to_path + random_string + '/' }
+            proc {
+                s = up_to_path.dup
+                s << random_string
+                s << '/'
+                s
+            }
         ]
+
+        if !(rn = uri.resource_name.to_s).empty?
+            # Append a random string to the resource name.
+            g << proc { url.gsub( rn, "#{rn}#{random_string[0..precision]}" ) }
+
+            # Prepend a random string to the resource name.
+            g << proc { url.gsub( rn, "#{random_string[0..precision]}#{rn}" ) }
+        end
+
+        g
     end
 
     # @return   [Array<Proc>]
@@ -413,12 +444,37 @@ class Dynamic404Handler
 
         if !resource_name.empty?
             # Get an existing resource with a random extension.
-            probes << proc { up_to_path + resource_name + '.' + random_string[0..precision] }
+            probes << proc {
+                s = up_to_path.dup
+                s << resource_name
+                s << '.'
+                s << random_string[0..precision]
+                s
+            }
         end
 
         if resource_extension
             # Get a random filename with an existing extension.
-            probes << proc { up_to_path + random_string + '.' + resource_extension }
+            probes << proc {
+                s = up_to_path.dup
+                s << random_string
+                s << '.'
+                s << resource_extension
+                s
+            }
+        end
+
+        # Some webapps do routing based on name resources with "-" as a separator.
+        if uri.resource_name.include?( '-' )
+            rn = uri.resource_name
+
+            probes << proc {
+                up_to_path.sub( rn, rn.gsub( '-', "#{random_string}-" ) )
+            }
+
+            probes << proc {
+                up_to_path.sub( rn, rn.gsub( '-', "-#{random_string}" ) )
+            }
         end
 
         if uri.resource_name.include?( '~' )
@@ -436,13 +492,15 @@ class Dynamic404Handler
     def request( url, &block )
         Client.get( url,
             # This is important, helps us reduce waiting callers.
-            high_priority: true,
+            high_priority:   true,
 
             # We're going to be checking for a lot of non-existent resources,
             # don't bother fingerprinting them
-            fingerprint:   false,
+            fingerprint:     false,
+
+            follow_location: true,
 
-            performer:     self,
+            performer:       self,
             &block
         )
     end