arachni 1.4 → 1.6.0

data/lib/arachni/element/server.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -67,7 +67,7 @@ class Server < Base
     #   * `true` if everything went fine.
     #
     # @see #remote_file_exist?
-    def log_remote_file_if_exists( url, silent = false, &block )
+    def log_remote_file_if_exists( url, silent = false, options = {}, &block )
         # Make sure the URL is valid.
         return false if !(url.start_with?( 'http://' ) || url.start_with?( 'https://' ))
 
@@ -76,7 +76,7 @@ class Server < Base
             auditor.print_status( "Analyzing response for: #{url}" ) if !silent
             next if !bool
 
-            @candidates << [response, block]
+            @candidates << [response, block, options]
         end
     end
     alias :log_remote_directory_if_exists :log_remote_file_if_exists
@@ -105,7 +105,7 @@ class Server < Base
             #
             # If a resource does exist though it will be fingerprinted down the
             # line.
-            http.get( url, performer: self, fingerprint: false ) do |r|
+            http.get( url, performer: self, fingerprint: false, follow_location: true ) do |r|
                 if r.code == 200
                     http.dynamic_404_handler._404?( r ) { |bool| block.call( !bool, r ) }
                 else
@@ -113,7 +113,7 @@ class Server < Base
                 end
             end
         else
-            http.request( url, method: :head, performer: self ) do |response|
+            http.request( url, method: :head, performer: self, follow_location: true ) do |response|
                 block.call( response.code == 200, response )
             end
         end
@@ -158,11 +158,11 @@ class Server < Base
         return if @candidates.empty?
 
         if @candidates.size == 1
-            response, block = @candidates.first
+            response, block, options = @candidates.first
 
             # Single issue, not enough confidence to use it for training, err
             # on the side of caution.
-            log response, false, &block
+            log response, false, options, &block
 
             return
         end
@@ -188,8 +188,8 @@ class Server < Base
         train = similarity < SIMILARITY_TOLERANCE
 
         issue_digests = []
-        @candidates.each do |response, block|
-            issue_digests << log( response, train, &block ).digest
+        @candidates.each do |response, block, options|
+            issue_digests << log( response, train, options, &block ).digest
         end
 
         return if train
@@ -199,10 +199,10 @@ class Server < Base
         @candidates.clear
     end
 
-    def log( response, train = true, &block )
+    def log( response, train = true, options = {}, &block )
         block.call( response ) if block_given?
 
-        issue = auditor.log_remote_file( response )
+        issue = auditor.log_remote_file( response, false, options )
 
         return issue if !train
 

data/lib/arachni/element/ui_form/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/ui_form.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -16,7 +16,7 @@ class UIForm < Base
 
     include Arachni::Element::Capabilities::DOMOnly
 
-    SUPPORTED_TYPES = Set.new([:input, :button])
+    SUPPORTED_TYPES = %w(input button)
 
     attr_accessor :opening_tags
 
@@ -46,8 +46,7 @@ class UIForm < Base
         return ui_forms if inputs.empty?
 
         # Looks like we have input groups, get buttons with events.
-        browser.elements_with_events.each do |locator, events|
-            next if !SUPPORTED_TYPES.include?( locator.tag_name )
+        browser.each_element_with_events SUPPORTED_TYPES do |locator, events|
             next if locator.tag_name == :input &&
                 locator.attributes['type'] != 'button' &&
                 locator.attributes['type'] != 'submit'
@@ -76,7 +75,7 @@ class UIForm < Base
         inputs       = {}
 
         if UIInput.with_textarea_in_html?( page.body )
-            page.document.css( 'textarea' ).each do |textarea|
+            page.document.nodes_by_name( :textarea ).each do |textarea|
                 name = node_to_name( textarea )
 
                 inputs[name]       = textarea.text
@@ -86,7 +85,7 @@ class UIForm < Base
         end
 
         if UIInput.with_input_in_html?( page.body )
-            page.document.css( 'input' ).each do |input|
+            page.document.nodes_by_name( :input ).each do |input|
                 next if input['type'] && input['type'] != 'text'
 
                 name = node_to_name( input )

data/lib/arachni/element/ui_input/dom.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/ui_input.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -16,7 +16,7 @@ class UIInput < Base
 
     include Arachni::Element::Capabilities::DOMOnly
 
-    SUPPORTED_TYPES = Set.new([:input, :textarea])
+    SUPPORTED_TYPES = %w(input textarea)
 
     def self.type
         :ui_input
@@ -27,10 +27,8 @@ class UIInput < Base
 
         return inputs if !browser.javascript.supported? || !in_html?( page.body )
 
-        browser.elements_with_events.each do |locator, events|
-            next if !SUPPORTED_TYPES.include?( locator.tag_name )
-            next if locator.attributes['type'] &&
-                locator.attributes['type'] != 'text'
+        browser.each_element_with_events SUPPORTED_TYPES do |locator, events|
+            next if locator.attributes['type'] && locator.attributes['type'] != 'text'
 
             events.each do |event, _|
                 name = locator.attributes['name'] || locator.attributes['id'] ||

data/lib/arachni/element/xml/capabilities/inputtable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/xml/capabilities/mutable.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/xml.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -19,6 +19,8 @@ class XML < Base
     Dir.glob( lib ).each { |f| require f }
 
     include Arachni::Element::Capabilities::Auditable
+    include Arachni::Element::Capabilities::Auditable::Buffered
+    include Arachni::Element::Capabilities::Auditable::LineBuffered
     include Arachni::Element::Capabilities::Submittable
     include Arachni::Element::Capabilities::Analyzable
     include Arachni::Element::Capabilities::WithSource
@@ -87,17 +89,11 @@ class XML < Base
         { self.action => self.inputs }
     end
 
-    # @param   (see .encode)
-    # @return  (see .encode)
-    #
     # @see .encode
     def encode( *args )
         self.class.encode( *args )
     end
 
-    # @param   (see .decode)
-    # @return  (see .decode)
-    #
     # @see .decode
     def decode( *args )
         self.class.decode( *args )

data/lib/arachni/element_filter.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/error.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/ethon/easy.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/audit.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -188,6 +188,11 @@ module Audit
                 pending_jobs = browser_cluster.pending_job_counter
                 if pending_jobs != last_pending_jobs
                     browser_cluster.print_info "Pending jobs: #{pending_jobs}"
+
+                    browser_cluster.print_debug 'Current jobs:'
+                    browser_cluster.workers.each do |worker|
+                        browser_cluster.print_debug worker.job.to_s
+                    end
                 end
                 last_pending_jobs = pending_jobs
 

data/lib/arachni/framework/parts/browser.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -76,17 +76,14 @@ module Browser
         @browser_job     = nil
     end
 
-    def browser_sitemap
-        return {} if !@browser_cluster
-        browser_cluster.sitemap
-    end
-
     def browser_job_update_skip_states( states )
         return if states.empty?
         browser_cluster.update_skip_states browser_job.id, states
     end
 
-    def handle_browser_page( page )
+    def handle_browser_page( result, * )
+        page = result.is_a?( Page ) ? result : result.page
+
         synchronize do
             return if !push_to_page_queue page
 
@@ -114,9 +111,11 @@ module Browser
         # needs to have a clean state.
         schedule_dom_metadata_application( page )
 
-        browser_cluster.queue( browser_job.forward( resource: page.dom.state ) ) do |result|
-            handle_browser_page result.page
-        end
+        @perform_browser_analysis_cb ||= method(:handle_browser_page)
+        browser_cluster.queue(
+            browser_job.forward( resource: page.dom.state ),
+            @perform_browser_analysis_cb
+        )
 
         true
     end
@@ -132,16 +131,17 @@ module Browser
             find { |c| c.check? page, [Element::Form::DOM, Element::Cookie::DOM], true }
 
         dom = page.dom.state
-        browser_cluster.with_browser do |browser|
-            apply_dom_metadata( browser, dom )
-        end
+        dom.page = nil # Help out the GC.
+
+        @dom_metadata_application_cb ||= method(:apply_dom_metadata)
+        browser_cluster.with_browser dom, @dom_metadata_application_cb
     end
 
     def apply_dom_metadata( browser, dom )
         bp = nil
 
         begin
-            bp = browser.load( dom ).to_page
+            bp = browser.load( dom, take_snapshot: false ).to_page
         rescue Selenium::WebDriver::Error::WebDriverError,
             Watir::Exception::Error => e
             print_debug "Could not apply metadata to '#{dom.url}'" <<

data/lib/arachni/framework/parts/check.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/data.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/platform.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/plugin.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/report.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -79,12 +79,12 @@ module Report
                      "Reporter '#{name}' cannot format the audit results as a String."
             end
 
-            outfile = "#{Arachni.tmpdir}/#{generate_token}"
+            outfile = "#{Options.paths.tmpdir}/#{generate_token}"
             @reporters.run( name, external_report, outfile: outfile )
 
             IO.binread( outfile )
         ensure
-            File.delete( outfile ) if outfile
+            File.delete( outfile ) if outfile && File.exists?( outfile )
             @reporters.clear
             @reporters.load loaded
         end

data/lib/arachni/framework/parts/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework/parts/state.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/framework.rb

@@ -1,7 +1,7 @@
 # encoding: utf-8
 
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/http/client/dynamic_404_handler.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -35,7 +35,9 @@ class Dynamic404Handler
     # @param  [Block]   block
     #   To be passed `true` or `false` depending on the result of the analysis.
     def _404?( response, &block )
-        url = response.url
+        # This matters, the request URL may differ from the response one due to
+        # redirections and we need to test the original.
+        url = response.request.url
 
         # Easy pickins, well-behaved static 404 handler and a URL that doesn't
         # need advanced analysis.
@@ -253,7 +255,7 @@ class Dynamic404Handler
                         # Both attempts yielded in the same result, the webapp
                         # was stable during the process and the signature can be
                         # considered accurate.
-                        if control_data[:rdiff] == signature_data[:rdiff]
+                        if control_data[:rdiff].similar? signature_data[:rdiff]
                             block.call response, :done
 
                         # Coo-coo for cocoa puffs, can't work with it.
@@ -329,7 +331,8 @@ class Dynamic404Handler
         !!(
             !resource_name.empty? ||
             uri.resource_extension ||
-            uri.resource_name.to_s.include?( '~' )
+            uri.resource_name.to_s.include?( '~' ) ||
+            uri.resource_name.to_s.include?( '-' )
         )
     end
 
@@ -371,15 +374,22 @@ class Dynamic404Handler
         up_to_path = uri.up_to_path
 
         trv_back = File.dirname( Arachni::URI( up_to_path ).path )
-        trv_back += '/' if trv_back[-1] != '/'
+        trv_back << '/' if trv_back[-1] != '/'
 
         parsed = uri.dup
-        parsed.path  = trv_back
-        trv_back_url = parsed.to_s
+        parsed.path   = trv_back
+        parsed.query  = ''
+        trv_back_url  = parsed.to_s
 
-        [
+        g = [
             # Get a random path with an extension.
-            proc { up_to_path + random_string + '.' + random_string[0..precision] },
+            proc {
+                s = up_to_path.dup
+                s << random_string
+                s << '.'
+                s << random_string[0..precision]
+                s
+            },
 
             # Get a random path without an extension.
             proc { up_to_path + random_string },
@@ -395,11 +405,32 @@ class Dynamic404Handler
             proc { trv_back_url + random_string_alpha_capital },
 
             # Move up a dir and get a random file with an extension.
-            proc { trv_back_url + random_string + '.' + random_string[0..precision] },
+            proc {
+                s = trv_back_url.dup
+                s << random_string
+                s << '.'
+                s << random_string[0..precision]
+                s
+            },
 
             # Get a random directory.
-            proc { up_to_path + random_string + '/' }
+            proc {
+                s = up_to_path.dup
+                s << random_string
+                s << '/'
+                s
+            }
         ]
+
+        if !(rn = uri.resource_name.to_s).empty?
+            # Append a random string to the resource name.
+            g << proc { url.gsub( rn, "#{rn}#{random_string[0..precision]}" ) }
+
+            # Prepend a random string to the resource name.
+            g << proc { url.gsub( rn, "#{random_string[0..precision]}#{rn}" ) }
+        end
+
+        g
     end
 
     # @return   [Array<Proc>]
@@ -413,12 +444,37 @@ class Dynamic404Handler
 
         if !resource_name.empty?
             # Get an existing resource with a random extension.
-            probes << proc { up_to_path + resource_name + '.' + random_string[0..precision] }
+            probes << proc {
+                s = up_to_path.dup
+                s << resource_name
+                s << '.'
+                s << random_string[0..precision]
+                s
+            }
         end
 
         if resource_extension
             # Get a random filename with an existing extension.
-            probes << proc { up_to_path + random_string + '.' + resource_extension }
+            probes << proc {
+                s = up_to_path.dup
+                s << random_string
+                s << '.'
+                s << resource_extension
+                s
+            }
+        end
+
+        # Some webapps do routing based on name resources with "-" as a separator.
+        if uri.resource_name.include?( '-' )
+            rn = uri.resource_name
+
+            probes << proc {
+                up_to_path.sub( rn, rn.gsub( '-', "#{random_string}-" ) )
+            }
+
+            probes << proc {
+                up_to_path.sub( rn, rn.gsub( '-', "-#{random_string}" ) )
+            }
         end
 
         if uri.resource_name.include?( '~' )
@@ -436,13 +492,15 @@ class Dynamic404Handler
     def request( url, &block )
         Client.get( url,
             # This is important, helps us reduce waiting callers.
-            high_priority: true,
+            high_priority:   true,
 
             # We're going to be checking for a lot of non-existent resources,
             # don't bother fingerprinting them
-            fingerprint:   false,
+            fingerprint:     false,
+
+            follow_location: true,
 
-            performer:     self,
+            performer:       self,
             &block
         )
     end