arachni 1.4 → 1.6.0

data/lib/arachni/data/framework/rpc.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -37,7 +37,7 @@ class RPC
         distributed_page_queue.buffer.each do |page|
             IO.binwrite(
                 "#{page_queue_directory}/#{page.persistent_hash}",
-                Marshal.dump( page )
+                distributed_page_queue.serialize( page )
             )
         end
 

data/lib/arachni/data/framework.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -49,6 +49,7 @@ class Framework
         @sitemap = {}
 
         @page_queue            = Support::Database::Queue.new
+        @page_queue.max_buffer_size = 10
         @page_queue_total_size = 0
 
         @url_queue                 = Support::Database::Queue.new
@@ -117,7 +118,7 @@ class Framework
         page_queue.buffer.each do |page|
             IO.binwrite(
                 "#{page_queue_directory}/#{page.persistent_hash}",
-                 Marshal.dump( page )
+                page_queue.serialize( page )
             )
         end
 

data/lib/arachni/data/issues.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/data/plugins.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/data/session.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/data.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/base.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/body.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/element/capabilities/analyzable/differential.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -110,7 +110,7 @@ module Differential
         return if self.inputs.empty?
 
         with_missing_values = Set.new( self.inputs.select { |k, v| v.to_s.empty? }.keys )
-        if self.inputs.size == with_missing_values.size
+        if self.inputs.size > 1 && self.inputs.size == with_missing_values.size
             print_debug 'Differential analysis: Inputs are missing default values.'
             return false
         end
@@ -127,12 +127,12 @@ module Differential
         @differential_analysis_options = opts.dup
         opts = self.class::MUTATION_OPTIONS.merge( DIFFERENTIAL_OPTIONS.merge( opts ) )
         opts[:skip_like] = proc do |mutation|
-            with_missing_values.include? mutation.affected_input_name
+            self.inputs.size > 1 &&
+                with_missing_values.include?( mutation.affected_input_name )
         end
 
         mutations_size = 0
         each_mutation( opts[:false], opts ) { mutations_size += 1 }
-        mutations_size *= opts[:precision]
 
         @data_gathering = {
             mutations_size:     mutations_size,
@@ -143,7 +143,7 @@ module Differential
         }
 
         # Holds all the data from the probes.
-        signatures = {
+        @signatures = {
             # Control baseline per input.
             controls:              {},
 
@@ -158,14 +158,14 @@ module Differential
         }
 
         # Populate the baseline/control forced-false signatures.
-        populate_control_signatures( opts, signatures )
+        populate_control_signatures( opts )
 
         http.after_run do
             # Populate the 'true' signatures.
-            populate_signatures( :true, opts, signatures )
+            populate_signatures( :true, opts )
 
             # Populate the 'false' signatures.
-            populate_signatures( :false, opts, signatures )
+            populate_signatures( :false, opts )
         end
 
         true
@@ -187,126 +187,86 @@ module Differential
 
     # Performs requests using the 'false' control seed and generates/stores
     # signatures based on the response bodies.
-    def populate_control_signatures( opts, signatures )
-        gathered = {}
-        opts[:precision].times do
-            audit( opts[:false], opts ) do |res, elem|
-                altered_hash = elem.affected_input_name.hash
-
-                if signatures[:corrupted][altered_hash]
-                    increase_received_responses( opts, signatures )
-                    next
-                end
-
-                gathered[altered_hash] ||= 0
-                gathered[altered_hash]  += 1
-
-                response_check( res, signatures, elem )
+    def populate_control_signatures( opts )
+        gather_signatures( opts[:false], opts ) do |signature, _, elem|
+            altered_hash = elem.affected_input_name.hash
 
-                if gathered[altered_hash] == @data_gathering[:mutations_size]
-                    print_status "Got default/control response for #{elem.type} " +
-                        "variable '#{elem.affected_input_name}' with action '#{elem.action}'."
+            if @signatures[:corrupted][altered_hash]
+                increase_received_responses( opts )
+                next
+            end
 
-                    @data_gathering[:controls][altered_hash] = true
-                end
+            @signatures[:controls][altered_hash] = signature
 
-                body = res.body.gsub( elem.seed, '' )
+            increase_received_responses( opts )
 
-                # Create a signature from the response body and refine it with
-                # subsequent ones to remove noise (like context-irrelevant dynamic
-                # content such as banners etc.).
-                signatures[:controls][altered_hash] =
-                    signatures[:controls][altered_hash] ?
-                        signatures[:controls][altered_hash].refine!(body) :
-                        Support::Signature.new(body)
+            print_status "Got default/control response for #{elem.type} " <<
+                "variable '#{elem.affected_input_name}' with action '#{elem.action}'."
 
-                increase_received_responses( opts, signatures )
-            end
+            @data_gathering[:controls][altered_hash] = true
         end
     end
 
-    def populate_signatures( bool, opts, signatures )
-        gathered = {}
-
+    def populate_signatures( bool, opts )
         opts[:pairs].each do |pair|
             pair_hash = pair.hash
 
-            signatures[pair_hash]      ||= {}
+            @signatures[pair_hash]      ||= {}
             @data_gathering[pair_hash] ||= {}
-            gathered[pair_hash]        ||= {}
 
             expr = pair.to_a.first[bool == :true ? 0 : 1]
 
             print_status "Gathering '#{bool}' data for #{self.type} with " <<
                              "action '#{self.action}' using seed: #{expr}"
 
-            opts[:precision].times do
-                audit( expr, opts ) do |res, elem|
-                    altered_hash = elem.affected_input_name.hash
-
-                    gathered[pair_hash][altered_hash] ||= 0
-                    gathered[pair_hash][altered_hash] += 1
-
-                    signatures[pair_hash][altered_hash]      ||= {}
-                    @data_gathering[pair_hash][altered_hash] ||= {}
-
-                    response_check( res, signatures, elem )
-
-                    if signatures[pair_hash][altered_hash][:corrupted] ||
-                        signatures[:corrupted][altered_hash]
+            gather_signatures( expr, opts ) do |signature, res, elem|
+                altered_hash = elem.affected_input_name.hash
 
-                        increase_received_responses( opts, signatures )
-                        next
-                    end
+                @signatures[pair_hash][altered_hash]      ||= {}
+                @data_gathering[pair_hash][altered_hash] ||= {}
 
-                    if signature_sieve( altered_hash, signatures, pair_hash )
-                        increase_received_responses( opts, signatures )
-                        next
-                    end
+                if @signatures[:corrupted][altered_hash]
+                    increase_received_responses( opts  )
+                    next
+                end
 
-                    if gathered[pair_hash][altered_hash] == opts[:precision]
-                        elem.print_status "Got '#{bool}' response for #{elem.type}" <<
-                            " variable '#{elem.affected_input_name}' with action" <<
-                            " '#{elem.action}' using seed: #{expr}"
+                if signature_sieve( altered_hash, pair_hash )
+                    increase_received_responses( opts )
+                    next
+                end
 
-                        @data_gathering[pair_hash][altered_hash]["#{bool}_probes".to_sym] = true
-                    end
+                elem.print_status "Got '#{bool}' response for #{elem.type}" <<
+                    " variable '#{elem.affected_input_name}' with action" <<
+                    " '#{elem.action}' using seed: #{expr}"
 
-                    # Store the mutation for the {Arachni::Issue}.
-                    signatures[pair_hash][altered_hash][:mutation] ||= elem
+                @data_gathering[pair_hash][altered_hash]["#{bool}_probes".to_sym] = true
 
-                    # Keep the latest response for the {Arachni::Issue}.
-                    signatures[pair_hash][altered_hash][:response] ||= res
+                # Store the mutation for the {Arachni::Issue}.
+                @signatures[pair_hash][altered_hash][:mutation] ||= elem
 
-                    signatures[pair_hash][altered_hash][:injected_string] ||= expr
+                # Keep the latest response for the {Arachni::Issue}.
+                @signatures[pair_hash][altered_hash][:response] ||= res
 
-                    body = res.body.gsub( elem.seed, '' )
+                @signatures[pair_hash][altered_hash][:injected_string] ||= expr
 
-                    # Create a signature from the response body and refine it with
-                    # subsequent ones to remove noise (like context-irrelevant dynamic
-                    # content such as banners etc.).
-                    signatures[pair_hash][altered_hash][bool] =
-                        signatures[pair_hash][altered_hash][bool] ?
-                            signatures[pair_hash][altered_hash][bool].refine!(body) :
-                            Support::Signature.new(body)
+                @signatures[pair_hash][altered_hash][bool] = signature
 
-                    signature_sieve( altered_hash, signatures, pair_hash )
+                signature_sieve( altered_hash, pair_hash )
 
-                    increase_received_responses( opts, signatures )
-                end
+                increase_received_responses( opts )
             end
         end
     end
 
-    def increase_received_responses( opts, signatures )
+    def increase_received_responses( opts )
         @data_gathering[:received_responses] += 1
-        finalize_if_done( opts, signatures )
+        finalize_if_done( opts )
     end
 
     # Check if we're done with data gathering and proceed to establishing a
     # {#populate_control_verification_signatures verification control baseline}
     # and {#match_signatures final analysis}.
-    def finalize_if_done( opts, signatures )
+    def finalize_if_done( opts )
         return if @data_gathering[:done] ||
             @data_gathering[:expected_responses] != @data_gathering[:received_responses]
         @data_gathering[:done] = true
@@ -314,63 +274,78 @@ module Differential
         # Lastly, we need to re-establish a new baseline in order to compare
         # it with the initial one so as to be sure that server behavior
         # hasn't suddenly changed in a way that would corrupt our analysis.
-        populate_control_verification_signatures( opts, signatures )
+        populate_control_verification_signatures( opts )
     end
 
     # Re-establishes a control baseline at the end of the audit, to make sure
     # that website behavior has remained stable, otherwise its behavior won't
     # be trustworthy.
-    def populate_control_verification_signatures( opts, signatures )
+    def populate_control_verification_signatures( opts )
         received_responses = 0
-        gathered           = {}
 
-        opts[:precision].times do
-            audit( opts[:false], opts ) do |res, elem|
-                altered_hash = elem.affected_input_name.hash
+        gather_signatures( opts[:false], opts ) do |signature, _, elem|
+            altered_hash = elem.affected_input_name.hash
 
-                gathered[altered_hash] ||= 0
-                gathered[altered_hash]  += 1
+            if @signatures[:corrupted][altered_hash]
+                @data_gathering[:received_responses] += 1
+                next
+            end
 
-                response_check( res, signatures, elem )
+            print_status 'Got control verification response ' <<
+                "for #{elem.type} variable '#{elem.affected_input_name}' with" <<
+                " action '#{elem.action}'."
 
-                if signatures[:corrupted][altered_hash]
-                    @data_gathering[:received_responses] += 1
-                    next
-                end
+            @signatures[:controls_verification][altered_hash] = signature
 
-                if gathered[altered_hash] == opts[:precision]
-                    print_status 'Got control verification response ' <<
-                        "for #{elem.type} variable '#{elem.affected_input_name}' with" <<
-                        " action '#{elem.action}'."
-                end
+            received_responses += 1
+            next if received_responses != @data_gathering[:mutations_size]
+
+            # Once the new baseline has been established and we've got all the
+            # data we need, crunch them and see if server behavior indicates
+            # a vulnerability.
+            match_signatures
+        end
+    end
+
+    def gather_signatures( seed, opts, &block )
+        buffer             = {}
+        received_responses = {}
+
+        opts[:precision].times do |i|
+            audit( seed, opts ) do |r, e|
+                altered_hash = e.affected_input_name.hash
 
-                body = res.body.gsub( elem.seed, '' )
+                body = r.body.gsub( e.seed, '' )
 
-                # Create a signature from the response body and refine it with
-                # subsequent ones to remove noise (like context-irrelevant dynamic
-                # content such as banners etc.).
-                signatures[:controls_verification][altered_hash] =
-                    signatures[:controls_verification][altered_hash] ?
-                        signatures[:controls_verification][altered_hash].refine!(body) :
-                        Support::Signature.new(body)
+                buffer[altered_hash] ||= []
+
+                received_responses[altered_hash] ||= 0
+                received_responses[altered_hash]  += 1
+
+                buffer[altered_hash][i] = Support::Signature.new( body )
+
+                response_check( r, e )
+
+                next if received_responses[altered_hash] != opts[:precision]
+
+                refined = buffer[altered_hash].pop
+                buffer[altered_hash].each do |signature|
+                    refined = refined.refine!( signature )
+                end
 
-                received_responses += 1
-                next if received_responses != @data_gathering[:mutations_size]
+                signature_check( refined, e )
 
-                # Once the new baseline has been established and we've got all the
-                # data we need, crunch them and see if server behavior indicates
-                # a vulnerability.
-                match_signatures( signatures )
+                block.call refined, r, e
             end
         end
     end
 
-    def match_signatures( signatures )
-        controls              = signatures.delete( :controls )
-        controls_verification = signatures.delete( :controls_verification )
-        corrupted             = signatures.delete( :corrupted )
+    def match_signatures
+        controls              = @signatures.delete( :controls )
+        controls_verification = @signatures.delete( :controls_verification )
+        corrupted             = @signatures.delete( :corrupted )
 
-        signatures.each do |pair_hash, data|
+        @signatures.each do |pair_hash, data|
             data.each do |input, result|
                 next if !result[:response] || result[:corrupted] || corrupted[input]
 
@@ -388,80 +363,72 @@ module Differential
                 #
                 #   force_false_baseline == false_response_body AND
                 #   false_response_body != true_response_body AND
-                #   force_false_response_code == 200 AND
-                #   true_response_code == 200 AND
-                #   false_response_code == 200
-
-                # Check to see if the `true` response we're analyzing
-                # is a custom 404 page.
-                http.dynamic_404_handler._404?( result[:response] ) do |is_custom_404|
-                    # If this is a custom 404 page bail out.
-                    next if is_custom_404
-
-                    options = result[:mutation].differential_analysis_options
-                    pair    = options[:pairs].find { |pair| pair.hash == pair_hash }
-
-                    issue_data = {
-                        vector:   result[:mutation],
-                        response: result[:response]
+                #   force_false_response_code in DIFFERENTIAL_ALLOWED_STATUS AND
+                #   true_response_code in DIFFERENTIAL_ALLOWED_STATUS AND
+                #   false_response_code in DIFFERENTIAL_ALLOWED_STATUS
+
+                options = result[:mutation].differential_analysis_options
+                pair    = options[:pairs].find { |pair| pair.hash == pair_hash }
+
+                issue_data = {
+                    vector:   result[:mutation],
+                    response: result[:response]
+                }
+
+                if pair
+                    issue_data[:remarks] = {
+                        differential_analysis: [
+                            "True expression: #{pair.keys.first}",
+                            "False expression: #{pair.values.first}",
+                            "Control false expression: #{options[:false]}"
+                        ]
                     }
-
-                    if pair
-                        issue_data[:remarks] = {
-                            :differential_analysis => [
-                                "True expression: #{pair.keys.first}",
-                                "False expression: #{pair.values.first}",
-                                "Control false expression: #{options[:false]}"
-                            ]
-                        }
-                    end
-
-                    @auditor.log( issue_data )
                 end
+
+                @auditor.log( issue_data )
             end
         end
     end
 
-    def response_check( response, signatures, elem, pair = nil )
+    def response_check( response, elem )
         corrupted = false
 
         if !DIFFERENTIAL_ALLOWED_STATUS.include?( response.code )
-            print_status "Server returned status (#{response.code})," <<
+            print_bad "Server returned status (#{response.code})," <<
                 " aborting analysis for #{elem.type} variable " <<
                 "'#{elem.affected_input_name}' with action '#{elem.action}'."
             corrupted = true
         end
 
         if !corrupted && response.partial?
-            print_status "Server returned partial response, aborting analysis " <<
+            print_bad 'Server returned partial response, aborting analysis ' <<
                 "for #{elem.type} variable '#{elem.affected_input_name}' with " <<
                 "action '#{elem.action}'."
             corrupted = true
         end
 
-        if !corrupted && response.body.empty?
-            print_status 'Server returned empty response body,' <<
-                " aborting analysis for #{elem.type} variable " <<
-                "'#{elem.affected_input_name}' with action '#{self.action}'."
-            corrupted = true
-        end
-
         return if !corrupted
 
-        if pair
-            signatures[pair][elem.affected_input_name.hash][:corrupted] = true
-        else
-            signatures[:corrupted][elem.affected_input_name.hash] = true
-        end
+        @signatures[:corrupted][elem.affected_input_name.hash] = true
+    end
+
+    def signature_check( signature, elem )
+        return if !signature.empty?
+
+        print_bad 'Server returned empty response body,' <<
+            " aborting analysis for #{elem.type} variable " <<
+            "'#{elem.affected_input_name}' with action '#{self.action}'."
+
+        @signatures[:corrupted][elem.affected_input_name.hash] = true
     end
 
-    def signature_sieve( input, signatures, pair )
+    def signature_sieve( input, pair )
         gathered  = @data_gathering[pair][input]
-        signature = signatures[pair][input]
+        signature = @signatures[pair][input]
 
         # If data has been corrupted for the given input, remove it.
         if signature[:corrupted]
-            signatures[pair].delete( input )
+            @signatures[pair].delete( input )
             return true
         end
 
@@ -471,9 +438,9 @@ module Differential
         #   * Remove the data if forced-false and boolean-false signatures
         #       don't match.
         if (@data_gathering[:controls][input] && gathered[:false_probes]) &&
-            (signatures[:controls][input] != signature[:false])
+            !@signatures[:controls][input].similar?( signature[:false], 0.1 )
 
-            signatures[pair].delete( input )
+            @signatures[pair].delete( input )
             return true
         end
 
@@ -485,7 +452,7 @@ module Differential
         if (gathered[:false_probes] && gathered[:true_probes]) &&
             signature[:false].similar?( signature[:true], 0.1 )
 
-            signatures[pair].delete( input )
+            @signatures[pair].delete( input )
             return true
         end