arachni 1.3.2 → 1.4

data/lib/arachni/element/capabilities/analyzable/differential.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -36,8 +36,9 @@ module Differential
         # content -- like banners etc.
         precision:      2,
 
-        # Override global fuzzing settings and only use the default method of the
-        # element under audit.
+        # Override global fuzzing settings and only use the default method of
+        # the element under audit.
+        with_raw_payloads:      false,
         with_both_http_methods: false,
         parameter_names:        false,
         with_extra_parameter:   false,
@@ -59,6 +60,8 @@ module Differential
         false:          '-1'
     }
 
+    DIFFERENTIAL_ALLOWED_STATUS = Set.new([200, 404])
+
     attr_accessor :differential_analysis_options
 
     # Performs differential analysis and logs an issue should there be one.
@@ -422,13 +425,20 @@ module Differential
     def response_check( response, signatures, elem, pair = nil )
         corrupted = false
 
-        if response.code != 200
-            print_status "Server returned non 200 status (#{response.code})," <<
+        if !DIFFERENTIAL_ALLOWED_STATUS.include?( response.code )
+            print_status "Server returned status (#{response.code})," <<
                 " aborting analysis for #{elem.type} variable " <<
                 "'#{elem.affected_input_name}' with action '#{elem.action}'."
             corrupted = true
         end
 
+        if !corrupted && response.partial?
+            print_status "Server returned partial response, aborting analysis " <<
+                "for #{elem.type} variable '#{elem.affected_input_name}' with " <<
+                "action '#{elem.action}'."
+            corrupted = true
+        end
+
         if !corrupted && response.body.empty?
             print_status 'Server returned empty response body,' <<
                 " aborting analysis for #{elem.type} variable " <<

data/lib/arachni/element/capabilities/analyzable/signature.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -20,30 +20,73 @@ module Signature
     }
 
     SIGNATURE_OPTIONS = {
-        # The regular expression to match against the response body.
-        #
-        # Alternatively, you can use the :substring option.
-        regexp:    nil,
+        # The signatures to look for the response body, if `Regexp` it will be
+        # matched against it, if `String` it'll be used as a needle.
+        signatures: [],
 
-        # The substring to look for the response body.
+        # Array of signatures to ignore.
         #
-        # Alternatively, you can use the :regexp option.
-        substring: nil,
+        # Useful when needing to narrow down what to log without having to
+        # construct overly complex signatures.
+        ignore:     []
+    }
 
-        # Array of patterns to ignore.
-        #
-        # Useful when needing to narrow down what to log without
-        # having to construct overly complex match regexps.
-        ignore:    nil,
+    FILE_SIGNATURES = {
+        'environ'  => proc do |response|
+            next if !response.body.include?( 'DOCUMENT_ROOT=' )
+            /DOCUMENT_ROOT=.*HTTP_USER_AGENT=/m
+        end,
+        'passwd'   => proc do |response|
+            next if !response.body.include?( 'bin/' )
+            /:.+:\d+:\d+:.+:[0-9a-zA-Z\/]+/
+        end,
+        'boot.ini' => proc do |response|
+            next if !response.body.include?( '[boot loader]' )
+            /\[boot loader\].*\[operating systems\]/m
+        end,
+        'win.ini'  => proc do |response|
+            next if !response.body.include?( '[fonts]' )
+            /\[fonts\].*\[extensions\]/m
+        end,
+        'web.xml'  => '<web-app'
+    }
 
-        # Extract the longest word from each regexp and only proceed to the
-        # full match only if that word is included in the response body.
-        #
-        # The check is case insensitive.
-        longest_word_optimization: false
+    FILE_SIGNATURES_PER_PLATFORM = {
+        unix:   [
+            FILE_SIGNATURES['environ'],
+            FILE_SIGNATURES['passwd']
+        ],
+        windows: [
+            FILE_SIGNATURES['boot.ini'],
+            FILE_SIGNATURES['win.ini']
+        ],
+        java:    [
+            FILE_SIGNATURES['web.xml']
+        ]
     }
 
-    # Performs signatures analysis and logs an issue should there be one.
+    SOURCE_CODE_SIGNATURES_PER_PLATFORM = {
+        php:  [
+            '<?php'
+        ],
+
+        # No need to optimize the following with procs, OR'ed Regexps perform
+        # better than multiple substring checks, so long as the Regexp parts are
+        # fairly simple.
+
+        java: [
+            /<%|<%=|<%@\s+page|<%@\s+include|<%--|import\s+javax.servlet|
+                import\s+java.io|import=['"]java.io|request\.getParameterValues\(|
+                response\.setHeader|response\.setIntHeader\(/m
+        ],
+        asp:  [
+            /<%|Response\.Write|Request\.Form|Request\.QueryString|
+                Response\.Flush|Session\.SessionID|Session\.Timeout|
+                Server\.CreateObject|Server\.MapPath/m
+        ]
+    }
+
+    # Performs signatures analysis and logs an issue, should there be one.
     #
     # It logs an issue when:
     #
@@ -96,43 +139,44 @@ module Signature
     def get_matches( response )
         vector = response.request.performer
         opts   = vector.audit_options.dup
-        opts[:substring] = vector.seed if !opts[:regexp] && !opts[:substring]
 
-        match_patterns( opts[:regexp], method( :match_regexp_and_log ), response, opts.dup )
-        match_patterns( opts[:substring], method( :match_substring_and_log ), response, opts.dup )
+        if !opts[:signatures].is_a?( Array ) && !opts[:signatures].is_a?( Hash )
+            opts[:signatures] = [opts[:signatures]]
+        end
+
+        opts[:signatures] = [vector.seed] if opts[:signatures].empty?
+
+        find_signatures( opts[:signatures], response, opts.dup )
     end
+    public :get_matches
 
-    def match_patterns( patterns, matcher, response, opts )
-        k = [patterns, response.body]
+    def find_signatures( signatures, response, opts )
+        k = [signatures, response.body]
         return if SIGNATURE_CACHE[:match][k] == false
 
-        if opts[:longest_word_optimization]
-            opts[:downcased_body] = response.body.downcase
-        end
-
-        case patterns
+        case signatures
             when Regexp, String, Array
-                [patterns].flatten.compact.each do |pattern|
-                    res = matcher.call( pattern, response, opts )
+                [signatures].flatten.compact.each do |signature|
+                    res = find_signature( signature, response, opts )
                     SIGNATURE_CACHE[:match][k] ||= !!res
                 end
 
             when Hash
-                if opts[:platform] && patterns[opts[:platform]]
-                    [patterns[opts[:platform]]].flatten.compact.each do |p|
-                        [p].flatten.compact.each do |pattern|
-                            res = matcher.call( pattern, response, opts )
+                if opts[:platform] && signatures[opts[:platform]]
+                    [signatures[opts[:platform]]].flatten.compact.each do |p|
+                        [p].flatten.compact.each do |signature|
+                            res = find_signature( signature, response, opts )
                             SIGNATURE_CACHE[:match][k] ||= !!res
                         end
                     end
 
                 else
-                    patterns.each do |platform, p|
+                    signatures.each do |platform, p|
                         dopts = opts.dup
                         dopts[:platform] = platform
 
-                        [p].flatten.compact.each do |pattern|
-                            res = matcher.call( pattern, response, dopts )
+                        [p].flatten.compact.each do |signature|
+                            res = find_signature( signature, response, dopts )
                             SIGNATURE_CACHE[:match][k] ||= !!res
                         end
                     end
@@ -140,22 +184,36 @@ module Signature
 
                 return if !opts[:payload_platforms]
 
-                # Find out if there are any patterns without associated payloads
+                # Find out if there are any signatures without associated payloads
                 # and match them against every payload's response.
-                patterns.select { |p, _|  !opts[:payload_platforms].include?( p ) }.
+                signatures.select { |p, _|  !opts[:payload_platforms].include?( p ) }.
                     each do |platform, p|
                         dopts = opts.dup
                         dopts[:platform] = platform
 
-                        [p].flatten.compact.each do |pattern|
-                            res = matcher.call( pattern, response, dopts )
+                        [p].flatten.compact.each do |signature|
+                            res = find_signature( signature, response, dopts )
                             SIGNATURE_CACHE[:match][k] ||= !!res
                         end
                     end
         end
     end
 
-    def match_substring_and_log( substring, response, opts )
+    def find_signature( signature, response, opts )
+        if signature.respond_to?( :call )
+            signature = signature.call( response )
+        end
+
+        return if !signature
+
+        if signature.is_a? Regexp
+            match_regexp_and_log( signature, response, opts )
+        else
+            find_substring_and_log( signature, response, opts )
+        end
+    end
+
+    def find_substring_and_log( substring, response, opts )
         return if substring.to_s.empty?
 
         k = [substring, response.body]
@@ -164,14 +222,13 @@ module Signature
         SIGNATURE_CACHE[:match][k] = includes = response.body.include?( substring )
         return if !includes || ignore?( response, opts )
 
-        @candidate_issues << {
+        control_and_log(
             response:  response,
             platform:  opts[:platform],
             proof:     substring,
             signature: substring,
             vector:    response.request.performer
-        }
-        setup_verification_callbacks
+        )
 
         true
     end
@@ -180,13 +237,6 @@ module Signature
         k = [regexp, response.body]
         return if SIGNATURE_CACHE[:match][k] == false
 
-        regexp = regexp.is_a?( Regexp ) ? regexp :
-            Regexp.new( regexp.to_s, Regexp::IGNORECASE )
-
-        if opts[:downcased_body]
-            return if !opts[:downcased_body].include?( longest_word_for_regexp( regexp ) )
-        end
-
         match_data = response.body.match( regexp )
         return if !match_data
 
@@ -196,58 +246,66 @@ module Signature
 
         return if match_data.empty? || ignore?( response, opts )
 
-        @candidate_issues << {
+        control_and_log(
             response:  response,
             platform:  opts[:platform],
             proof:     match_data,
             signature: regexp,
             vector:    response.request.performer
-        }
-        setup_verification_callbacks
+        )
 
         true
     end
 
-    def ignore?( res, opts )
-        [opts[:ignore]].flatten.compact.each do |r|
-            r = r.is_a?( Regexp ) ? r : Regexp.new( r.to_s, Regexp::IGNORECASE )
-            return true if res.body.scan( r ).flatten.first
+    def ignore?( response, opts )
+        [opts[:ignore]].flatten.compact.each do |signature|
+            return true if signature_match?( signature, response )
         end
+
         false
     end
 
-    def setup_verification_callbacks
-        return if @setup_verification_callbacks
-        @setup_verification_callbacks = true
-
-        # Go over the issues to ensure that the signature that identified them
-        # does not match by default.
-        http.after_run do
-            @setup_verification_callbacks = false
-            next if @candidate_issues.empty?
-
-            # Grab the default response.
-            submit do |response|
-                # Something has gone wrong, timed-out request or closed connection.
-                # If we can't verify the issue bail out...
-                next if response.code == 0
-
-                while (issue = @candidate_issues.pop)
-                    # If the body of the control response matches the proof
-                    # of the current issue don't bother, it'll be a coincidence
-                    # causing a false positive.
-                    next if response.body.include?( issue[:proof] )
-
-                    @auditor.log( issue )
-                end
-            end
+    def signature_match?( signature, response )
+        if signature.respond_to?( :call )
+            signature = signature.call( response )
+        end
+
+        return if !signature
+
+        if signature.is_a? Regexp
+            response.body =~ signature
+        else
+            response.body.include?( signature )
         end
     end
 
-    def longest_word_for_regexp( regexp )
-        @@longest_word_for_regex ||= {}
-        @@longest_word_for_regex[regexp.source.hash] ||=
-            regexp.source.longest_word.downcase
+    def control_and_log( issue )
+        control = issue[:vector].dup
+
+        if control.parameter_name_audit?
+            inputs = control.inputs.dup
+            value  = inputs.delete( control.seed )
+
+            control.inputs = inputs.merge( Utilities.random_seed => value )
+        else
+            control.affected_input_value = Utilities.random_seed
+        end
+
+        control.submit do |response|
+            # Something has gone wrong, timed-out request or closed connection.
+            # If we can't verify the issue bail out...
+            next if response.code == 0
+
+            # If the signature matches the control response don't bother, it'll
+            # be a coincidence causing a false positive.
+            next if signature_match?( issue[:signature], response )
+
+            # We can't have procs in there, we only log stuff that
+            # can be serialized.
+            issue[:vector].audit_options.delete :signatures
+
+            @auditor.log( issue )
+        end
     end
 
 end

data/lib/arachni/element/capabilities/analyzable/timeout.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -86,6 +86,9 @@ module Timeout
             @candidates_phase_3    = []
             @phase_3_candidate_ids = Support::LookUp::HashSet.new( hasher: :timeout_id )
 
+            @candidates_phase_4    = []
+            @phase_4_candidate_ids = Support::LookUp::HashSet.new( hasher: :timeout_id )
+
             @logged = Support::LookUp::HashSet.new( hasher: :timeout_id )
 
             deduplicate
@@ -124,19 +127,16 @@ module Timeout
             end
 
             while (candidate = @candidates_phase_3.pop)
-
-                # We've allowed multiple variations of the same element in
-                # previous operations because, during the audit, the payload
-                # that hit could have made the server unresponsive and fooled us
-                # into thinking that other valid variations exist too.
-                #
-                # That's why Phase 3 is here, to shift through these possible
-                # issues and verify them once again, however, if a variation
-                # is logged, it's game over for that input vector.
                 next if Timeout.deduplicate? && logged?( candidate )
 
                 analysis_phase_3( candidate )
             end
+
+            while (candidate = @candidates_phase_4.pop)
+                next if Timeout.deduplicate? && logged?( candidate )
+
+                analysis_phase_4( candidate )
+            end
         end
 
         def payload_delay_from_options( options )
@@ -177,6 +177,26 @@ module Timeout
             elem.print_status "Phase 3 for #{elem.type} input " <<
                 "'#{elem.affected_input_name}' with action #{elem.action}"
 
+            elem.timing_attack_verify( delay ) do
+                elem.print_info '* Verification was successful, candidate can ' <<
+                    'progress to Phase 4.'
+
+                add_phase4_candidate( elem )
+            end
+        end
+
+        # (Called by {.run}, do *NOT* call manually.)
+        #
+        # Runs phase e of the timing attacks, auditing an individual element
+        # (which passed phase 3) with a higher delay and timeout.
+        def analysis_phase_4( elem )
+            # Use the same delay, we don't want to overdo it because the server
+            # might cut us off due to its own restrictions.
+            delay = elem.audit_options[:delay]
+
+            elem.print_status "Phase 4 for #{elem.type} input " <<
+                "'#{elem.affected_input_name}' with action #{elem.action}"
+
             elem.timing_attack_verify( delay ) do |response|
                 # Update the payload stub with a real value, for the user's
                 # sake at this point.
@@ -195,7 +215,7 @@ module Timeout
 
                 controls = elem.timing_attack_remark_data[:control_times]
                 remarks << ('Response times (in seconds) for control requests ' <<
-                    "prior to phases 2 & 3: #{controls.join(', ')}")
+                    "prior to phases 2, 3, 4: #{controls.join(', ')}")
 
                 stabilizations = elem.timing_attack_remark_data[:stabilization_times]
                 remarks << ('Response times (in seconds) for stabilization ' <<
@@ -215,6 +235,11 @@ module Timeout
             @candidates_phase_3    << elem
         end
 
+        def add_phase4_candidate( elem )
+            @phase_4_candidate_ids << elem
+            @candidates_phase_4    << elem
+        end
+
         # @param    [Element::Capabilities::Analyzable::Timeout]    element
         #
         # @return   [Bool]
@@ -313,7 +338,9 @@ module Timeout
             'wear off, this may take a while (max waiting time is ' <<
              "#{options[:timeout] / 1000.0} seconds)."
 
-        if (response = timeout_control.submit( options )).timed_out?
+        response = timeout_control.submit( options )
+
+        if response.timed_out? || response.partial?
             print_bad "#{prepend}Max waiting time exceeded."
             false
         else
@@ -358,11 +385,11 @@ module Timeout
         }
 
         if debug_level_2?
-            print_debug_level_2 "#{__method__}: #{options}"
+            print_debug_level_2 "#{options}"
         end
 
         audit( payloads, options ) do |response, mutation|
-            next if !response.timed_out?
+            next if !response.timed_out? || response.partial?
 
             mutation.timing_attack_remark_data[:delays] << options[:delay]
             block.call( mutation, response )
@@ -420,7 +447,7 @@ module Timeout
             print_verbose "  * Payload:         #{payload.inspect}"
 
             submit( response_max_size: 0, timeout: timeout ) do |response|
-                if !response.timed_out?
+                if !response.timed_out? || response.partial?
                     print_info '* Verification failed.'
                     print_verbose "  * Server responded in #{response.time} seconds."
                     next
@@ -462,7 +489,7 @@ module Timeout
         # tricked by WAFs/IDS/IPS dropping packets.
         self.affected_input_value = seed.sub( '__TIME__', '0' )
         submit( response_max_size: 0, timeout: timeout ) do |control|
-            if control.timed_out?
+            if control.timed_out? || control.partial?
                 print_info '* Control check failed, aborting.'
                 next
             end