arachni 0.4.0.4 → 0.4.1

data/lib/arachni/element/capabilities/auditable/rdiff.rb

@@ -0,0 +1,243 @@
+=begin
+    Copyright 2010-2012 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+module Arachni
+
+require Options.dir['lib'] + 'bloom_filter'
+
+module Element::Capabilities
+
+#
+# Performs boolean, fault injection and behavioral analysis (using the rDiff algorithm)
+# in order to determine whether the web application is responding to the injected data and how.
+#
+# If the behavior can be manipulated by the injected data in ways that it's not supposed to
+# (like when evaluating injected code) then the element is deemed vulnerable.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+module Auditable::RDiff
+
+    def self.included( mod )
+        # the rdiff attack performs it own redundancy checks so we need this to
+        # keep track of audited elements
+        @@rdiff_audited ||= BloomFilter.new
+    end
+
+    RDIFF_OPTIONS =  {
+        # append our seeds to the default values
+        format:    [Mutable::Format::APPEND],
+
+        # allow duplicate requests
+        redundant: true,
+
+        # amount of rdiff iterations
+        precision: 2,
+
+        respect_method: true
+    }
+
+    #
+    # Performs differential analysis and logs an issue should there be one.
+    #
+    #    opts = {
+    #        :precision => 3,
+    #        :faults    => [ 'fault injections' ],
+    #        :bools     => [ 'boolean injections' ]
+    #    }
+    #
+    #    element.rdiff_analysis( opts )
+    #
+    # Here's how it goes:
+    # * let _default_ be the default/original response
+    # * let _fault_   be the response of the fault injection
+    # * let _bool_    be the response of the boolean injection
+    #
+    # A vulnerability is logged if:
+    #     default == bool AND bool.code == 200 AND fault != bool
+    #
+    # The "bool" response is also checked in order to determine if it's a custom 404, if it is it'll be skipped.
+    #
+    # If a block has been provided analysis and logging will be delegated to it.
+    #
+    # @param    [Hash]      opts        available options:
+    #                                   * :format -- as seen in {Arachni::Element::Mutable::MUTATION_OPTIONS}
+    #                                   * :precision -- amount of rdiff iterations
+    #                                   * :faults -- array of fault injection strings (these are supposed to force erroneous conditions when interpreted)
+    #                                   * :bools -- array of boolean injection strings (these are supposed to not alter the webapp behavior when interpreted)
+    # @param    [Block]     block      block to be used for custom analysis of responses; will be passed the following:
+    #                                   * injected string
+    #                                   * audited element
+    #                                   * default response body
+    #                                   * boolean response
+    #                                   * fault injection response body
+    #
+    def rdiff_analysis( opts = {}, &block )
+        opts = self.class::MUTATION_OPTIONS.merge( RDIFF_OPTIONS.merge( opts ) )
+
+        # don't continue if there's a missing value
+        auditable.values.each { |val| return if !val || val.empty? }
+
+        return if rdiff_audited?
+        rdiff_audited
+
+        responses = {
+            # will hold the original, default, response that results from submitting
+            orig: nil,
+
+            # will hold responses of boolean injections
+            good: {},
+
+            # will hold responses of fault injections
+            bad:  {}
+        }
+
+        # submit the element, as is, opts[:precision] amount of times and
+        # rdiff the responses in order to arrive to a refined response without
+        # any superfluous dynamic content
+        opts[:precision].times {
+            # get the default responses
+            audit( '', opts ) do |res|
+                responses[:orig] ||= res.body
+                # remove context-irrelevant dynamic content like banners and such
+                responses[:orig] = responses[:orig].rdiff( res.body )
+            end
+        }
+
+        # perform fault injection opts[:precision] amount of times and
+        # rdiff the responses in order to arrive to a refined response without
+        # any superfluous dynamic content
+        opts[:precision].times {
+            opts[:faults].each do |str|
+                # get mutations of self using the fault seed, which will
+                # cause an internal/silent error when evaluated
+                mutations( str, opts ).each do |elem|
+                    print_status elem.status_string
+
+                    # submit the mutation and store the response
+                    elem.submit( opts ) do |res|
+                        responses[:bad][elem.altered] ||= res.body.clone
+                        # remove context-irrelevant dynamic content like banners and such
+                        # from the error page
+                        responses[:bad][elem.altered] =
+                            responses[:bad][elem.altered].rdiff( res.body.clone )
+                    end
+                end
+            end
+        }
+
+        # get injection variations that will not affect the outcome of the query
+        opts[:bools].each do |str|
+
+            # get mutations of self using the boolean seed, which will not
+            # alter the execution flow
+            mutations( str, opts ).each do |elem|
+                print_status elem.status_string
+
+                # submit the mutation and store the response
+                elem.submit( opts ) do |res|
+                    responses[:good][elem.altered] ||= []
+                    # save the response and some data for analysis
+                    responses[:good][elem.altered] << {
+                        'str'  => str,
+                        'res'  => res,
+                        'elem' => elem
+                    }
+                end
+            end
+        end
+
+        # when this runs the "responses" hash will have been populated and we
+        # can continue with analysis
+        http.after_run {
+
+            responses[:good].keys.each do |key|
+                responses[:good][key].each do |res|
+
+                    # if there's a block passed then delegate analysis to it
+                    if block
+                        exception_jail( false ){
+                            block.call( res['str'], res['elem'], responses[:orig],
+                                        res['res'], responses[:bad][key] )
+                        }
+
+                    # if default_response_body == bool_response_body AND
+                    #    bool_response_code == 200 AND
+                    #    fault_response_body != bool_response_body
+                    elsif responses[:orig] == res['res'].body &&
+                            responses[:bad][key] != res['res'].body &&
+                            res['res'].code == 200
+
+                        # check to see if the current boolean response we're analyzing
+                        # is a custom 404 page
+                        http.custom_404?( res['res'] ) do |bool|
+                            # if this is a custom 404 page bail out
+                            next if bool
+
+                            # if this isn't a custom 404 page then it means that
+                            # the element is vulnerable, so go ahead and log the issue
+
+                            url = res['res'].effective_url
+
+                            # information for the Metareport report
+                            opts = {
+                                injected_orig: res['str'],
+                                combo:         res['elem'].auditable
+                            }
+
+                            @auditor.log_issue(
+                                var:          key,
+                                url:          url,
+                                method:       res['res'].request.method.to_s,
+                                opts:         opts,
+                                injected:     res['str'],
+                                id:           res['str'],
+                                regexp:       'n/a',
+                                regexp_match: 'n/a',
+                                elem:         res['elem'].type,
+                                response:     res['res'].body,
+                                # :verification => true,
+                                headers:      {
+                                    request:  res['res'].request.headers,
+                                    response: res['res'].headers,
+                                }
+                            )
+
+                            print_ok "In #{res['elem'].type} var '#{key}' ( #{url} )"
+                        end
+                    end
+
+                end
+            end
+        }
+    end
+
+    private
+    def rdiff_audited
+        @@rdiff_audited << rdiff_audit_id
+    end
+
+    def rdiff_audited?
+        @@rdiff_audited.include?( rdiff_audit_id )
+    end
+
+    def rdiff_audit_id
+        @action + @auditable.keys.to_s
+    end
+
+end
+end
+end

data/lib/arachni/element/capabilities/auditable/taint.rb

@@ -0,0 +1,141 @@
+=begin
+    Copyright 2010-2012 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+module Arachni::Element::Capabilities
+
+#
+# Looks for specific substrings or patterns in response bodies.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+module Auditable::Taint
+
+    TAINT_OPTIONS = {
+        #
+        # The regular expression to match against the response body.
+        #
+        # Alternatively, you can use the :substring option.
+        #
+        regexp:    nil,
+
+        #
+        # Verify the matched string with this value when using a regexp.
+        #
+        match:     nil,
+
+        #
+        # The substring to look for the response body.
+        #
+        # Alternatively, you can use the :regexp option.
+        #
+        substring: nil,
+
+        #
+        # Array of patterns to ignore.
+        #
+        # Useful when needing to narrow down what to log without
+        # having to construct overly complex match regexps.
+        #
+        ignore:    nil
+    }
+
+    #
+    # Performs taint analysis and logs an issue should there be one.
+    #
+    # It logs an issue when:
+    # * _:match_ == nil AND _:regexp_ matches the response body
+    # * _:match_ == not nil AND  _:regexp_ match == _:match_
+    # * _:substring_ exists in the response body
+    #
+    # @param  [String]  seed      the string to be injected
+    # @param  [Hash]    opts      options as described in {Arachni::Module::Auditor::OPTIONS} and {TAINT_OPTIONS}
+    #
+    def taint_analysis( seed, opts = { } )
+        opts = self.class::OPTIONS.merge( TAINT_OPTIONS.merge( opts ) )
+        opts[:substring] = seed if !opts[:regexp] && !opts[:substring]
+        audit( seed, opts ) { |res, c_opts| get_matches( res, c_opts ) }
+    end
+
+    private
+
+    #
+    # Tries to identify an issue through pattern matching.
+    #
+    # If a issue is found a message will be printed and the issue will be logged.
+    #
+    # @param  [Typhoeus::Response]  res
+    # @param  [Hash]  opts
+    #
+    def get_matches( res, opts )
+        [opts[:regexp]].flatten.compact.each { |regexp| match_regexp_and_log( regexp, res, opts ) }
+        [opts[:substring]].flatten.compact.each { |substring| match_substring_and_log( substring, res, opts ) }
+    end
+
+    def match_substring_and_log( substring, res, opts )
+        opts[:verification] = false
+
+        # an annoying encoding exception may be thrown by scan()
+        # the sob started occurring again....
+        begin
+            opts[:verification] = true if @auditor.page.body.substring?( substring )
+        rescue
+        end
+
+        if res.body.substring?( substring ) && !ignore?( res, opts )
+            opts[:regexp] = opts[:id] = opts[:match]  = substring.clone
+            @auditor.log( opts, res )
+        end
+    end
+
+    def match_regexp_and_log( regexp, res, opts )
+        regexp = regexp.is_a?( Regexp ) ? regexp :
+            Regexp.new( regexp.to_s, Regexp::IGNORECASE )
+
+        match_data = res.body.scan( regexp )[0]
+        match_data = match_data.to_s
+
+        opts[:verification] = false
+
+        # an annoying encoding exception may be thrown by scan()
+        # the sob started occuring again....
+        begin
+            opts[:verification] = true if @auditor.page.body.scan( regexp )[0]
+        rescue
+        end
+
+        # fairly obscure condition...pardon me...
+        if ( opts[:match] && match_data == opts[:match] ) ||
+           ( !opts[:match] && match_data && match_data.size > 0 )
+
+            return if ignore?( res, opts )
+
+            opts[:id] = opts[:match]  = opts[:match] ? opts[:match] : match_data
+            opts[:regexp] = regexp
+
+            @auditor.log( opts, res )
+        end
+    end
+
+    def ignore?( res, opts )
+        [opts[:ignore]].flatten.compact.each do |r|
+            r = r.is_a?( Regexp ) ? r : Regexp.new( r.to_s, Regexp::IGNORECASE )
+            return true if res.body.scan( r ).flatten.first
+        end
+        false
+    end
+
+end
+end

data/lib/arachni/element/capabilities/auditable/timeout.rb

@@ -0,0 +1,330 @@
+=begin
+    Copyright 2010-2012 Tasos Laskos <tasos.laskos@gmail.com>
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+=end
+
+require 'set'
+
+module Arachni::Element::Capabilities
+
+#
+# Evaluates whether or not the injection of specific data affects the response
+# time of the web application.
+#
+# It takes into account unstable network conditions and server-side failures and
+# verifies the results before logging.
+#
+# == Methodology
+#
+# Here's how it works:
+# * Loop 1 ({#timeout_analysis}) -- Populates the candidate queue. We're picking the low hanging
+#   fruit here so we can run this in larger concurrent bursts which cause *lots* of noise.
+#   - Initial probing for candidates -- If element times out it is added to a queue.
+#   - Stabilization ({#responsive?}) -- The element is submitted with its default values in
+#     order to wait until the effects of the timing attack have worn off.
+# * Loop 2 ({timeout_analysis_phase_2}) -- Verifies the candidates. This is much more delicate so the
+#   concurrent requests are lowered to pairs.
+#   - Liveness test -- Ensures that the webapp is alive and not just timing-out by default
+#   - Verification using an increased timeout delay -- Any elements that time out again are logged.
+#   - Stabilization ({#responsive?})
+#
+# Ideally, all requests involved with timing attacks would be run in sync mode
+# but the performance penalties are too high, thus we compromise and make the best of it
+# by running as little an amount of concurrent requests as possible for any given phase.
+#
+# == Usage
+#
+# Call {#timeout_analysis} to schedule a timeout audit and execute {timeout_audit_run}
+# to run the scheduled operations.
+#
+# This deviates from the normal framework structure because it is preferable
+# to run timeout audits separately in order to avoid interference by other
+# audit operations.
+#
+# If you want to be notified every time a timeout audit is performed you can pass
+# callback block to {on_timing_attacks}.
+#
+# @author Tasos "Zapotek" Laskos <tasos.laskos@gmail.com>
+#
+module Auditable::Timeout
+
+    def self.included( mod )
+        @@parent = mod
+
+        #
+        # Returns the names of all loaded modules that use timing attacks.
+        #
+        # @return   [Set]
+        #
+        def @@parent.timeout_loaded_modules
+            @@timeout_loaded_modules
+        end
+
+        #
+        # Holds timing-attack performing Procs to be run after all
+        # non-timing-attack modules have finished.
+        #
+        # @return   [Queue]
+        #
+        def @@parent.timeout_audit_blocks
+            @@timeout_audit_blocks
+        end
+
+        #
+        # @return   [Integer]    amount of timeout-audit related operations
+        #                           (audit blocks + candidate elements)
+        #
+        def @@parent.current_timeout_audit_operations_cnt
+            @@timeout_audit_blocks.size + @@timeout_candidates.size
+        end
+
+        def @@parent.add_timeout_audit_block( &block )
+            @@timeout_audit_operations_cnt += 1
+            @@timeout_audit_blocks << block
+        end
+
+        def @@parent.add_timeout_candidate( elem )
+            @@timeout_audit_operations_cnt += 1
+            @@timeout_candidates << elem
+        end
+
+        #
+        # @return   [Bool]  true if timeout attacks are currently running
+        #
+        def @@parent.running_timeout_attacks?
+            @@running_timeout_attacks
+        end
+
+        #
+        # Adds a block to be executed every time a timing attack is performed
+        #
+        def @@parent.on_timing_attacks( &block )
+            @@on_timing_attacks << block
+        end
+
+        #
+        # @return   [Integer]    amount of timeout-audit operations
+        #
+        def @@parent.timeout_audit_operations_cnt
+            @@timeout_audit_operations_cnt
+        end
+
+        def @@parent.call_on_timing_blocks( res, elem )
+            @@on_timing_attacks.each { |block| block.call( res, elem ) }
+        end
+
+        #
+        # Runs all blocks in {timeout_audit_blocks} and verifies
+        # and logs the candidate elements.
+        #
+        def @@parent.timeout_audit_run
+            @@running_timeout_attacks = true
+
+            while !@@timeout_audit_blocks.empty?
+                @@timeout_audit_blocks.pop.call
+            end
+
+            while !@@timeout_candidates.empty?
+                self.timeout_analysis_phase_2( @@timeout_candidates.pop )
+            end
+        end
+
+        #
+        # (Called by {timeout_audit_run}, do *NOT* call manually.)
+        #
+        # Runs phase 2 of the timing attack auditing an individual element
+        # (which passed phase 1) with a higher delay and timeout.
+        #
+        # * Liveness check: Element is submitted as is to make sure that the page is alive and responsive
+        #   * If liveness check fails then phase 2 is aborted
+        #   * If liveness check succeeds it progresses to verification
+        # * Verification: Element is submitted with an increased delay to verify the vulnerability
+        #   * If verification fails it aborts
+        #   * If verification succeeds the issue is logged
+        # * Stabilize responsiveness: Wait for the effects of the timing attack to wear off
+        #
+        def @@parent.timeout_analysis_phase_2( elem )
+
+            # reset the audited list since we're going to re-audit the elements
+            # @@timeout_audited = Set.new
+
+            opts = elem.opts
+            opts[:timeout] *= 2
+            # opts[:async]    = false
+            # self.audit_timeout_debug_msg( 2, opts[:timeout] )
+
+            str = opts[:timing_string].gsub( '__TIME__',
+                ( opts[:timeout] / opts[:timeout_divider] ).to_s )
+
+            opts[:timeout] *= 0.7
+
+            elem.auditable = elem.orig
+
+            # this is the control; request the URL of the element to make sure
+            # that the web page is alive i.e won't time-out by default
+            elem.submit do |res|
+                self.call_on_timing_blocks( res, elem )
+
+                if res.timed_out?
+                    elem.print_info 'Liveness check failed, bailing out...'
+                    next
+                end
+
+                elem.print_info 'Liveness check was successful, progressing to verification...'
+                elem.audit( str, opts ) do |c_res, c_opts|
+                    if !c_res.timed_out?
+                        elem.print_info 'Verification failed.'
+                        next
+                    end
+
+                    # all issues logged by timing attacks need manual verification.
+                    # end of story.
+                    # c_opts[:verification] = true
+                    elem.auditor.log( c_opts, c_res )
+                    elem.responsive?
+                end
+
+            end
+
+            elem.http.run
+        end
+
+        def call_on_timing_blocks( res, elem )
+            @@parent.call_on_timing_blocks( res, elem )
+        end
+
+        # holds timing-attack performing Procs to be run after all
+        # non-timing-attack modules have finished.
+        @@timeout_audit_blocks   ||= Queue.new
+
+        @@timeout_audit_operations_cnt ||= 0
+
+        # populated by timing attack phase 1 with
+        # candidate elements to be verified by phase 2
+        @@timeout_candidates     ||= Queue.new
+
+        # modules which have called the timing attack audit method (audit_timeout)
+        # we're interested in the amount, not the names, and is used to
+        # determine scan progress
+        @@timeout_loaded_modules ||= Set.new
+
+        @@on_timing_attacks      ||= []
+
+        @@running_timeout_attacks ||= false
+    end
+
+    #
+    # Performs timeout/time-delay analysis and logs an issue should there be one.
+    #
+    # @param   [Array]     strings     injection strings
+    #                                       __TIME__ will be substituted with (timeout / timeout_divider)
+    # @param   [Hash]      opts        options as described in {Arachni::Element::Mutable::OPTIONS} with the following extra:
+    #                                   * :timeout -- milliseconds to wait for the request to complete
+    #                                   * :timeout_divider -- __TIME__ = timeout / timeout_divider
+    #
+    def timeout_analysis( strings, opts )
+        @@timeout_loaded_modules << @auditor.fancy_name
+
+        @@parent.add_timeout_audit_block {
+            delay = opts[:timeout]
+
+            audit_timeout_debug_msg( 1, delay )
+            timing_attack( strings, opts ) do |_, _, elem|
+                elem.auditor = @auditor
+
+                print_info "Found a candidate -- #{elem.type.capitalize} input '#{elem.altered}' at #{elem.action}"
+
+                @@parent.add_timeout_candidate( elem ) if elem.responsive?
+            end
+        }
+    end
+
+    #
+    # Submits self with a high timeout value and blocks until it gets a response.
+    #
+    # That is to make sure that responsiveness has been restored before progressing further.
+    #
+    # @param    [Float] limit   how much time to afford the server to respond
+    #
+    # @return   [Bool] +true+ if server responds within the given time limit, +false+ otherwise
+    #
+    def responsive?( limit = 120.0 )
+        d_opts = {
+            skip_orig: true,
+            redundant: true,
+            timeout:   limit * 1000,
+            silent:    true,
+            async:     false
+        }
+
+        orig_opts = opts
+
+        print_info 'Waiting for the effects of the timing attack to wear off.'
+        print_info "Max waiting time: #{limit} seconds."
+
+        @auditable = @orig
+        res = submit( d_opts ).response
+
+        @opts.merge!( orig_opts )
+
+        if !res.timed_out?
+            print_info 'Server seems responsive again.'
+        else
+            print_error 'Max waiting time exceeded, the server may be dead.'
+            return false
+        end
+
+        true
+    end
+
+    private
+    def audit_timeout_debug_msg( phase, delay )
+        print_debug '---------------------------------------------'
+        print_debug "Running phase #{phase.to_s} of timing attack."
+        print_debug "Delay set to: #{delay.to_s} milliseconds"
+        print_debug '---------------------------------------------'
+    end
+
+    #
+    # Audits elements using a timing attack.
+    #
+    # 'opts' needs to contain a :timeout value in milliseconds.</br>
+    # Optionally, you can add a :timeout_divider.
+    #
+    # @param   [Array]     strings     injection strings
+    #                                       '__TIME__' will be substituted with (timeout / timeout_divider)
+    # @param    [Hash]      opts       options as described in {Arachni::Element::Mutable::OPTIONS}
+    # @param    [Block]     block      block to call if a timeout occurs,
+    #                                       it will be passed the response and opts
+    #
+    def timing_attack( strings, opts, &block )
+        opts[:timeout_divider] ||= 1
+
+        [strings].flatten.each do |str|
+
+            opts[:timing_string] = str
+            str = str.gsub( '__TIME__', ( opts[:timeout] / opts[:timeout_divider] ).to_s )
+            opts[:skip_orig] = true
+
+            audit( str, opts ) do |res, c_opts, elem|
+                call_on_timing_blocks( res, elem )
+                block.call( res, c_opts, elem ) if block && res.timed_out?
+            end
+        end
+
+        @auditor.http.run
+    end
+
+end
+end