arachni 1.2.1 → 1.3

data/lib/arachni/check/auditor.rb

@@ -13,16 +13,14 @@ module Check
 #
 # There are 3 main types of audit and analysis techniques available:
 #
-# * {Arachni::Element::Capabilities::Analyzable::Taint Taint analysis}
+# * {Arachni::Element::Capabilities::Analyzable::Signature Signature analysis}
 #   -- {#audit}
 # * {Arachni::Element::Capabilities::Analyzable::Timeout Timeout analysis}
 #   -- {#audit_timeout}
 # * {Arachni::Element::Capabilities::Analyzable::Differential Differential analysis}
 #   -- {#audit_differential}
 #
-# It should be noted that actual analysis takes place at the element level,
-# and to be more specific, the {Arachni::Element::Capabilities::Auditable}
-# element level.
+# It should be noted that actual analysis takes place at the {Arachni::Element element} level.
 #
 # It also provides:
 #
@@ -119,6 +117,12 @@ module Auditor
                         proc { audit.jsons? && page.jsons.find { |e| e.inputs.any? } },
                     Element::XML               =>
                         proc { audit.xmls? && page.xmls.find { |e| e.inputs.any? } },
+                    Element::UIInput             => false,
+                    Element::UIInput::DOM        =>
+                        proc { audit.ui_inputs? && page.ui_inputs.any? },
+                    Element::UIForm            => false,
+                    Element::UIForm::DOM       =>
+                        proc { audit.ui_forms? && page.ui_forms.any? },
                     Element::Body              => !page.body.empty?,
                     Element::GenericDOM        => page.has_script?,
                     Element::Path              => true,
@@ -189,7 +193,7 @@ module Auditor
     # Auditable DOM elements.
     DOM_ELEMENTS_WITH_INPUTS = [
         Element::Link::DOM, Element::Form::DOM, Element::Cookie::DOM,
-        Element::LinkTemplate::DOM
+        Element::LinkTemplate::DOM, Element::UIInput::DOM, Element::UIForm::DOM
     ]
 
     # Default audit options.
@@ -523,6 +527,12 @@ module Auditor
                 when Element::LinkTemplate::DOM.type
                     prepare_each_dom_element( page.link_templates, &block )
 
+                when Element::UIInput::DOM.type
+                    prepare_each_dom_element( page.ui_inputs, &block )
+
+                when Element::UIForm::DOM.type
+                    prepare_each_dom_element( page.ui_forms, &block )
+
                 else
                     fail ArgumentError, "Unknown DOM element: #{elem}"
             end
@@ -530,17 +540,17 @@ module Auditor
     end
 
     # If a block has been provided it calls {Arachni::Element::Capabilities::Auditable#audit}
-    # for every element, otherwise, it defaults to {#audit_taint}.
+    # for every element, otherwise, it defaults to {#audit_signature}.
     #
     # Uses {#each_candidate_element} to decide which elements to audit.
     #
     # @see OPTIONS
     # @see Arachni::Element::Capabilities::Auditable#audit
-    # @see #audit_taint
+    # @see #audit_signature
     def audit( payloads, opts = {}, &block )
         opts = OPTIONS.merge( opts )
         if !block_given?
-            audit_taint( payloads, opts )
+            audit_signature( payloads, opts )
         else
             each_candidate_element( opts[:elements] ) do |e|
                 e.audit( payloads, opts, &block )
@@ -549,17 +559,17 @@ module Auditor
         end
     end
 
-    # Provides easy access to element auditing using simple taint analysis
+    # Provides easy access to element auditing using simple signature analysis
     # and automatically logs results.
     #
     # Uses {#each_candidate_element} to decide which elements to audit.
     #
     # @see OPTIONS
-    # @see Arachni::Element::Capabilities::Analyzable::Taint
-    def audit_taint( payloads, opts = {} )
+    # @see Arachni::Element::Capabilities::Analyzable::Signature
+    def audit_signature( payloads, opts = {} )
         opts = OPTIONS.merge( opts )
         each_candidate_element( opts[:elements] )do |e|
-            e.taint_analysis( payloads, opts )
+            e.signature_analysis( payloads, opts )
             audited( e.coverage_id )
         end
     end

data/lib/arachni/data/framework.rb

@@ -89,7 +89,19 @@ class Framework
     # @param    [Page]  page
     #   Page with which to update the {#sitemap}.
     def add_page_to_sitemap( page )
-        sitemap[page.dom.url] = page.code
+        update_sitemap( page.dom.url => page.code )
+    end
+
+    def update_sitemap( entries )
+        entries.each do |url, code|
+            # Feedback from the trainer or whatever, don't include it in the
+            # sitemap, it'll just add noise.
+            next if url.include?( Utilities.random_seed )
+
+            @sitemap[url] = code
+        end
+
+        @sitemap
     end
 
     def dump( directory )

data/lib/arachni/data/issues.rb

@@ -34,8 +34,7 @@ class Issues
     def initialize
         super
 
-        # Stores all issues with Issue#digest as the key as a way to deduplicate
-        # and group variations.
+        # Stores all issues with Issue#digest as the key as a way to deduplicate.
         @collection = {}
 
         # We also use this Set for deduplication in case #do_not_store has been
@@ -92,24 +91,11 @@ class Issues
     end
 
     # @return   [Array<Issue>]
-    #   All logged issues grouped as variations.
+    #   All logged issues.
     def all
         @collection.values
     end
 
-    # @return   [Array<Issue>]
-    #   First variation of all issues (as solo issues) sorted by severity.
-    def summary
-        all.map { |issue| issue.variations.first.to_solo issue }.flatten.
-            sort_by(&:severity).reverse
-    end
-
-    # @return   [Array<Issue>]
-    #   All logged issues as solo objects, without variations.
-    def flatten
-        all.map { |issue| issue.variations.map { |v| v.to_solo issue } }.flatten
-    end
-
     def each( &block )
         all.each( &block )
     end
@@ -124,7 +110,7 @@ class Issues
         @digests.include? issue.digest
     end
 
-    # @note Will deduplicate and group issues as variations.
+    # @note Will deduplicate issues.
     #
     # @param    [Issue] issue
     #   Issue to push to the collection.
@@ -133,18 +119,16 @@ class Issues
     def <<( issue )
         notify_on_new_pre_deduplication( issue )
 
-        # Only allow passive issues to have variations.
-        return self if include?( issue ) && issue.active?
+        return self if include?( issue )
 
-        @digests << issue.digest
+        digest = issue.digest
+        @digests << digest
 
         synchronize do
             notify_on_new( issue )
 
             if store?
-                id = issue.digest
-                @collection[id] ||= issue.with_variations
-                @collection[id].variations << issue.as_variation
+                @collection[digest] = issue
             end
         end
 
@@ -165,11 +149,11 @@ class Issues
     end
 
     def first
-        @collection.values.first
+        all.first
     end
 
     def last
-        @collection.values.last
+        all.last
     end
 
     def any?

data/lib/arachni/element/base.rb

@@ -24,9 +24,13 @@ module Capabilities
     end
 end
 
-# load and include all available capabilities
-lib = File.dirname( __FILE__ ) + '/capabilities/*.rb'
-Dir.glob( lib ).each { |f| require f }
+file = File.dirname( __FILE__ )
+# Need to be loaded in order.
+%w(inputtable submittable mutable auditable analyzable).each do |name|
+    require_relative "#{file}/capabilities/#{name}.rb"
+end
+# Load the rest automatically.
+Dir.glob( "#{file}/capabilities/*.rb" ).each { |f| require f }
 
 # Base class for all element types.
 #
@@ -166,6 +170,8 @@ class Base
         data.delete 'scope'
         data['class'] = self.class.to_s
 
+        data['initialization_options'] = initialization_options
+
         if data['initialization_options'].is_a? Hash
             data['initialization_options'] =
                 data['initialization_options'].my_stringify_keys(false)

data/lib/arachni/element/capabilities/analyzable.rb

@@ -6,19 +6,15 @@
     web site for more information on licensing and terms of use.
 =end
 
-require_relative 'auditable'
-
 module Arachni
 module Element::Capabilities
 
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 module Analyzable
-    include Auditable
-
-    # Load and include all available analysis/audit techniques.
+    # Load all available analysis techniques.
     Dir.glob( File.dirname( __FILE__ ) + '/analyzable/*.rb' ).each { |f| require f }
 
-    include Taint
+    include Signature
     include Timeout
     include Differential
 

data/lib/arachni/element/capabilities/analyzable/differential.rb

@@ -6,6 +6,8 @@
     web site for more information on licensing and terms of use.
 =end
 
+require_relative '../mutable'
+
 module Arachni
 module Element::Capabilities
 module Analyzable
@@ -28,7 +30,7 @@ module Differential
     end
 
     DIFFERENTIAL_OPTIONS =  {
-        format:         [Mutable::Format::STRAIGHT],
+        format:         [Arachni::Element::Capabilities::Mutable::Format::STRAIGHT],
 
         # Amount of refinement operations to remove context-irrelevant dynamic
         # content -- like banners etc.
@@ -104,6 +106,12 @@ module Differential
     def differential_analysis( opts = {} )
         return if self.inputs.empty?
 
+        with_missing_values = Set.new( self.inputs.select { |k, v| v.to_s.empty? }.keys )
+        if self.inputs.size == with_missing_values.size
+            print_debug 'Differential analysis: Inputs are missing default values.'
+            return false
+        end
+
         return false if audited? audit_id
         audited audit_id
 
@@ -115,6 +123,9 @@ module Differential
 
         @differential_analysis_options = opts.dup
         opts = self.class::MUTATION_OPTIONS.merge( DIFFERENTIAL_OPTIONS.merge( opts ) )
+        opts[:skip_like] = proc do |mutation|
+            with_missing_values.include? mutation.affected_input_name
+        end
 
         mutations_size = 0
         each_mutation( opts[:false], opts ) { mutations_size += 1 }
@@ -196,13 +207,15 @@ module Differential
                     @data_gathering[:controls][altered_hash] = true
                 end
 
+                body = res.body.gsub( elem.seed, '' )
+
                 # Create a signature from the response body and refine it with
                 # subsequent ones to remove noise (like context-irrelevant dynamic
                 # content such as banners etc.).
                 signatures[:controls][altered_hash] =
                     signatures[:controls][altered_hash] ?
-                        signatures[:controls][altered_hash].refine!(res.body) :
-                        Support::Signature.new(res.body)
+                        signatures[:controls][altered_hash].refine!(body) :
+                        Support::Signature.new(body)
 
                 increase_received_responses( opts, signatures )
             end
@@ -264,13 +277,15 @@ module Differential
 
                     signatures[pair_hash][altered_hash][:injected_string] ||= expr
 
+                    body = res.body.gsub( elem.seed, '' )
+
                     # Create a signature from the response body and refine it with
                     # subsequent ones to remove noise (like context-irrelevant dynamic
                     # content such as banners etc.).
                     signatures[pair_hash][altered_hash][bool] =
                         signatures[pair_hash][altered_hash][bool] ?
-                            signatures[pair_hash][altered_hash][bool].refine!(res.body) :
-                            Support::Signature.new(res.body)
+                            signatures[pair_hash][altered_hash][bool].refine!(body) :
+                            Support::Signature.new(body)
 
                     signature_sieve( altered_hash, signatures, pair_hash )
 
@@ -326,13 +341,15 @@ module Differential
                         " action '#{elem.action}'."
                 end
 
+                body = res.body.gsub( elem.seed, '' )
+
                 # Create a signature from the response body and refine it with
                 # subsequent ones to remove noise (like context-irrelevant dynamic
                 # content such as banners etc.).
                 signatures[:controls_verification][altered_hash] =
                     signatures[:controls_verification][altered_hash] ?
-                        signatures[:controls_verification][altered_hash].refine!(res.body) :
-                        Support::Signature.new(res.body)
+                        signatures[:controls_verification][altered_hash].refine!(body) :
+                        Support::Signature.new(body)
 
                 received_responses += 1
                 next if received_responses != @data_gathering[:mutations_size]

data/lib/arachni/element/capabilities/analyzable/{taint.rb → signature.rb}

@@ -13,13 +13,13 @@ module Analyzable
 # Looks for specific substrings or patterns in response bodies.
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
-module Taint
+module Signature
 
-    TAINT_CACHE = {
+    SIGNATURE_CACHE   = {
         match: Support::Cache::LeastRecentlyPushed.new( 10_000 )
     }
 
-    TAINT_OPTIONS = {
+    SIGNATURE_OPTIONS = {
         # The regular expression to match against the response body.
         #
         # Alternatively, you can use the :substring option.
@@ -43,13 +43,13 @@ module Taint
         longest_word_optimization: false
     }
 
-    # Performs taint analysis and logs an issue should there be one.
+    # Performs signatures analysis and logs an issue should there be one.
     #
     # It logs an issue when:
     #
     # * `:match` == nil AND `:regexp` matches the response body
-    # * `:match`` == not nil AND  `:regexp` match == `:match`
-    # * `:substring`exists in the response body
+    # * `:match` != nil AND  `:regexp` match == `:match`
+    # * `:substring` exists in the response body
     #
     # @param  [String, Array<String>, Hash{Symbol => <String, Array<String>>}]  payloads
     #   Payloads to inject, if given:
@@ -63,16 +63,16 @@ module Taint
     #       for the {Element::Capabilities::Submittable#action resource} to be audited.
     # @param  [Hash]    opts
     #   Options as described in {Arachni::Check::Auditor::OPTIONS} and
-    #   {TAINT_OPTIONS}.
+    #   {SIGNATURE_OPTIONS}.
     #
     # @return   [Bool]
     #   `true` if the audit was scheduled successfully, `false` otherwise (like
     #   if the resource is out of scope).
-    def taint_analysis( payloads, opts = { } )
+    def signature_analysis( payloads, opts = { } )
         return false if self.inputs.empty?
 
         if scope.out?
-            print_debug 'Taint analysis: Element is out of scope,' <<
+            print_debug 'Signature analysis: Element is out of scope,' <<
                             " skipping: #{audit_id}"
             return false
         end
@@ -81,8 +81,8 @@ module Taint
         # we've evaluated our control response.
         @candidate_issues = []
 
-        # Perform the taint analysis.
-        opts = self.class::OPTIONS.merge( TAINT_OPTIONS.merge( opts ) )
+        # Perform the analysis.
+        opts = self.class::OPTIONS.merge( SIGNATURE_OPTIONS.merge( opts ) )
         audit( payloads, opts ) { |response| get_matches( response ) }
     end
 
@@ -104,7 +104,7 @@ module Taint
 
     def match_patterns( patterns, matcher, response, opts )
         k = [patterns, response.body]
-        return if TAINT_CACHE[:match][k] == false
+        return if SIGNATURE_CACHE[:match][k] == false
 
         if opts[:longest_word_optimization]
             opts[:downcased_body] = response.body.downcase
@@ -114,7 +114,7 @@ module Taint
             when Regexp, String, Array
                 [patterns].flatten.compact.each do |pattern|
                     res = matcher.call( pattern, response, opts )
-                    TAINT_CACHE[:match][k] ||= !!res
+                    SIGNATURE_CACHE[:match][k] ||= !!res
                 end
 
             when Hash
@@ -122,7 +122,7 @@ module Taint
                     [patterns[opts[:platform]]].flatten.compact.each do |p|
                         [p].flatten.compact.each do |pattern|
                             res = matcher.call( pattern, response, opts )
-                            TAINT_CACHE[:match][k] ||= !!res
+                            SIGNATURE_CACHE[:match][k] ||= !!res
                         end
                     end
 
@@ -133,7 +133,7 @@ module Taint
 
                         [p].flatten.compact.each do |pattern|
                             res = matcher.call( pattern, response, dopts )
-                            TAINT_CACHE[:match][k] ||= !!res
+                            SIGNATURE_CACHE[:match][k] ||= !!res
                         end
                     end
                 end
@@ -149,7 +149,7 @@ module Taint
 
                         [p].flatten.compact.each do |pattern|
                             res = matcher.call( pattern, response, dopts )
-                            TAINT_CACHE[:match][k] ||= !!res
+                            SIGNATURE_CACHE[:match][k] ||= !!res
                         end
                     end
         end
@@ -159,9 +159,9 @@ module Taint
         return if substring.to_s.empty?
 
         k = [substring, response.body]
-        return if TAINT_CACHE[:match][k] == false
+        return if SIGNATURE_CACHE[:match][k] == false
 
-        TAINT_CACHE[:match][k] = includes = response.body.include?( substring )
+        SIGNATURE_CACHE[:match][k] = includes = response.body.include?( substring )
         return if !includes || ignore?( response, opts )
 
         @candidate_issues << {
@@ -178,7 +178,7 @@ module Taint
 
     def match_regexp_and_log( regexp, response, opts )
         k = [regexp, response.body]
-        return if TAINT_CACHE[:match][k] == false
+        return if SIGNATURE_CACHE[:match][k] == false
 
         regexp = regexp.is_a?( Regexp ) ? regexp :
             Regexp.new( regexp.to_s, Regexp::IGNORECASE )
@@ -192,7 +192,7 @@ module Taint
 
         match_data = match_data[0].to_s
 
-        TAINT_CACHE[:match][k] = !match_data.empty?
+        SIGNATURE_CACHE[:match][k] = !match_data.empty?
 
         return if match_data.empty? || ignore?( response, opts )
 
@@ -220,13 +220,13 @@ module Taint
         return if @setup_verification_callbacks
         @setup_verification_callbacks = true
 
-        # Go over the issues and flag them as untrusted if the pattern that
-        # caused them to be logged matches the untainted response.
+        # Go over the issues to ensure that the signature that identified them
+        # does not match by default.
         http.after_run do
             @setup_verification_callbacks = false
             next if @candidate_issues.empty?
 
-            # Grab an untainted response.
+            # Grab the default response.
             submit do |response|
                 # Something has gone wrong, timed-out request or closed connection.
                 # If we can't verify the issue bail out...