arachni 1.1 → 1.2

data/lib/arachni/element/generic_dom.rb

@@ -74,7 +74,7 @@ class GenericDOM < Base
     def value
         transition.options[:value]
     end
-    alias :affected_input_value value
+    alias :affected_input_value :value
 
     # @return   [Hash]
     def to_h

data/lib/arachni/element/json.rb

@@ -44,7 +44,7 @@ class JSON < Base
         self.inputs = (self.inputs || {}).merge( options[:inputs] || {} )
 
         if @source && self.inputs.empty?
-            self.inputs = JSON.load( self.source )
+            self.inputs = ::JSON.load( self.source )
         end
 
         @default_inputs = self.inputs.dup.freeze
@@ -104,6 +104,7 @@ class JSON < Base
         # @return   [JSON, nil]
         def from_request( url, request )
             return if !request.body.is_a?( String ) || request.body.empty?
+            return if too_big?( request.body )
 
             data =  begin
                 ::JSON.load( request.body )

data/lib/arachni/element/json/capabilities/inputtable.rb

@@ -45,7 +45,7 @@ module Inputtable
     #
     # @return   [Object]
     #
-    # @see  Capabilities::Inputtable#[]
+    # @see  Arachni::Element::Capabilities::Inputtable#[]
     def []( name )
         key, data = find( name )
         data[key]
@@ -65,7 +65,7 @@ module Inputtable
     # @return   [Object]
     #   `value`
     #
-    # @see  Capabilities::Inputtable#[]=
+    # @see  Arachni::Element::Capabilities::Inputtable#[]=
     def []=( name, value )
         @inputs = @inputs.dup
         key, data = find( name )
@@ -80,11 +80,11 @@ module Inputtable
     # Overrides {Capabilities::Inputtable#update} to allow for non-string data
     # of variable depth.
     #
-    # @param    (see Capabilities::Inputtable#update)
-    # @return   (see Capabilities::Inputtable#update)
-    # @raise    (see Capabilities::Inputtable#update)
+    # @param    (see Arachni::Element::Capabilities::Inputtable#update)
+    # @return   (see Arachni::Element::Capabilities::Inputtable#update)
+    # @raise    (see Arachni::Element::Capabilities::Inputtable#update)
     #
-    # @see  Capabilities::Inputtable#update
+    # @see  Arachni::Element::Capabilities::Inputtable#update
     def update( hash )
         traverse_data hash do |path, value|
             self[path] = value

data/lib/arachni/element/json/capabilities/mutable.rb

@@ -26,7 +26,7 @@ module Mutable
     #   If the `name` is an `Array`, it will be treated as a path to the location
     #   of the input.
     #
-    # @see  Capabilities::Mutable#affected_input_name=
+    # @see  Arachni::Element::Capabilities::Mutable#affected_input_name=
     def affected_input_name=( name )
         if name.is_a?( Array ) && name.size == 1
             name = name.first

data/lib/arachni/element/link.rb

@@ -55,19 +55,11 @@ class Link < Base
     def to_s
         uri = uri_parse( self.action ).dup
         uri.query = self.inputs.
-            map { |k, v| "#{encode_query_params(k)}=#{encode_query_params(v)}" }.
+            map { |k, v| "#{encode(k)}=#{encode(v)}" }.
             join( '&' )
         uri.to_s
     end
 
-    # @param   (see .encode_query_params)
-    # @return  (see .encode_query_params)
-    #
-    # @see .encode_query_params
-    def encode_query_params( *args )
-        self.class.encode_query_params( *args )
-    end
-
     # @param   (see .encode)
     # @return  (see .encode)
     #
@@ -114,7 +106,14 @@ class Link < Base
         #
         # @return   [Array<Link>]
         def from_document( url, document )
-            document = Nokogiri::HTML( document.to_s ) if !document.is_a?( Nokogiri::HTML::Document )
+            if !document.is_a?( Nokogiri::HTML::Document )
+                document = document.to_s
+
+                return [] if !(document =~ /\?.*=/)
+
+                document = Nokogiri::HTML( document )
+            end
+
             base_url =  begin
                 document.search( '//base[@href]' )[0]['href']
             rescue
@@ -122,6 +121,8 @@ class Link < Base
             end
 
             document.search( '//a' ).map do |link|
+                next if too_big?( link['href'] )
+
                 href = to_absolute( link['href'], base_url )
                 next if !href
 
@@ -137,12 +138,8 @@ class Link < Base
             end.compact
         end
 
-        def encode_query_params( param )
-            encode( encode( param.recode ), '=' )
-        end
-
-        def encode( *args )
-            ::URI.encode( *args )
+        def encode( string )
+            Arachni::HTTP::Request.encode string
         end
 
         def decode( *args )

data/lib/arachni/element/link/dom.rb

@@ -57,7 +57,7 @@ class DOM < Base
     #   URL including the DOM {#inputs}.
     def to_s
         "#{@action}##{fragment_path}?" << inputs.
-            map { |k, v| "#{encode_query_params(k)}=#{encode_query_params(v)}" }.
+            map { |k, v| "#{encode(k)}=#{encode(v)}" }.
             join( '&' )
     end
 
@@ -65,22 +65,9 @@ class DOM < Base
         "#{@action}##{fragment}"
     end
 
-    def encode_query_params( *args )
-        Link.encode_query_params( *args )
-    end
-
-    def encode( *args )
-        Link.encode( *args )
-    end
-
-    def decode( *args )
-        Link.decode( *args )
-    end
-
     def type
         self.class.type
     end
-
     def self.type
         :link_dom
     end

data/lib/arachni/element/link_template.rb

@@ -170,6 +170,7 @@ class LinkTemplate < Base
             end
 
             document.search( '//a' ).map do |link|
+                next if too_big?( link['href'] )
                 next if !(href = to_absolute( link['href'], base_url ))
 
                 template, inputs = extract_inputs( href, templates )
@@ -212,11 +213,11 @@ class LinkTemplate < Base
         end
 
         def encode( string )
-            URI.encode( URI.encode( URI.encode( string.to_s, ';' ) ), '/' )
+            Link.encode string
         end
 
         def decode( *args )
-            URI.decode( *args )
+            Link.decode( *args )
         end
 
         def type

data/lib/arachni/element/link_template/dom.rb

@@ -68,22 +68,6 @@ class DOM < Base
         LinkTemplate.extract_inputs( url, templates )
     end
 
-    def encode( string )
-        self.class.encode( string )
-    end
-
-    def self.encode( string )
-        string
-    end
-
-    def decode( *args )
-        self.class.decode( *args )
-    end
-
-    def self.decode( *args )
-        Link.decode( *args )
-    end
-
     def type
         self.class.type
     end

data/lib/arachni/element/server.rb

@@ -16,14 +16,27 @@ module Arachni::Element
 class Server < Base
     include Capabilities::WithAuditor
 
-    # Used to determine how different a resource should be in order to be sent
-    # to the {Trainer#push}.
+    # Valid responses to discovery checks should vary *wildly*, especially when
+    # considering the types of directories and files that these checks look for.
     #
-    # Ideally, all identified resources should be analyzed by the {Trainer} but
-    # there can be cases where unreliable custom-4o4 signatures lead to FPs and
-    # feeding FPs to the system can create an infinite loop.
+    # On the other hand, custom-404 or such responses will have many things in
+    # common which makes it possible to spot them without much bother.
+    #
+    # Ideally, custom-404s will be identified properly by the
+    # {HTTP::Client::Dynamic404Handler} but this is here to save our ass in case
+    # there's a bug or an unforeseen edge-case or something.
+    #
+    # Also, identified resources should be analyzed by the {Trainer} but there
+    # can be cases where unreliable custom-404 signatures lead to FPs and feeding
+    # FPs to the system can create an infinite loop.
     SIMILARITY_TOLERANCE = 0.25
 
+    # Remark in case of an untrusted issue.
+    REMARK = 'This issue was logged by a discovery check but ' +
+        'the response for the resource it identified is very similar to responses ' +
+        'for other resources of similar type. This is a strong indication that ' +
+        'the logged issue is a false positive.'
+
     def initialize( url )
         super url: url
         @initialization_options = url
@@ -86,7 +99,13 @@ class Server < Base
         return false if !full_and_absolute_url?( url )
 
         if http.dynamic_404_handler.needs_check?( url )
-            http.get( url, performer: self ) do |r|
+
+            # Don't enable fingerprinting if there's a dynamic handler, we don't
+            # want to keep analyzing non existent resources.
+            #
+            # If a resource does exist though it will be fingerprinted down the
+            # line.
+            http.get( url, performer: self, fingerprint: false ) do |r|
                 if r.code == 200
                     http.dynamic_404_handler._404?( r ) { |bool| block.call( !bool, r ) }
                 else
@@ -118,6 +137,23 @@ class Server < Base
         s << '>'
     end
 
+    def self.flag_issues_as_untrusted( issue_digests )
+        issue_digests.uniq.each do |digest|
+            next if !Arachni::Data.issues[digest]
+
+            Arachni::Data.issues[digest].variations.each do |issue|
+                issue.add_remark :meta_analysis, REMARK
+                issue.trusted = false
+            end
+        end
+    end
+
+    def self.flag_issues_if_untrusted( similarity, issue_digests )
+        return if similarity < SIMILARITY_TOLERANCE
+
+        flag_issues_as_untrusted( issue_digests )
+    end
+
     private
 
     def analyze
@@ -153,10 +189,14 @@ class Server < Base
         # framework custom-404s and get into an infinite loop.
         train = similarity < SIMILARITY_TOLERANCE
 
+        issue_digests = []
         @candidates.each do |response, block|
-            log response, train, &block
+            issue_digests << log( response, train, &block ).digest
         end
 
+        return if train
+
+        self.class.flag_issues_as_untrusted( issue_digests )
     ensure
         @candidates.clear
     end
@@ -164,12 +204,14 @@ class Server < Base
     def log( response, train = true, &block )
         block.call( response ) if block_given?
 
-        auditor.log_remote_file( response )
+        issue = auditor.log_remote_file( response )
 
-        return if !train
+        return issue if !train
 
         # Use the newly identified resource to increase the scan scope.
         auditor.framework.trainer.push( response )
+
+        issue
     end
 
 end

data/lib/arachni/element/xml.rb

@@ -137,6 +137,7 @@ class XML < Base
         # @return   [XML, nil]
         def from_request( url, request )
             return if !request.body.is_a?( String ) || request.body.empty?
+            return if too_big?( request.body )
 
             data = parse_inputs( request.body )
             return if data.empty?

data/lib/arachni/ethon/easy.rb

@@ -11,9 +11,12 @@ class Easy
 module Callbacks
     def debug_callback
         @debug_callback ||= proc do |handle, type, data, size, udata|
+            # We only care about these so that we can have access to raw
+            # HTTP request traffic for reporting/debugging purposes.
+            next if type != :header_out && type != :data_out
+
             message = data.read_string( size )
             @debug_info.add type, message
-            # print message unless [:data_in, :data_out].include?(type)
             0
         end
     end

data/lib/arachni/framework/parts/audit.rb

@@ -117,25 +117,28 @@ module Audit
         # resulting pages back to the framework.
         perform_browser_analysis( page )
 
-        # Remove elements which have already passed through here.
-        pre_audit_element_filter( page )
-
-        # Filter the page through the browser and apply DOM metadata to itself
-        # and its elements in order to allow for audit optimizations down the
-        # line.
-        #
-        # For example, if a DOM element has no associated events, there's no
-        # point in it getting audited.
-        apply_dom_metadata( page )
-
-        notify_on_effective_page_audit( page )
-
-        # Run checks which **don't** benefit from fingerprinting first, so that
-        # we can use the responses of their HTTP requests to fingerprint the
-        # webapp platforms, so that the checks which **do** benefit from knowing
-        # the remote platforms can run more efficiently.
-        run_http = run_checks( @checks.without_platforms, page )
-        run_http = true if run_checks( @checks.with_platforms, page )
+        run_http = false
+
+        if checks.any?
+            # Remove elements which have already passed through here.
+            pre_audit_element_filter( page )
+
+            notify_on_effective_page_audit( page )
+
+            # Run checks which **don't** benefit from fingerprinting first, so
+            # that we can use the responses of their HTTP requests to fingerprint
+            # the webapp platforms, so that the checks which **do** benefit from
+            # knowing the remote platforms can run more efficiently.
+            run_http = run_checks( @checks.without_platforms, page )
+            run_http = true if run_checks( @checks.with_platforms, page )
+        end
+
+        notify_after_page_audit( page )
+
+        # Makes it easier on the GC but it is important that it be called
+        # **after** all the callbacks have been executed because they may need
+        # access to the cached data and there's no sense in re-parsing.
+        page.clear_cache
 
         if Arachni::Check::Auditor.has_timeout_candidates?
             print_line
@@ -145,10 +148,6 @@ module Audit
             run_http = true
         end
 
-        # Makes it easier on the GC.
-        page.clear_cache
-
-        notify_after_page_audit( page )
         run_http
     end
 
@@ -207,66 +206,21 @@ module Audit
 
         @audit_queues_done = false
 
-        # If for some reason we've got pages in the page queue this early,
-        # consume them and get it over with.
-        audit_page_queue
-
-        next_page = nil
-        while !suspended? && !page_limit_reached? &&
-            (page = next_page || pop_page_from_url_queue)
-
-            # Schedule the next page to be grabbed along with the audit requests
-            # for the current page in order to avoid blocking.
-            next_page = nil
-            next_page_call = proc do
-                pop_page_from_url_queue { |p| next_page = p }
-            end
+        while !suspended? && !page_limit_reached? && (page = pop_page)
 
-            # If we have login capabilities make sure that our session is valid
-            # before grabbing and auditing the next page.
-            if session.can_login?
-                # Schedule the login check to happen along with the audit requests
-                # to prevent blocking and grab the next page as well.
-                session.logged_in? do |bool|
-                    next next_page_call.call if bool
+            session.ensure_logged_in
 
-                    session.login
-                    next_page_call
-                end
-            else
-                next_page_call.call
-            end
+            replenish_page_queue_from_url_queue
 
-            # We're counting on piggybacking the next page retrieval with the
+            # We're counting on piggybacking the page queue replenishing on the
             # page audit, however if there wasn't an audit we need to force an
             # HTTP run.
             audit_page( page ) or http.run
 
-            if next_page && suspend?
-                data.page_queue << next_page
-            end
-
             handle_signals
-
-            # Consume pages somehow triggered by the audit and pushed by the
-            # trainer or plugins or whatever.
-            audit_page_queue
         end
 
-        audit_page_queue
-
         @audit_queues_done = true
-        true
-    end
-
-    # Audits the page queue.
-    #
-    # @see #pop_page_from_queue
-    def audit_page_queue
-        while !suspended? && !page_limit_reached? && (page = pop_page_from_queue)
-            audit_page( page )
-            handle_signals
-        end
     end
 
     def harvest_http_responses
@@ -274,10 +228,6 @@ module Audit
         print_info 'Depending on server responsiveness and network' <<
                        ' conditions this may take a while.'
 
-        # Run all the queued HTTP requests and harvest the responses.
-        http.run
-
-        # Needed for some HTTP callbacks.
         http.run
     end
 
@@ -286,4 +236,3 @@ end
 end
 end
 end
-