arachni 1.3.2 → 1.4

data/lib/arachni/http/client/dynamic_404_handler.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -41,7 +41,7 @@ class Dynamic404Handler
         # need advanced analysis.
         if checked_and_static?( url )
             result = (response.code == 404)
-            print_debug "#{__method__} [static]: #{block} #{url} #{result}"
+            print_debug "[static]: #{block} #{url} #{result}"
             block.call( result )
             return
         end
@@ -54,14 +54,14 @@ class Dynamic404Handler
             # If we've got a positive result that's all we need to know, return
             # it immediately.
             if result
-                print_debug "#{__method__} [cached]: #{block} #{url} #{result}"
+                print_debug "[cached]: #{block} #{url} #{result}"
                 return block.call( result )
             end
 
             # If the result was negative only return it if there's no need for
             # advanced analysis for this resource.
             if !needs_advanced_analysis?( url )
-                print_debug "#{__method__} [cached]: #{block} #{url} #{result}"
+                print_debug "[cached]: #{block} #{url} #{result}"
                 return block.call( result )
             end
         end
@@ -71,7 +71,7 @@ class Dynamic404Handler
         # complete.
         data_for( url )[:waiting] << [url, response.code, response.body, block]
         if data_for( url )[:in_progress]
-            print_debug "#{__method__} [waiting]: #{url} #{block}"
+            print_debug "[waiting]: #{url} #{block}"
             return
         end
         data_for( url )[:in_progress] = true
@@ -81,12 +81,12 @@ class Dynamic404Handler
         # don't waste time redoing it, we can jump straight into the advanced
         # analysis.
         if checked?( url ) && needs_advanced_analysis?( url )
-            print_debug "#{__method__} [checking-advanced]: #{url} #{block}"
+            print_debug "[checking-advanced]: #{url} #{block}"
             process_advanced_analysis_callers_for( url )
             return
         end
 
-        print_debug "#{__method__} [checking]: #{url} #{block}"
+        print_debug "[checking]: #{url} #{block}"
 
         # So... we've got nothing cached for the handler for this URL, let's
         # start from scratch.
@@ -154,47 +154,36 @@ class Dynamic404Handler
 
         real_404s          = 0
         corrupted          = false
-        gathered_responses = 0
-        expected_responses = generators.size * PRECISION
+        gathered_signatures = 0
+        expected_signatures = generators.size
 
         generators.each.with_index do |generator, i|
             current_signature = (preliminary_signatures_for( url )[i] ||= {})
 
-            PRECISION.times do
-                request( generator.call ) do |c_res|
-                    next if corrupted
-
-                    print_debug "#{__method__} [gathering]: #{c_res.request.url} #{c_res.url} #{c_res.code} #{block}"
+            signature_from_url generator.call, current_signature do |c_res, status|
+                next if corrupted
 
-                    # Well, bad luck, bail out to avoid FPs.
-                    if corrupted_response?( c_res )
-                        print_debug "#{__method__} [corrupted]: #{url} #{c_res.code} #{block}"
-                        corrupted = true
-                        next clear_data_for( url )
-                    end
+                if status == :done
+                    print_debug "[gathering]: #{c_res.request.url} #{c_res.url} #{c_res.code} #{block}"
 
-                    gathered_responses += 1
+                    gathered_signatures += 1
                     if c_res.code == 404
                         real_404s += 1
                     end
 
-                    if current_signature[:body]
-                        current_signature[:rdiff] =
-                            current_signature[:body].refine( c_res.body )
-
-                        next if gathered_responses != expected_responses
-
-                        if real_404s == expected_responses
+                    if gathered_signatures == expected_signatures
+                        if real_404s == expected_signatures
                             @static << url_for( url )
                         end
 
                         block.call
-                    else
-                        current_signature[:body] = Support::Signature.new(
-                            c_res.body, threshold: SIGNATURE_THRESHOLD
-                        )
                     end
                 end
+
+                if status == :corrupted
+                    print_debug "[corrupted]: #{url} #{block}"
+                    corrupted = true
+                end
             end
         end
     end
@@ -207,40 +196,76 @@ class Dynamic404Handler
             return
         end
 
-        corrupted          = false
-        gathered_responses = 0
-        expected_responses = generators.size * PRECISION
+        corrupted           = false
+        gathered_signatures = 0
+        expected_signatures = generators.size
 
         generators.each.with_index do |generator, i|
             current_signature = (advanced_signatures_for( url )[i] ||= {})
 
-            PRECISION.times do
-                request( generator.call ) do |c_res|
-                    next if corrupted
+            signature_from_url generator.call, current_signature do |c_res, status|
+                next if corrupted
 
-                    print_debug "#{__method__} [gathering]: #{c_res.request.url} #{c_res.url} #{c_res.code} #{block}"
+                print_debug "[gathering]: #{c_res.request.url} #{c_res.url} #{c_res.code} #{block}"
 
-                    # Well, bad luck, bail out to avoid FPs.
-                    if corrupted_response?( c_res )
-                        print_debug "#{__method__} [corrupted]: #{url} #{block}"
-                        corrupted = true
-                        next clear_data_for( url )
-                    end
+                gathered_signatures += 1
 
-                    gathered_responses += 1
+                if status == :done && gathered_signatures == expected_signatures
+                    block.call
+                end
 
-                    if current_signature[:body]
-                        current_signature[:rdiff] =
-                            current_signature[:body].refine( c_res.body )
+                if status == :corrupted
+                    print_debug "[corrupted]: #{url} #{block}"
+                    corrupted = true
+                end
+            end
+        end
+    end
 
-                        next if gathered_responses != expected_responses
+    def signature_from_url( url, signature_data, precision = PRECISION, &block )
+        controlled_precision = precision * 2
+        control_data         = {}
+
+        corrupted          = false
+        gathered_responses = 0
+
+        controlled_precision.times do
+            request( url ) do |response|
+                next if corrupted
+
+                signature = gathered_responses >= precision ?
+                    control_data : signature_data
+
+                # Well, bad luck, bail out to avoid FPs.
+                if corrupted_response?( response )
+                    block.call response, :corrupted
+                    corrupted = true
+                    next
+                end
+
+                gathered_responses += 1
+
+                if signature[:body]
+                    signature[:rdiff] = signature[:body].refine( response.body )
+
+                    if gathered_responses == controlled_precision
+
+                        # Both attempts yielded in the same result, the webapp
+                        # was stable during the process and the signature can be
+                        # considered accurate.
+                        if control_data[:rdiff] == signature_data[:rdiff]
+                            block.call response, :done
+
+                        # Coo-coo for cocoa puffs, can't work with it.
+                        else
+                            block.call response, :corrupted
+                        end
 
-                        block.call
-                    else
-                        current_signature[:body] = Support::Signature.new(
-                            c_res.body, threshold: SIGNATURE_THRESHOLD
-                        )
                     end
+                else
+                    signature[:body] = Support::Signature.new(
+                        response.body, threshold: SIGNATURE_THRESHOLD
+                    )
                 end
             end
         end
@@ -248,10 +273,10 @@ class Dynamic404Handler
 
     def perform_advanced_analysis_if_necessary( url, body, &block )
         result = matches_preliminary_signatures?( url, body )
-        print_debug "#{__method__} [checked]: #{block} #{url} #{result}"
+        print_debug "[checked]: #{block} #{url} #{result}"
 
         if result
-            print_debug "#{__method__} [notify]: #{block} #{url} #{result}"
+            print_debug "[notify]: #{block} #{url} #{result}"
             checked( url )
             block.call( result )
             return
@@ -264,7 +289,7 @@ class Dynamic404Handler
             # then this handler will always require advanced analysis for each
             # URL.
             result = matches_advanced_signatures?( url, body )
-            print_debug "#{__method__} [notify]: #{block} #{url} #{result}"
+            print_debug "[notify]: #{block} #{url} #{result}"
             block.call result
 
             # More callers may have been added to the waiting queue during
@@ -286,7 +311,7 @@ class Dynamic404Handler
         while (waiting = data_for( url )[:waiting].pop)
             curl, code, _, callback = waiting
             result = (code == 404)
-            print_debug "#{__method__} [notify]: #{callback} #{curl} #{result}"
+            print_debug "[notify]: #{callback} #{curl} #{result}"
             callback.call result
         end
     end
@@ -314,7 +339,7 @@ class Dynamic404Handler
     # In that case we should bail out to avoid corrupted signatures which can
     # lead to FPs.
     def corrupted_response?( response )
-        response.code != 404 && response.code != 200
+        !response.ok? || (response.code != 404 && response.code != 200)
     end
 
     def url_for( url )

data/lib/arachni/http/cookie_jar.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -39,7 +39,7 @@ class CookieJar
     # @param    [String]    cookie_jar_file
     #   Path to a Netscape cookie-jar.
     def initialize( cookie_jar_file = nil )
-        @domains = {}
+        @cookies = {}
         load( cookie_jar_file ) if cookie_jar_file
     end
 
@@ -52,10 +52,11 @@ class CookieJar
     #
     # @return   [CookieJar]  self
     def load( cookie_jar_file, url = '' )
-        # make sure that the provided cookie-jar file exists
         if !File.exist?( cookie_jar_file )
-            fail Error::CookieJarFileNotFound, "Cookie-jar '#{cookie_jar_file}' doesn't exist."
+            fail Error::CookieJarFileNotFound,
+                 "Cookie-jar '#{cookie_jar_file}' doesn't exist."
         end
+
         update( cookies_from_file( url, cookie_jar_file ) )
         self
     end
@@ -66,8 +67,9 @@ class CookieJar
     # @return   [CookieJar]
     #   `self`
     def <<( cookies )
-        [cookies].flatten.compact.each do |cookie|
-            ((@domains[cookie.domain] ||= {})[cookie.path] ||= {})[cookie.name] = cookie.dup
+        [cookies].flatten.each do |cookie|
+            next if !cookie
+            set_cookie( cookie )
         end
         self
     end
@@ -79,7 +81,9 @@ class CookieJar
     #
     # @return   [CookieJar]  self
     def update( cookies )
-        [cookies].flatten.compact.each do |c|
+        [cookies].flatten.each do |c|
+            next if !c
+
             self << case c
                         when String
                             begin
@@ -115,15 +119,17 @@ class CookieJar
 
         return [] if !request_domain || !request_path
 
-        @domains.map do |domain, paths|
-            next if !in_domain?( domain, request_domain )
+        unique_cookies = {}
+        @cookies.values.map do |cookie|
+            if cookie.expired? || !request_path.start_with?( cookie.path ) ||
+                !in_domain?( cookie.domain, request_domain )
+                next
+            end
 
-            paths.map do |path, cookies|
-                next if !request_path.start_with?( path )
+            unique_cookies[cookie.name] = cookie
+        end
 
-                cookies.values.reject{ |c| c.expired? }
-            end
-        end.flatten.compact.sort do |lhs, rhs|
+        unique_cookies.values.sort do |lhs, rhs|
             rhs.path.length <=> lhs.path.length
         end
     end
@@ -134,15 +140,10 @@ class CookieJar
     # @return   [Array<Cookie>]
     #   All cookies.
     def cookies( include_expired = false )
-        @domains.values.map do |paths|
-            paths.values.map do |cookies|
-                if !include_expired
-                    cookies.values.reject{ |c| c.expired? }
-                else
-                    cookies.values
-                end
-            end
-        end.flatten.compact
+        @cookies.values.map do |cookie|
+            next if !include_expired && cookie.expired?
+            cookie
+        end.compact
     end
 
     # @param    [CookieJar] other
@@ -152,13 +153,13 @@ class CookieJar
 
     # Empties the cookiejar.
     def clear
-        @domains.clear
+        @cookies.clear
     end
 
     # @return   [Bool]
     #   `true` if cookiejar is empty, `false` otherwise.
     def empty?
-        @domains.empty?
+        @cookies.empty?
     end
 
     # @return   [Bool]
@@ -178,10 +179,30 @@ class CookieJar
 
     private
 
+    def get_cookie( cookie )
+        @cookies[make_key( cookie )]
+    end
+
+    def set_cookie( cookie )
+        key = make_key( cookie )
+
+        if (existing = @cookies[key]) &&
+            cookie.to_set_cookie == existing.to_set_cookie
+            return
+        end
+
+        @cookies[key] = cookie.dup.tap { |c| c.page = nil }
+    end
+
+    def make_key( cookie )
+        "#{cookie.domain}:#{cookie.path}:#{cookie.name}".hash
+    end
+
     def in_domain?( cookie_domain, request_domain )
         request_domain == cookie_domain ||
-            ( cookie_domain.start_with?( '.' ) &&
-              request_domain.end_with?( cookie_domain[1...cookie_domain.size] )
+            (
+                cookie_domain.start_with?( '.' ) &&
+                    request_domain.end_with?( cookie_domain[1...cookie_domain.size] )
             )
     end
 

data/lib/arachni/http/headers.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -18,7 +18,7 @@ module HTTP
 # @author Tasos Laskos <tasos.laskos@arachni-scanner.com>
 class Headers < Hash
 
-    FORMATTED_NAMES_CACHE = Support::Cache::LeastRecentlyPushed.new( 100 )
+    FORMATTED_NAMES_CACHE = Support::Cache::LeastRecentlyPushed.new( 10_000 )
 
     CONTENT_TYPE = 'content-type'
     SET_COOKIE   = 'set-cookie'
@@ -138,8 +138,9 @@ class Headers < Hash
         # audit payload.
         return field if field.include?( '--' )
 
-        FORMATTED_NAMES_CACHE[field] ||=
+        FORMATTED_NAMES_CACHE.fetch field do
             field.split( '-' ).map( &:capitalize ).join( '-' )
+        end
     end
 
 end

data/lib/arachni/http/message.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -37,6 +37,14 @@ class Message
     # @option   options [String]    :body
     #   Body.
     def initialize( options = {} )
+        options = options.dup
+
+        @normalize_url = options.delete( :normalize_url )
+        @normalize_url = true if @normalize_url.nil?
+
+        # Headers are necessary for subsequent operations to set them first.
+        @headers = Headers.new( options.delete( :headers ) || {} )
+
         options.each do |k, v|
             begin
                 send( "#{k}=", v )
@@ -46,8 +54,10 @@ class Message
         end
 
         fail ArgumentError, 'Missing :url.' if url.to_s.empty?
+    end
 
-        @headers = Headers.new( @headers )
+    def headers=( h )
+        @headers = Headers.new( h || {} )
     end
 
     # @return   [Scope]
@@ -61,7 +71,11 @@ class Message
     end
 
     def url=( url )
-        @url = Arachni::URI( url ).to_s.freeze
+        if @normalize_url
+            @url = URI.normalize_url( url ).to_s.freeze
+        else
+            @url = url.to_s.freeze
+        end
     end
 
 end