arachni 1.3.2 → 1.4

data/lib/arachni/http/request.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -15,6 +15,8 @@ module HTTP
 class Request < Message
     require_relative 'request/scope'
 
+    ENCODE_CACHE = Support::Cache::LeastRecentlyPushed.new( 10_000 )
+
     # Default redirect limit, RFC says 5 max.
     REDIRECT_LIMIT = 5
 
@@ -61,6 +63,9 @@ class Request < Message
     #   Cookies set for this request.
     attr_reader   :cookies
 
+    # @return   [Array<Element::Cookie>]
+    attr_reader   :raw_cookies
+
     # @return   [Symbol]
     #   Mode of operation for the request.
     #
@@ -101,6 +106,10 @@ class Request < Message
     #   Maximum HTTP response size to accept, in bytes.
     attr_accessor :response_max_size
 
+    # @return   [Array]
+    #   Parameters which should not be encoded, by name.
+    attr_accessor :raw_parameters
+
     # @param  [Hash]  options
     #   Request options.
     # @option options [String] :url
@@ -132,10 +141,20 @@ class Request < Message
         @max_redirects   = (Options.http.request_redirect_limit || REDIRECT_LIMIT)
         @on_complete     = []
 
-        @timeout       ||= Options.http.request_timeout
-        @mode          ||= :async
-        @parameters    ||= {}
-        @cookies       ||= {}
+        @raw_parameters ||= []
+        @timeout        ||= Options.http.request_timeout
+        @mode           ||= :async
+        @parameters     ||= {}
+        @cookies        ||= {}
+        @raw_cookies    ||= []
+    end
+
+    def raw_parameters=( names )
+        if names
+            @raw_parameters = names
+        else
+            @raw_parameters.clear
+        end
     end
 
     def high_priority?
@@ -209,14 +228,29 @@ class Request < Message
     end
 
     def effective_cookies
-        Cookie.from_string( url, headers['Cookie'] || '' ).inject({}) do |h, cookie|
-            h[cookie.name] = cookie.value
+        effective_cookies = self.cookies.dup
+
+        if !headers['Cookie'].to_s.empty?
+            Cookie.from_string( url, headers['Cookie'] ).
+                inject( effective_cookies ) do |h, cookie|
+                h[cookie.name] ||= cookie.value
+                h
+            end
+        end
+
+        @raw_cookies.inject( effective_cookies ) do |h, cookie|
+            h[cookie.raw_name] ||= cookie.raw_value
             h
-        end.merge( cookies )
+        end
+
+        effective_cookies
     end
 
     def effective_parameters
-        Utilities.uri_parse_query( url ).merge( parameters || {} )
+        ep = Utilities.uri_parse_query( url )
+        return ep if parameters.empty?
+
+        ep.merge!( parameters )
     end
 
     def body_parameters
@@ -337,12 +371,22 @@ class Request < Message
         max_size = 1   if max_size == 0
         max_size = nil if max_size < 0
 
+        ep = self.class.encode_hash( self.effective_parameters, @raw_parameters )
+
+        eb = self.body
+        if eb.is_a?( Hash )
+            eb = self.class.encode_hash( eb, @raw_parameters )
+        end
+
         options = {
             method:          method,
             headers:         headers,
-            body:            body,
-            params:          effective_parameters,
+
+            body:            eb,
+            params:          ep,
+
             userpwd:         userpwd,
+
             followlocation:  follow_location?,
             maxredirs:       @max_redirects,
 
@@ -365,10 +409,19 @@ class Request < Message
             # the reading of the response body if it exceeds this limit.
             maxfilesize:     max_size,
 
-            # Don't keep the socket alive if this is a blocking request because
-            # it's going to be performed by an one-off Hydra.
-            forbid_reuse:    blocking?,
-            verbose:         true
+            # Reusing connections for blocking requests used to cause FD leaks
+            # but doesn't appear to do so anymore.
+            #
+            # Let's allow reuse for all request types again but keep an eye on it.
+            # forbid_reuse:    blocking?,
+
+            # Enable debugging messages in order to capture raw traffic data.
+            verbose:         true,
+
+            # We're going to be escaping **a lot** of the same strings during
+            # the scan, so bypass Ethon's encoding and do our own cache-based
+            # encoding.
+            escape:          false
         }
 
         options[:timeout_ms] = timeout if timeout
@@ -404,8 +457,7 @@ class Request < Message
             end
         end
 
-        curl             = parsed_url.query ? url.gsub( "?#{parsed_url.query}", '' ) : url
-        typhoeus_request = Typhoeus::Request.new( curl, options )
+        typhoeus_request = Typhoeus::Request.new( url.split( '?').first, options )
 
         if @on_complete.any?
             response_body_buffer = ''
@@ -418,7 +470,10 @@ class Request < Message
                 end
 
                 fill_in_data_from_typhoeus_response typhoeus_response
-                handle_response Response.from_typhoeus( typhoeus_response )
+                handle_response Response.from_typhoeus(
+                    typhoeus_response,
+                    normalize_url: @normalize_url
+                )
             end
         end
 
@@ -446,10 +501,12 @@ class Request < Message
     end
 
     def marshal_dump
-        callbacks = @on_complete.dup
-        performer = @performer
+        raw_cookies = @raw_cookies.dup
+        callbacks   = @on_complete.dup
+        performer   = @performer
 
         @performer   = nil
+        @raw_cookies = []
         @on_complete = []
 
         instance_variables.inject( {} ) do |h, iv|
@@ -458,6 +515,7 @@ class Request < Message
             h
         end
     ensure
+        @raw_cookies = raw_cookies
         @on_complete = callbacks
         @performer   = performer
     end
@@ -500,31 +558,70 @@ class Request < Message
         # @return   [Hash]
         #   Parameters.
         def parse_body( body )
-            return {} if !body
+            return {} if body.to_s.empty?
 
-            body.to_s.split( '&' ).inject( {} ) do |h, pair|
+            body.split( '&' ).inject( {} ) do |h, pair|
                 name, value = pair.split( '=', 2 )
                 h[Form.decode( name.to_s )] = Form.decode( value )
                 h
             end
         end
 
+        def encode_hash( hash, skip = [] )
+            hash.inject({}) do |h, (k, v)|
+
+                if skip.include?( k )
+                    # We need to at least encode null-bytes since they can't
+                    # be transported at all.
+                    # If we don't Typhoeus/Ethon will raise errors.
+                    h.merge!( encode_null_byte( k ) => encode_null_byte( v ) )
+                else
+                    h.merge!( encode( k ) => encode( v ) )
+                end
+
+                h
+            end
+        end
+
+        def encode_null_byte( string )
+            string.to_s.gsub "\0", '%00'
+        end
+
         def encode( string )
+            string = string.to_s
             @easy ||= Ethon::Easy.new( url: 'www.example.com' )
-            @easy.escape string
+            ENCODE_CACHE.fetch( string ) { @easy.escape( string ) }
         end
     end
 
     def prepare_headers
-        headers['User-Agent'] ||= Options.http.user_agent
-        headers['Accept']     ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
-        headers['From']       ||= Options.authorized_by if Options.authorized_by
+        headers['User-Agent']      ||= Options.http.user_agent
+        headers['Accept']          ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+        headers['From']            ||= Options.authorized_by if Options.authorized_by
+        headers['Accept-Language'] ||= 'en-US,en;q=0.8,he;q=0.6'
 
         headers.each { |k, v| headers[k] = Header.encode( v ) if v }
 
-        headers['Cookie'] = effective_cookies.
-            map { |k, v| "#{Cookie.encode( k, true )}=#{Cookie.encode( v )}" }.
-            join( ';' )
+        final_cookies_hash = self.cookies
+        final_raw_cookies  = self.raw_cookies
+
+        if headers['Cookie']
+            final_raw_cookies_set = Set.new( final_raw_cookies.map(&:name) )
+            final_raw_cookies |= Cookie.from_string( url, headers['Cookie'] ).reject do |c|
+                final_cookies_hash.include?( c.name ) ||
+                    final_raw_cookies_set.include?( c.name )
+            end
+        end
+
+        headers['Cookie'] = final_cookies_hash.
+            map { |k, v| "#{Cookie.encode( k )}=#{Cookie.encode( v )}" }.join( ';' )
+
+        if !headers['Cookie'].empty? && final_raw_cookies.any?
+            headers['Cookie'] += ';'
+        end
+
+        headers['Cookie'] += final_raw_cookies.map { |c| c.to_s }.join( ';' )
+
         headers.delete( 'Cookie' ) if headers['Cookie'].empty?
 
         headers

data/lib/arachni/http/request/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/http/response.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -77,6 +77,16 @@ class Response < Message
         @time = t.to_f
     end
 
+    # @return   [Boolean]
+    #   `true` if the client could not read the entire response, `false` otherwise.
+    def partial?
+        # Streamed response which was aborted before completing.
+        return_code == :partial_file ||
+            # Normal response with some data written, but without reaching
+            # content-length.
+            (code != 0 && timed_out?)
+    end
+
     # @return   [Platform]
     #   Applicable platforms for the page.
     def platforms
@@ -117,6 +127,13 @@ class Response < Message
         code != 304
     end
 
+    # @return [Boolean]
+    #   `true` if the request was performed successfully and the response was
+    #   received in full, `false` otherwise.
+    def ok?
+        !return_code || return_code == :ok
+    end
+
     # @return [Bool]
     #   `true` if the response body is textual in nature, `false` if binary,
     #   `nil` if could not be determined.
@@ -142,7 +159,7 @@ class Response < Message
     # @return   [Boolean]
     #   `true` if timed out, `false` otherwise.
     def timed_out?
-        [:operation_timedout, :couldnt_connect].include? return_code
+        return_code == :operation_timedout
     end
 
     def body=( body )
@@ -168,6 +185,7 @@ class Response < Message
 
         hash[:headers] = {}.merge( hash[:headers] )
 
+        hash.delete( :normalize_url )
         hash.delete( :scope )
         hash.delete( :parsed_url )
         hash.delete( :redirections )
@@ -201,7 +219,7 @@ class Response < Message
         to_h.hash
     end
 
-    def self.from_typhoeus( response )
+    def self.from_typhoeus( response, options = {} )
         redirections = response.redirections.map do |redirect|
             rurl   = URI.to_absolute( redirect.headers['Location'],
                                       response.effective_url )
@@ -210,14 +228,26 @@ class Response < Message
             # Broken redirection, skip it...
             next if !rurl
 
-            new(
-                url:     rurl,
-                code:    redirect.code,
-                headers: redirect.headers
-            )
+            new( options.merge(
+                url:           rurl,
+                code:          redirect.code,
+                headers:       redirect.headers
+            ))
+        end
+
+        return_code    = response.return_code
+        return_message = response.return_message
+
+        # A write error in this case will be because body reading was aborted
+        # during our own callback in Request#set_body_reader.
+        #
+        # So, this is here just for consistency.
+        if response.return_code == :write_error
+            return_code    = :filesize_exceeded
+            return_message = 'Maximum file size exceeded'
         end
 
-        new(
+        new( options.merge(
             url:            response.effective_url,
             code:           response.code,
             ip_address:     response.primary_ip,
@@ -229,9 +259,9 @@ class Response < Message
             app_time:       (response.timed_out? ? response.time :
                                 response.start_transfer_time - response.pretransfer_time).to_f,
             total_time:     response.total_time.to_f,
-            return_code:    response.return_code,
-            return_message: response.return_message
-        )
+            return_code:    return_code,
+            return_message: return_message
+        ))
     end
 
 end

data/lib/arachni/http/response/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/issue.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -256,7 +256,7 @@ class Issue
             if object
                 # Once the object is logged we need a deep copy of it to ensure
                 # integrity.
-                object = object.deep_clone
+                object = object.rpc_clone
                 object.prepare_for_report
             end
 

data/lib/arachni/issue/severity.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/issue/severity/base.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_group.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_groups.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/option_groups/audit.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2015 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -42,6 +42,14 @@ class Audit < Arachni::OptionGroup
     # @see Element::Capabilities::Mutable#each_mutation
     attr_accessor :parameter_names
 
+    # @note Default is `false`.
+    #
+    # @return    [Bool]
+    #   Allows checks to sent payloads in raw format, without HTTP encoding.
+    #
+    # @see Element::Capabilities::Mutable#each_mutation
+    attr_accessor :with_raw_payloads
+
     # @note Default is `false`.
     #
     # @return    [Bool]
@@ -170,6 +178,10 @@ class Audit < Arachni::OptionGroup
         link_templates:          []
     )
 
+    def with_raw_payloads?
+        !!@with_raw_payloads
+    end
+
     # @param    [Array<Regexp>] templates
     #   Regular expressions with named captures, serving as templates used to
     #   extract input vectors from paths.
@@ -181,7 +193,9 @@ class Audit < Arachni::OptionGroup
         return @link_templates = [] if !templates
 
         @link_templates = [templates].flatten.compact.map do |s|
-            template = s.is_a?( Regexp ) ? s : Regexp.new( s.to_s )
+            template = s.is_a?( Regexp ) ?
+                s :
+                Regexp.new( s.to_s, Regexp::IGNORECASE )
 
             if template.names.empty?
                 fail Error::InvalidLinkTemplate,
@@ -196,7 +210,9 @@ class Audit < Arachni::OptionGroup
     %w(include_vector_patterns exclude_vector_patterns).each do |m|
         define_method "#{m}=" do |patterns|
             patterns = [patterns].flatten.compact.
-                map { |s| s.is_a?( Regexp ) ? s : Regexp.new( s.to_s ) }
+                map { |s| s.is_a?( Regexp ) ?
+                s :
+                Regexp.new( s.to_s, Regexp::IGNORECASE ) }
 
             instance_variable_set "@#{m}".to_sym, patterns
         end
@@ -295,7 +311,7 @@ class Audit < Arachni::OptionGroup
     def to_h
         h = super
         [:link_templates, :include_vector_patterns, :exclude_vector_patterns].each do |k|
-            h[k] = h[k].map(&:to_s)
+            h[k] = h[k].map(&:source)
         end
         h
     end