arachni 1.1 → 1.2

data/lib/arachni/http/client/dynamic_404_handler.rb

@@ -161,16 +161,14 @@ class Dynamic404Handler
             current_signature = (preliminary_signatures_for( url )[i] ||= {})
 
             PRECISION.times do
-                Client.get( generator.call,
-                            # This is important, helps us reduce waiting callers.
-                            high_priority:   true,
-                            performer:       self
-                ) do |c_res|
+                request( generator.call ) do |c_res|
                     next if corrupted
 
+                    print_debug "#{__method__} [gathering]: #{c_res.request.url} #{c_res.url} #{c_res.code} #{block}"
+
                     # Well, bad luck, bail out to avoid FPs.
-                    if c_res.code == 0
-                        print_debug "#{__method__} [corrupted]: #{url} #{block}"
+                    if corrupted_response?( c_res )
+                        print_debug "#{__method__} [corrupted]: #{url} #{c_res.code} #{block}"
                         corrupted = true
                         next clear_data_for( url )
                     end
@@ -217,15 +215,13 @@ class Dynamic404Handler
             current_signature = (advanced_signatures_for( url )[i] ||= {})
 
             PRECISION.times do
-                Client.get( generator.call,
-                            # This is important, helps us reduce waiting callers.
-                            high_priority:   true,
-                            performer:       self
-                ) do |c_res|
+                request( generator.call ) do |c_res|
                     next if corrupted
 
+                    print_debug "#{__method__} [gathering]: #{c_res.request.url} #{c_res.url} #{c_res.code} #{block}"
+
                     # Well, bad luck, bail out to avoid FPs.
-                    if c_res.code == 0
+                    if corrupted_response?( c_res )
                         print_debug "#{__method__} [corrupted]: #{url} #{block}"
                         corrupted = true
                         next clear_data_for( url )
@@ -305,7 +301,20 @@ class Dynamic404Handler
     def needs_advanced_analysis?( url )
         uri = uri_parse( url )
         resource_name = uri.resource_name.to_s.split('.').tap(&:pop).join('.')
-        !!(!resource_name.empty? || uri.resource_extension)
+        !!(
+            !resource_name.empty? ||
+            uri.resource_extension ||
+            uri.resource_name.to_s.include?( '~' )
+        )
+    end
+
+    # If this is neither a regular 404 nor a 202 the server probably freaked out
+    # -- 500 errors under stress and the like.
+    #
+    # In that case we should bail out to avoid corrupted signatures which can
+    # lead to FPs.
+    def corrupted_response?( response )
+        response.code != 404 && response.code != 200
     end
 
     def url_for( url )
@@ -387,9 +396,32 @@ class Dynamic404Handler
             probes << proc { up_to_path + random_string + '.' + resource_extension }
         end
 
+        if uri.resource_name.include?( '~' )
+            probes << proc {
+                up_to_path.sub(
+                    uri.resource_name,
+                    resource_name.gsub( '~', '~~' )
+                )
+            }
+        end
+
         probes
     end
 
+    def request( url, &block )
+        Client.get( url,
+            # This is important, helps us reduce waiting callers.
+            high_priority: true,
+
+            # We're going to be checking for a lot of non-existent resources,
+            # don't bother fingerprinting them
+            fingerprint:   false,
+
+            performer:     self,
+            &block
+        )
+    end
+
     def data_for( url )
         @signatures[url_for( url )] ||= signature_prototype
     end

data/lib/arachni/http/headers.rb

@@ -18,13 +18,28 @@ module HTTP
 # @author Tasos Laskos <tasos.laskos@arachni-scanner.com>
 class Headers < Hash
 
+    FORMATTED_NAMES_CACHE = Support::Cache::LeastRecentlyPushed.new( 100 )
+
+    CONTENT_TYPE = 'content-type'
+    SET_COOKIE   = 'set-cookie'
+    LOCATION     = 'location'
+
     # @param  [Headers, Hash] headers
     def initialize( headers = {} )
         merge!( headers || {} )
     end
 
     def merge!( headers )
-        headers.each { |k, v| self[k] = v }
+        headers.each do |k, v|
+            # Handle headers with identical normalized names, like a mixture of
+            # Set-Cookie and SET-COOKIE.
+            if include? k
+                self[k] = [self[k]].flatten
+                self[k] << v
+            else
+                self[k] = v
+            end
+        end
     end
 
     # @note `field` will be capitalized appropriately before storing.
@@ -77,20 +92,20 @@ class Headers < Hash
     # @return   [String, nil]
     #   Value of the `Content-Type` field.
     def content_type
-        self['content-type']
+        self[CONTENT_TYPE]
     end
 
     # @return   [String, nil]
     #   Value of the `Location` field.
     def location
-        self['location']
+        self[LOCATION]
     end
 
     # @return   [Array<String>]
     #   Set-cookie strings.
     def set_cookie
-        return [] if self['set-cookie'].to_s.empty?
-        [self['set-cookie']].flatten
+        return [] if self[SET_COOKIE].to_s.empty?
+        [self[SET_COOKIE]].flatten
     end
 
     # @return   [Array<Hash>]
@@ -119,15 +134,12 @@ class Headers < Hash
     end
 
     def self.format_field_name( field )
-        # return field
-
         # If there's a '--' somewhere in there then skip it, it probably is an
         # audit payload.
         return field if field.include?( '--' )
 
-        @formatted ||= Hash.new
-        @formatted[field.downcase.hash] ||=
-            field.to_s.split( '-' ).map( &:capitalize ).join( '-' )
+        FORMATTED_NAMES_CACHE[field] ||=
+            field.split( '-' ).map( &:capitalize ).join( '-' )
     end
 
 end

data/lib/arachni/http/proxy_server.rb

@@ -21,11 +21,17 @@ module HTTP
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 class ProxyServer < WEBrick::HTTPProxyServer
 
-    INTERCEPTOR_CERTIFICATE =
-        File.dirname( __FILE__ ) + '/proxy_server/ssl-interceptor-cert.pem'
+    CACHE = {
+        format_field_name: Support::Cache::LeastRecentlyPushed.new( 100 )
+    }
 
-    INTERCEPTOR_PRIVATE_KEY =
-        File.dirname( __FILE__ ) + '/proxy_server/ssl-interceptor-pkey.pem'
+    SKIP_HEADERS = Set.new( HopByHop | ['content-encoding'] )
+
+    INTERCEPTOR_CA_CERTIFICATE =
+        File.dirname( __FILE__ ) + '/proxy_server/ssl-interceptor-cacert.pem'
+
+    INTERCEPTOR_CA_KEY =
+        File.dirname( __FILE__ ) + '/proxy_server/ssl-interceptor-cakey.pem'
 
     # @param   [Hash]  options
     # @option options   [String]    :address    ('0.0.0.0')
@@ -168,8 +174,12 @@ class ProxyServer < WEBrick::HTTPProxyServer
     # @see #service
     # @see Webrick::HTTPProxyServer#service
     def do_CONNECT( req, res )
+        host = req.unparsed_uri.split(':').first
+
         req.instance_variable_set( :@unparsed_uri, "127.0.0.1:#{interceptor_port}" )
-        start_ssl_interceptor
+
+        start_ssl_interceptor( host )
+
         super( req, res )
     end
 
@@ -177,8 +187,10 @@ class ProxyServer < WEBrick::HTTPProxyServer
     #   Merges the given HTTP options with some default ones.
     def http_opts( options = {} )
         options.merge(
+            performer:         self,
+
             # Don't follow redirects, the client should handle this.
-            follow_location: false,
+            follow_location:   false,
 
             # Set the HTTP request timeout.
             timeout:           @options[:timeout],
@@ -198,18 +210,54 @@ class ProxyServer < WEBrick::HTTPProxyServer
     # Starts the SSL interceptor proxy server.
     #
     # The interceptor will listen on {#interceptor_port}.
-    def start_ssl_interceptor
+    def start_ssl_interceptor( host )
         return @interceptor if @interceptor
 
+        ca     = OpenSSL::X509::Certificate.new( File.read( INTERCEPTOR_CA_CERTIFICATE ) )
+        ca_key = OpenSSL::PKey::RSA.new( File.read( INTERCEPTOR_CA_KEY ) )
+
+        keypair = OpenSSL::PKey::RSA.new( 4096 )
+
+        req            = OpenSSL::X509::Request.new
+        req.version    = 0
+        req.subject    = OpenSSL::X509::Name.parse(
+            "CN=#{host}/subjectAltName=#{host}/O=Arachni/OU=Proxy/L=Athens/ST=Attika/C=GR"
+        )
+        req.public_key = keypair.public_key
+        req.sign( keypair, OpenSSL::Digest::SHA1.new )
+
+        cert            = OpenSSL::X509::Certificate.new
+        cert.version    = 2
+        cert.serial     = rand( 999999 )
+        cert.not_before = Time.new
+        cert.not_after  = cert.not_before + (60 * 60 * 24 * 365)
+        cert.public_key = req.public_key
+        cert.subject    = req.subject
+        cert.issuer     = ca.subject
+
+        ef = OpenSSL::X509::ExtensionFactory.new
+        ef.subject_certificate = cert
+        ef.issuer_certificate  = ca
+
+        cert.extensions = [
+            ef.create_extension( 'basicConstraints', 'CA:FALSE', true ),
+            ef.create_extension( 'extendedKeyUsage', 'serverAuth', false ),
+            ef.create_extension( 'subjectKeyIdentifier', 'hash' ),
+            ef.create_extension( 'authorityKeyIdentifier', 'keyid:always,issuer:always' ),
+            ef.create_extension( 'keyUsage',
+                'nonRepudiation,digitalSignature,keyEncipherment,dataEncipherment',
+                true
+            )
+        ]
+        cert.sign( ca_key, OpenSSL::Digest::SHA1.new )
+
         # The interceptor is only used for SSL decryption/encryption, the actual
         # proxy functionality is forwarded to the plain proxy server.
         @interceptor = self.class.new(
             address:        '127.0.0.1',
             port:            interceptor_port,
-            ssl_certificate:
-                OpenSSL::X509::Certificate.new( File.read( INTERCEPTOR_CERTIFICATE ) ),
-            ssl_private_key:
-                OpenSSL::PKey::RSA.new( File.read( INTERCEPTOR_PRIVATE_KEY ) ),
+            ssl_certificate: cert,
+            ssl_private_key: keypair,
             service_handler: method( :proxy_service )
         )
 
@@ -283,24 +331,21 @@ class ProxyServer < WEBrick::HTTPProxyServer
     # @param    [#[]=]    dst
     #   Headers of the forwarded/proxy response.
     def choose_header( src, dst )
-        connections = split_field( [src['connection']].flatten.first )
+        connections = Set.new( split_field( [src['connection']].flatten.first ) )
 
         src.each do |key, value|
             key = key.downcase
+            next if SKIP_HEADERS.include?( key ) || connections.include?( key )
 
-            if HopByHop.member?( key )          || # RFC2616: 13.5.1
-                connections.member?( key )      || # RFC2616: 14.10
-                key == 'content-encoding'
-                @logger.debug( "choose_header: `#{key}: #{value}'" )
-                next
-            end
-
-            field = key.to_s.split( /_|-/ ).
-                map { |segment| segment.capitalize }.join( '-' )
-            dst[field] = value
+            dst[self.class.format_field_name( key )] = value
         end
     end
 
+    def self.format_field_name( field )
+        CACHE[:format_field_name][field] ||=
+            field.split( /_|-/ ).map( &:capitalize ).join( '-' )
+    end
+
 end
 end
 end

data/lib/arachni/http/proxy_server/ssl-interceptor-cacert.pem

@@ -0,0 +1,34 @@
+-----BEGIN CERTIFICATE-----
+MIIF6zCCA9OgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBjzELMAkGA1UEBhMCR1Ix
+DzANBgNVBAgMBkF0dGlrYTEPMA0GA1UEBwwGQXRoZW5zMRAwDgYDVQQKDAdBcmFj
+aG5pMQ4wDAYDVQQLDAVQcm94eTEQMA4GA1UEAwwHQXJhY2huaTEqMCgGCSqGSIb3
+DQEJARYbYXJhY2huaUBhcmFjaG5pLXNjYW5uZXIuY29tMB4XDTE1MDYxNDAyMDgz
+MVoXDTI1MDYxMTAyMDgzMVowgY8xCzAJBgNVBAYTAkdSMQ8wDQYDVQQIDAZBdHRp
+a2ExDzANBgNVBAcMBkF0aGVuczEQMA4GA1UECgwHQXJhY2huaTEOMAwGA1UECwwF
+UHJveHkxEDAOBgNVBAMMB0FyYWNobmkxKjAoBgkqhkiG9w0BCQEWG2FyYWNobmlA
+YXJhY2huaS1zY2FubmVyLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoC
+ggIBAKfH1iQiuG8zYS514F8zZLp8//00gPIML7+wcJn4cw2iN+yEix6RuZEIzqva
+TfC4H6lZTyUhsoGZCeYAzzH9BMri/9uHJF+4worPfVKYIsm3TnMOVYoIC1kP1/Gj
+Y1ih8KI/3baw0pddtJJeQ5/GjDaxx4+ynY4ZxrNcFmbTYXSrPcd62V/D4+edVnLi
+uQsUezWYx7gNFiAuPRtlgJVBwzRoPV+Fh7Es2/SfmNSfGBCCYOpj4Fh0GOv7pSV0
+9TeF1W/XqDoq/eZ7RzLXoFK0Rz70/22MnFWAIdEUHZqwh3ktndNEK2QHq9FRGUx8
+cUlXVAJgYv8tTErYVBltKIi2qgbnkh0Rb+rT2OkgmSL9lg0PwpXChMeSo6o6riC7
+5a8PQi6OmIseY742QYmBXApXDHtSzaY8onHUvqgxFrFpP0Bmca3AoF6kWQfXfRwS
+ClMLwfBBDVeb+Tt97MO1G4m2VEW6c7o9H1t4td55LGslUzfJrmFe99vjAtdRTVqG
+t3qDjbYi5VpE9kIyKcPHZkSKelMQ4VO1qB14CdaK/3ufqHTk7Ro2hKgstKDqnTCF
+R7Qb9yXFsb1QyNtW8898T5mQm0HWQxdkaxcodizVjY5inHgqNwPa7A469EYFm5in
+dLSOQtdPOV4q+y5lfhA2MkE3pRdSZPpnTqCEkSVVoKfdVlftAgMBAAGjUDBOMB0G
+A1UdDgQWBBRvmR7gGqIfTQB0GygwgI22Kyr1bDAfBgNVHSMEGDAWgBRvmR7gGqIf
+TQB0GygwgI22Kyr1bDAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4ICAQAx
+g+ZjxJZXW1dYkc9ItXwAZba7oQJapLPu1iWCFy5cU13gck2MwDqfaDApNdr+erHg
+WN7N+smMO+x3+lZZptzTfc6g/hBthBBAnetj8CUehjnWCo3aBGgVLE/mIEyHyFym
+JX6xgcNYpvEzHT2o3Kmu/dAHCqY/3P9NtGJMhf7fy/Zz72tGY+ZTlthFSGWOjIEV
+KXTtYnRUKmIRBLMacZmrJKIZCp/qGVSnFh9yjxHTWPNXXngGMxF9ItsFbdakjefn
+hi2sHqns6/YbMaD2wK42dRQH1wH66DCGbyDPQO2j8iGK1q4Ggps+mGNYNBzMSAO/
+ybdGRLQNq8ag7RXr/tNp/jYHopS/Ga0+3bOnCKf6MXNOolknSZhsOo16BWKDRd+d
+m9ZTlro9AQr9+jdychG41IQNHXySrC5F1jLtzpEE5CJZIXkEFNYRcO9HMByJ3qwG
+759oYcMklwhU+NSC5qXpD2Z9KGf5rc0HmoO6OyD4T8hnQXkuAqoIN/NBg6YSNisN
+H2C2gbl+taRLt0/RVCiacylo5pl3XSZuQxtGaQl55gRXQDPnlfB2CtIrV44gHZOJ
+88s+Ld9h44aoT2rWbLld6dU5ElZXWEJOim+aYKJewxX7PwEHn4iCpvMLu+4jXH3j
+OkDTHheVJkxyhTDQ43ebg3/qi4yFaQyAHk3bQItwCw==
+-----END CERTIFICATE-----

data/lib/arachni/http/proxy_server/ssl-interceptor-cakey.pem

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAp8fWJCK4bzNhLnXgXzNkunz//TSA8gwvv7BwmfhzDaI37ISL
+HpG5kQjOq9pN8LgfqVlPJSGygZkJ5gDPMf0EyuL/24ckX7jCis99UpgiybdOcw5V
+iggLWQ/X8aNjWKHwoj/dtrDSl120kl5Dn8aMNrHHj7KdjhnGs1wWZtNhdKs9x3rZ
+X8Pj551WcuK5CxR7NZjHuA0WIC49G2WAlUHDNGg9X4WHsSzb9J+Y1J8YEIJg6mPg
+WHQY6/ulJXT1N4XVb9eoOir95ntHMtegUrRHPvT/bYycVYAh0RQdmrCHeS2d00Qr
+ZAer0VEZTHxxSVdUAmBi/y1MSthUGW0oiLaqBueSHRFv6tPY6SCZIv2WDQ/ClcKE
+x5KjqjquILvlrw9CLo6Yix5jvjZBiYFcClcMe1LNpjyicdS+qDEWsWk/QGZxrcCg
+XqRZB9d9HBIKUwvB8EENV5v5O33sw7UbibZURbpzuj0fW3i13nksayVTN8muYV73
+2+MC11FNWoa3eoONtiLlWkT2QjIpw8dmRIp6UxDhU7WoHXgJ1or/e5+odOTtGjaE
+qCy0oOqdMIVHtBv3JcWxvVDI21bzz3xPmZCbQdZDF2RrFyh2LNWNjmKceCo3A9rs
+Djr0RgWbmKd0tI5C1085Xir7LmV+EDYyQTelF1Jk+mdOoISRJVWgp91WV+0CAwEA
+AQKCAgAo/qXvDGC+IvKy1HBvMnKBMnul1YdQHPQpxSWuKUuLYECD1NrdLEQIEPvW
+d6+lioeJ7F1vOC2Sht8pSLdXgngCTravX/TeQpmeKxZ28N9HJDfR2wXBhTeomjts
+OjzS8jaGnk5BDjFWdLnjLY8eYffugT++d6kRiHDJcE208B8Wz6R3siecw5NTC1mN
+FqKZ93YnYV4jNWdbk5CwuftR/NCCZJniVhESlGBmA/zmrrzFg+XEP4UYd72DI2h1
+n38vAs9k1W+wTsLc5vA9lvwAWTYzRs+GZ93m8jjRCjY1jr57OE8gyL5FYa50pXkl
+/B3+Co1nSz/FE79ZZkQeNlK6HM+sHM5K0UILW+lKcDci+ABGH9mwkEtJwvg6XkhZ
+2iGyBaXtEObzaDugZoNuttAURbA32r9kRJNtPXP/Q0/fkQHTpd70iq9ax0Ztdh3H
+hatt9VxN3h+NVl4xfZrBJdQYUrxb6wrraABnz7QUmRpHMulgaSnAIXe4MvZEleKe
+tAZg9sIRzzYy1bkQLdwgwTHuNNUm6usiK7HRBv94R4PycPsXTAq8CSGp0FheSW+S
+QmFCeJhaEaNGRC5mp5Wk7sycVrKDu63ch8cABdFewTuB7LZCnviY/9CGdfu+LFgD
+bQGEqq+OxHXOr6xb5c7W5LY7oxWafab6OohHPOulcxEIOF3NzQKCAQEA2gvN9i97
++ntPXg4q4uk3rq6QpLke+q6MjM289xaNgkN2fQthxAzo3bK7yc7+zKbmKXIxrb0n
+GWWY/xTci6NS8W44YxdwZNB0LORa6gkpH3znsYLa4HsOYQTtLDu/6rOwE+EUwPA/
+UUp1TaseD0+eV8Jrz8whqdU33wUSs2mi6R7kYcb8J9PXnlHz9qw3TuPPug2Tt7Q8
+AmydJ/XiniEN2gF0DS3+99cbqLb2C7CeX7W+E8sxYRs2tQliWIMYr1iXDkKRi2Qb
+KnUQWD/N4WwNYQ6mwYKwc6J1T37Ucp5FWYwVBqa5vV1KbrXb4JLJCWqwsy+mpE+i
+wlfVVKgIi/j7ywKCAQEAxPwv8GBx84amUVBLPAUo5LPFhTOvZmHGNMNvYjVdk5Ud
+B37M3XbQMiTKQoeDII4Rr1cnkLUDm4eqgROkmlBAJZB9QLIrc87Hre8jW3eucU8y
+kVc90yUprc7WmvOcF1zilvjcNbt2gsVlmhbWuyqqn1aWfzvxjzqUXW6Xju0jD0wi
+a5qeMOVhJXrSTdy0gjZ7qg4BVWr01rIAuqifBKt7En9ynxqI4XEyzK9RYpex3ek1
+yJzVAW3fn/HN1pKpBLS6QOsUtqWQDQKGZM6zYDR49mnUuTWYkhh3pXeHQ3uNsJwR
+wS+FPu8YaiodGLXclwTmLZz093D7eChsoAjDvvB0JwKCAQEAtOVkOyFL5xQUVYDF
+fblkk8yJfc+DbxAO1OX/JrMUNYUIsVcXBhJ7wyn8d8H+TAUPIEV4B57M6FoMo1tI
+WaTnNBtwNm2Etm7mYzQUZOOytUfn5LIeKmyNElqG9dKgNvRaWTO8BxGKRkPSq9wS
+NTulr0NCNIQzTXXyQ1kvGZ/DI0qYyLHQEq7CzLtK/lQEErQXa1DGQ3sI6i339+Yb
+23qqxjm8cQ6+4Bka/k7ENBCUY+0gw8Uos1pjebBOYgZpHVgPAiqiGxWzH/c81yog
+ASumseX43MQy5cxbLNeZI3pBKLh53SnHIN5b2RuRTnAYz3IvJImc4+aZrkg2WWSK
+qq2nHwKCAQAaqR8743HIygKcosdr+i7MtWAYZSRqMPWIkqLyodJmdRoWt5y2pKwM
+/Vm6o2il8VSHbL5YIYe5dyUmjygKEq575xBsvzCOXgA8lE8uxAYCI/vuG+asOy1m
+7sWw9yO7LcElOc1kIFkr3deggVLSxjWNl0SLN+u7vOvzsVIl8AZ8vYszERwz9feu
+AO+RxjtQHFukanzXuMAmhrT+jm/nS+Y+XK2AxzCbgpyjg176fxl9tWCoJEHYDazk
+ku+PCQ6DKorC2o5VIhdbC2pxHmC8tp1gjHZUEuLxcwpOhNzzzzcgHh9xDCN2nxmo
+1MZXX9XZQrp8le+5xbrjSmVZS5Zis1ylAoIBADOMSmu//rdDwCwptRByopmLiE+S
+2AayD1Xk7X9YjkotXNYttOfnnnXq6pyEj4X0c0ISL9MkyADJ6+mx5GWH6yQlWIjo
+T00AcL5//IreAIRGluUhkFeI45QvgFfinKRiIN9YzAqhNHCEM7lEYGhMygD0OK0Y
+ZluvUvYshFLXbZA7+rYCzLM5FgeY2dxMJ4lIiXZwC5cbE95mf6bGlGb8/deBp0eW
+iGVyOSoY/Eh6qDDrQV4FOFRVFg7+9CKr8VDNizKTE6/JZFOb/F85QLwzx1zaJD1A
+FmGleWRh50XEaSAB0lA4LPWUl/m6r45bB03d9A6mx4axgl7ttjaIz6Vw9WQ=
+-----END RSA PRIVATE KEY-----

data/lib/arachni/http/request.rb

@@ -101,9 +101,6 @@ class Request < Message
     #   Maximum HTTP response size to accept, in bytes.
     attr_accessor :response_max_size
 
-    # @private
-    attr_accessor :root_redirect_id
-
     # @param  [Hash]  options
     #   Request options.
     # @option options [String] :url
@@ -129,6 +126,7 @@ class Request < Message
         super( options )
 
         @train           = false if @train.nil?
+        @fingerprint     = true  if @fingerprint.nil?
         @update_cookies  = false if @update_cookies.nil?
         @follow_location = false if @follow_location.nil?
         @max_redirects   = (Options.http.request_redirect_limit || REDIRECT_LIMIT)
@@ -265,6 +263,13 @@ class Request < Message
         !!@follow_location
     end
 
+    # @return   [Bool]
+    #   `true` if the {Response} should be {Platform::Manager.fingerprint fingerprinted}
+    #   for platforms, `false` otherwise.
+    def fingerprint?
+        @fingerprint
+    end
+
     # @return   [Bool]
     #   `true` if the {Response} should be analyzed by the {Trainer}
     #   for new elements, `false` otherwise.
@@ -343,6 +348,10 @@ class Request < Message
 
             accept_encoding: 'gzip, deflate',
             nosignal:        true,
+
+            # If Content-Length is missing this option will have no effect, so
+            # we'll also stream the body to make sure that we can at least abort
+            # the reading of the response body if it exceeds this limit.
             maxfilesize:     max_size,
 
             # Don't keep the socket alive if this is a blocking request because
@@ -384,17 +393,25 @@ class Request < Message
             end
         end
 
-        curl = parsed_url.query ? url.gsub( "?#{parsed_url.query}", '' ) : url
-        r = Typhoeus::Request.new( curl, options )
+        curl             = parsed_url.query ? url.gsub( "?#{parsed_url.query}", '' ) : url
+        typhoeus_request = Typhoeus::Request.new( curl, options )
 
         if @on_complete.any?
-            r.on_complete do |typhoeus_response|
+            response_body_buffer = ''
+            set_body_reader( typhoeus_request, response_body_buffer )
+
+            typhoeus_request.on_complete do |typhoeus_response|
+                if typhoeus_request.options[:maxfilesize]
+                    typhoeus_response.options[:response_body] =
+                        response_body_buffer
+                end
+
                 fill_in_data_from_typhoeus_response typhoeus_response
                 handle_response Response.from_typhoeus( typhoeus_response )
             end
         end
 
-        r
+        typhoeus_request
     end
 
     def to_h
@@ -480,33 +497,69 @@ class Request < Message
                 h
             end
         end
+
+        def encode( string )
+            @easy ||= Ethon::Easy.new( url: 'www.example.com' )
+            @easy.escape string
+        end
     end
 
     def prepare_headers
-        headers['Cookie'] = effective_cookies.
-            map { |k, v| "#{Cookie.encode( k )}=#{Cookie.encode( v )}" }.
-            join( ';' )
-        headers.delete( 'Cookie' ) if headers['Cookie'].empty?
-
         headers['User-Agent'] ||= Options.http.user_agent
         headers['Accept']     ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
         headers['From']       ||= Options.authorized_by if Options.authorized_by
 
         headers.each { |k, v| headers[k] = Header.encode( v ) if v }
+
+        headers['Cookie'] = effective_cookies.
+            map { |k, v| "#{Cookie.encode( k )}=#{Cookie.encode( v )}" }.
+            join( ';' )
+        headers.delete( 'Cookie' ) if headers['Cookie'].empty?
+
+        headers
     end
 
     private
 
+    def client_run
+        typhoeus_request = to_typhoeus
+
+        response_body_buffer = ''
+        set_body_reader( typhoeus_request, response_body_buffer )
+
+        typhoeus_response = typhoeus_request.run
+
+        if typhoeus_request.options[:maxfilesize]
+            typhoeus_response.options[:response_body] = response_body_buffer
+        end
+
+        fill_in_data_from_typhoeus_response typhoeus_response
+
+        Response.from_typhoeus( typhoeus_response )
+    end
+
     def fill_in_data_from_typhoeus_response( response )
-        @headers_string = response.debug_info.header_out.first
-        @effective_body = response.debug_info.data_out.first
+        # Only grab the last data.
+        # In case of CONNECT calls for HTTPS via proxy the first data will be
+        # the proxy-related stuff.
+        @headers_string = response.debug_info.header_out.last
+        @effective_body = response.debug_info.data_out.last
     end
 
-    def client_run
-        response = to_typhoeus.run
-        fill_in_data_from_typhoeus_response response
+    def set_body_reader( typhoeus_request, buffer )
+        return if !typhoeus_request.options[:maxfilesize]
 
-        Response.from_typhoeus( response )
+        aborted = nil
+        typhoeus_request.on_body do |chunk|
+            next aborted if aborted
+
+            if buffer.size >= typhoeus_request.options[:maxfilesize]
+                buffer.clear
+                next aborted = :abort
+            end
+
+            buffer << chunk
+        end
     end
 
 end