arachni 1.4 → 1.6.0

data/lib/arachni/http/proxy_server/ssl-interceptor-cakey.pem

@@ -1,51 +1,30 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIJKAIBAAKCAgEAp8fWJCK4bzNhLnXgXzNkunz//TSA8gwvv7BwmfhzDaI37ISL
-HpG5kQjOq9pN8LgfqVlPJSGygZkJ5gDPMf0EyuL/24ckX7jCis99UpgiybdOcw5V
-iggLWQ/X8aNjWKHwoj/dtrDSl120kl5Dn8aMNrHHj7KdjhnGs1wWZtNhdKs9x3rZ
-X8Pj551WcuK5CxR7NZjHuA0WIC49G2WAlUHDNGg9X4WHsSzb9J+Y1J8YEIJg6mPg
-WHQY6/ulJXT1N4XVb9eoOir95ntHMtegUrRHPvT/bYycVYAh0RQdmrCHeS2d00Qr
-ZAer0VEZTHxxSVdUAmBi/y1MSthUGW0oiLaqBueSHRFv6tPY6SCZIv2WDQ/ClcKE
-x5KjqjquILvlrw9CLo6Yix5jvjZBiYFcClcMe1LNpjyicdS+qDEWsWk/QGZxrcCg
-XqRZB9d9HBIKUwvB8EENV5v5O33sw7UbibZURbpzuj0fW3i13nksayVTN8muYV73
-2+MC11FNWoa3eoONtiLlWkT2QjIpw8dmRIp6UxDhU7WoHXgJ1or/e5+odOTtGjaE
-qCy0oOqdMIVHtBv3JcWxvVDI21bzz3xPmZCbQdZDF2RrFyh2LNWNjmKceCo3A9rs
-Djr0RgWbmKd0tI5C1085Xir7LmV+EDYyQTelF1Jk+mdOoISRJVWgp91WV+0CAwEA
-AQKCAgAo/qXvDGC+IvKy1HBvMnKBMnul1YdQHPQpxSWuKUuLYECD1NrdLEQIEPvW
-d6+lioeJ7F1vOC2Sht8pSLdXgngCTravX/TeQpmeKxZ28N9HJDfR2wXBhTeomjts
-OjzS8jaGnk5BDjFWdLnjLY8eYffugT++d6kRiHDJcE208B8Wz6R3siecw5NTC1mN
-FqKZ93YnYV4jNWdbk5CwuftR/NCCZJniVhESlGBmA/zmrrzFg+XEP4UYd72DI2h1
-n38vAs9k1W+wTsLc5vA9lvwAWTYzRs+GZ93m8jjRCjY1jr57OE8gyL5FYa50pXkl
-/B3+Co1nSz/FE79ZZkQeNlK6HM+sHM5K0UILW+lKcDci+ABGH9mwkEtJwvg6XkhZ
-2iGyBaXtEObzaDugZoNuttAURbA32r9kRJNtPXP/Q0/fkQHTpd70iq9ax0Ztdh3H
-hatt9VxN3h+NVl4xfZrBJdQYUrxb6wrraABnz7QUmRpHMulgaSnAIXe4MvZEleKe
-tAZg9sIRzzYy1bkQLdwgwTHuNNUm6usiK7HRBv94R4PycPsXTAq8CSGp0FheSW+S
-QmFCeJhaEaNGRC5mp5Wk7sycVrKDu63ch8cABdFewTuB7LZCnviY/9CGdfu+LFgD
-bQGEqq+OxHXOr6xb5c7W5LY7oxWafab6OohHPOulcxEIOF3NzQKCAQEA2gvN9i97
-+ntPXg4q4uk3rq6QpLke+q6MjM289xaNgkN2fQthxAzo3bK7yc7+zKbmKXIxrb0n
-GWWY/xTci6NS8W44YxdwZNB0LORa6gkpH3znsYLa4HsOYQTtLDu/6rOwE+EUwPA/
-UUp1TaseD0+eV8Jrz8whqdU33wUSs2mi6R7kYcb8J9PXnlHz9qw3TuPPug2Tt7Q8
-AmydJ/XiniEN2gF0DS3+99cbqLb2C7CeX7W+E8sxYRs2tQliWIMYr1iXDkKRi2Qb
-KnUQWD/N4WwNYQ6mwYKwc6J1T37Ucp5FWYwVBqa5vV1KbrXb4JLJCWqwsy+mpE+i
-wlfVVKgIi/j7ywKCAQEAxPwv8GBx84amUVBLPAUo5LPFhTOvZmHGNMNvYjVdk5Ud
-B37M3XbQMiTKQoeDII4Rr1cnkLUDm4eqgROkmlBAJZB9QLIrc87Hre8jW3eucU8y
-kVc90yUprc7WmvOcF1zilvjcNbt2gsVlmhbWuyqqn1aWfzvxjzqUXW6Xju0jD0wi
-a5qeMOVhJXrSTdy0gjZ7qg4BVWr01rIAuqifBKt7En9ynxqI4XEyzK9RYpex3ek1
-yJzVAW3fn/HN1pKpBLS6QOsUtqWQDQKGZM6zYDR49mnUuTWYkhh3pXeHQ3uNsJwR
-wS+FPu8YaiodGLXclwTmLZz093D7eChsoAjDvvB0JwKCAQEAtOVkOyFL5xQUVYDF
-fblkk8yJfc+DbxAO1OX/JrMUNYUIsVcXBhJ7wyn8d8H+TAUPIEV4B57M6FoMo1tI
-WaTnNBtwNm2Etm7mYzQUZOOytUfn5LIeKmyNElqG9dKgNvRaWTO8BxGKRkPSq9wS
-NTulr0NCNIQzTXXyQ1kvGZ/DI0qYyLHQEq7CzLtK/lQEErQXa1DGQ3sI6i339+Yb
-23qqxjm8cQ6+4Bka/k7ENBCUY+0gw8Uos1pjebBOYgZpHVgPAiqiGxWzH/c81yog
-ASumseX43MQy5cxbLNeZI3pBKLh53SnHIN5b2RuRTnAYz3IvJImc4+aZrkg2WWSK
-qq2nHwKCAQAaqR8743HIygKcosdr+i7MtWAYZSRqMPWIkqLyodJmdRoWt5y2pKwM
-/Vm6o2il8VSHbL5YIYe5dyUmjygKEq575xBsvzCOXgA8lE8uxAYCI/vuG+asOy1m
-7sWw9yO7LcElOc1kIFkr3deggVLSxjWNl0SLN+u7vOvzsVIl8AZ8vYszERwz9feu
-AO+RxjtQHFukanzXuMAmhrT+jm/nS+Y+XK2AxzCbgpyjg176fxl9tWCoJEHYDazk
-ku+PCQ6DKorC2o5VIhdbC2pxHmC8tp1gjHZUEuLxcwpOhNzzzzcgHh9xDCN2nxmo
-1MZXX9XZQrp8le+5xbrjSmVZS5Zis1ylAoIBADOMSmu//rdDwCwptRByopmLiE+S
-2AayD1Xk7X9YjkotXNYttOfnnnXq6pyEj4X0c0ISL9MkyADJ6+mx5GWH6yQlWIjo
-T00AcL5//IreAIRGluUhkFeI45QvgFfinKRiIN9YzAqhNHCEM7lEYGhMygD0OK0Y
-ZluvUvYshFLXbZA7+rYCzLM5FgeY2dxMJ4lIiXZwC5cbE95mf6bGlGb8/deBp0eW
-iGVyOSoY/Eh6qDDrQV4FOFRVFg7+9CKr8VDNizKTE6/JZFOb/F85QLwzx1zaJD1A
-FmGleWRh50XEaSAB0lA4LPWUl/m6r45bB03d9A6mx4axgl7ttjaIz6Vw9WQ=
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-256-CBC,2A035C8A165173F5841E28DE1D0A3673
+
+3hM9SEm1c4Hhu4HPApkPtuA4sl34K0Ul46jH34Ep+rt1tPkC+rPSS+dmZffVOc+j
+eOHajJf0rJXxjJ3ugsnXqmoTIMTEVgTG81+RQ1DfF1S4HWmp0zyqEnltXzrUV22l
+H/B783dEBdaFnUbnAISNePqNuU4f5rNPzlSMZlZ9WtiTXsB6QMUrGaLyDBpVsG3S
+lNFiqeRsCumvheH5UXFECzN/jy+m569uYkla8W5DivjhLd4eg0KZsa0Z6NpdbFTL
+E7ZouXkm33UbyhIPgz8Z5PA9CVAbcVZEKmXoanHbvz0kUYZY6DTpm/E62dBoSSmn
+R9831JKJDcQ+VdGbsDrbwTMKqx8dezCk6S5XrBd6hp7eE5FjN4+8IU4XRs57B/X9
+rWDvzxBwKKRh1l6MRu1bSsmlKd1+pAXz9Bi+6goYQfn+LsWTluUWpB+HYSfT1/AN
+V9XNjbZ70nlmUJtRhgctAt1O6sWpG3gPkWuBlhWr5rngeQr7t/Q6N5BS8UJ5yq9L
+jhDjUT9aZDEUdqIIh+3WTAPO4uMHHIohq2AtFi1PXqjajGAj8sQlAsST/ePjNkLz
+A/jZ+M8wc8Q8pyVrci3m9m6Kme2JI9JbvuOvkr7aAe3B6NLfpBx6BR7yT7dmGDxu
+in3qph/23xRTj1FLG/QEbHFRTiFxTQhD6kximZk68D0/b5GEv9grXlZk53DSaZ/o
+MTlsC+UwniKtGw3BhKqlH1KE0sYkS0ivRyUeQTH6Vujg33/asdNGu1U4nhTtZZWb
+2y4DVK+ZW36bKudY1klWTS+C3E1kHI9pOYlvmdTIV8maU930AFRGSITZZ/3yLjgm
+2SLcs2YrPWLZVgPe5O0w1N2ushxUDnclxvUr5HDH+thndmCB498e/1YIMFzb+c4f
+QfJjvGlEtmo5LxSCodlhGJ9Uw/K1CTAefiXkcpUx8AZV6/UEaFSlVk7XW7e/vcDZ
+sUvLm1oZXCnOzQIv3++akf+kZnFrTuwnIaAtVaBczluJihUDEv69WPymJ3kVNJ8C
+vo/R2oXQrtwI4NjdZ2pg3tka3RUJ7q9yT7KTnjeAhI1ZzTV8MbsT+mxA8nDixVtS
+kXInhCkara1STTFt0rUsiuaZxV3/sK4TpVmkm+hrbeEfacaFq2ZtAO6dtfyPL+Kq
+U4GkjEzK6fZ7elFzZ+xISV7oS+PBcNk2pqKZxI5rg8JMtBAgiUnVmIVqGuD9nNzf
+sttPzjbw1S3ezC7hcmTNcKa9UzAUO4+RBmakiVMBIHYEPyXTGgvbsFUT6abXg6pb
+tvKNnMaBDkF+YP+QTcbo4kJ7f//e1XNHbtnGkLu9N+PL45vam7YQFpq3YpwYVl18
+PfjoYCWoR8CfUzXEdf7blURzjyvbGg1DQ/WskenfupmpdVtxsiqzEcEgzyOsWmL0
+e7kx4IrOI49VS0aePDUh92kCKLl9+wKNfD+UNEaYuXo0sqYBmlBNBdUBzljokzxC
+GIag0AYLoIMZe0ccEa3PpVuAa0uMMrfVujLx3bXY5PrVw69gEPwc+GZz8MgAQWX5
+BGxtnrv1qhe6RpIB0AibY3myO+qIlVEOszJHDoMK8J5MKwu50s5L8R0M+Y3JdGHi
 -----END RSA PRIVATE KEY-----

data/lib/arachni/http/proxy_server/ssl_interceptor.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -16,6 +16,7 @@ class SSLInterceptor < Connection
 
     include TLS
 
+    CA_PASSPHRASE = 'interceptor'
     CA_CERTIFICATE = File.dirname( __FILE__ ) + '/ssl-interceptor-cacert.pem'
     CA_KEY         = File.dirname( __FILE__ ) + '/ssl-interceptor-cakey.pem'
 
@@ -32,6 +33,7 @@ class SSLInterceptor < Connection
 
     def on_close( reason = nil )
         print_debug_level_3 "Closed because: [#{reason.class}] #{reason}"
+        @parent.mark_connection_inactive self
     end
 
     def start_tls
@@ -42,9 +44,9 @@ class SSLInterceptor < Connection
 
         if @role == :server
             ca     = OpenSSL::X509::Certificate.new( File.read( CA_CERTIFICATE ) )
-            ca_key = OpenSSL::PKey::RSA.new( File.read( CA_KEY ) )
+            ca_key = OpenSSL::PKey::RSA.new( File.read( CA_KEY ), CA_PASSPHRASE )
 
-            keypair = OpenSSL::PKey::RSA.new( 1024 )
+            keypair = OpenSSL::PKey::RSA.new( 2048 )
 
             req            = OpenSSL::X509::Request.new
             req.version    = 0
@@ -52,12 +54,12 @@ class SSLInterceptor < Connection
                 "CN=#{@origin_host}/subjectAltName=#{@origin_host}/O=Arachni/OU=Proxy/L=Athens/ST=Attika/C=GR"
             )
             req.public_key = keypair.public_key
-            req.sign( keypair, OpenSSL::Digest::SHA1.new )
+            req.sign( keypair, OpenSSL::Digest::SHA256.new )
 
             cert            = OpenSSL::X509::Certificate.new
             cert.version    = 2
             cert.serial     = rand( 999999 )
-            cert.not_before = Time.new
+            cert.not_before = Time.new - 600
             cert.not_after  = cert.not_before + (60 * 60 * 24 * 365)
             cert.public_key = req.public_key
             cert.subject    = req.subject
@@ -77,7 +79,7 @@ class SSLInterceptor < Connection
                                      true
                 )
             ]
-            cert.sign( ca_key, OpenSSL::Digest::SHA1.new )
+            cert.sign( ca_key, OpenSSL::Digest::SHA256.new )
 
             @ssl_context = OpenSSL::SSL::SSLContext.new
             @ssl_context.cert = cert

data/lib/arachni/http/proxy_server/tunnel.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -16,17 +16,17 @@ class Tunnel < Arachni::Reactor::Connection
     personalize_output
 
     def initialize( options )
-        print_debug_level_3 'New SSL tunnel.'
+        print_debug_level_3 'New tunnel.'
 
         @client = options[:client]
     end
 
     def on_connect
-        print_debug_level_3 'Connected to Interceptor.'
+        print_debug_level_3 'Connected.'
     end
 
     def write( data )
-        print_debug_level_3 " -> Forwarding #{data.size} bytes to Interceptor."
+        print_debug_level_3 " -> Forwarding #{data.size} bytes."
         super data
     end
 

data/lib/arachni/http/proxy_server.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -21,6 +21,8 @@ class ProxyServer
     include Arachni::UI::Output
     personalize_output
 
+    DEFAULT_CONCURRENCY = 4
+
     # @param   [Hash]  options
     # @option options   [String]    :address    ('0.0.0.0')
     #   Address to bind to.
@@ -28,6 +30,8 @@ class ProxyServer
     #   Port number to listen on -- defaults to a random port.
     # @option options   [Integer]    :timeout
     #   HTTP time-out for each request in milliseconds.
+    # @option options   [Integer]    :concurrency   (DEFAULT_CONCURRENCY)
+    #   Amount of origin requests to be active at any given time.
     # @option options   [Block]    :response_handler
     #   Block to be called to handle each response as it arrives -- will be
     #   passed the request and response.
@@ -35,42 +39,67 @@ class ProxyServer
     #   Block to be called to handle each request as it arrives -- will be
     #   passed the request and response.
     def initialize( options = {} )
-        @reactor = Arachni::Reactor.new
+        @reactor = Arachni::Reactor.new(
+            # Higher than the defaults to keep object allocations down.
+            select_timeout:    0.1,
+            max_tick_interval: 0.1
+        )
         @options = options
 
-        @active_connections = Set.new
+        @active_connections = Concurrent::Map.new
 
-        @options[:address] ||= '127.0.0.1'
-        @options[:port]    ||= Utilities.available_port
+        @options[:concurrency] ||= DEFAULT_CONCURRENCY
+        @options[:address]     ||= '127.0.0.1'
+        @options[:port]        ||= Utilities.available_port
+
+        @concurrency_control_tokens = @reactor.create_queue
     end
 
     # Starts the server without blocking, it'll only block until the server is
     # up and running and ready to accept connections.
     def start_async
-        print_debug_level_2 'Starting'
+        print_debug_level_2 'Starting...'
 
         @reactor.run_in_thread
 
+        @options[:concurrency].times do |i|
+            @concurrency_control_tokens << i
+        end
+
         @reactor.on_error do |_, e|
             print_exception e
         end
 
-        listener = @reactor.listen(
+        @reactor.listen(
             @options[:address], @options[:port], Connection,
             @options.merge( parent: self )
         )
 
-        print_debug_level_2 'Started'
+        print_debug_level_2 "...started at: #{url}"
         nil
     end
 
+    def get_request_token( &block )
+        @concurrency_control_tokens.pop( &block )
+    end
+
+    def return_request_token( token )
+        @concurrency_control_tokens << token
+    end
+
+    def has_available_request_tokens?
+        @concurrency_control_tokens.empty?
+    end
+
     def shutdown
-        print_debug_level_2 'Shutting down..'
+        print_debug_level_2 'Shutting down...'
+
+        @thread_pool.kill if @thread_pool
 
         @reactor.stop
         @reactor.wait
 
-        print_debug_level_2 'Shutdown.'
+        print_debug_level_2 '...shutdown.'
     end
 
     # @return   [Bool]
@@ -97,8 +126,12 @@ class ProxyServer
         @active_connections.size
     end
 
+    def active_connections
+        @active_connections.keys
+    end
+
     def mark_connection_active( connection )
-        @active_connections << connection
+        @active_connections.put_if_absent( connection, nil )
     end
 
     def mark_connection_inactive( connection )

data/lib/arachni/http/request/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/http/request.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -13,9 +13,12 @@ module HTTP
 #
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 class Request < Message
+    include Utilities
+    include UI::Output
+
     require_relative 'request/scope'
 
-    ENCODE_CACHE = Support::Cache::LeastRecentlyPushed.new( 10_000 )
+    ENCODE_CACHE = Support::Cache::LeastRecentlyPushed.new( 1_000 )
 
     # Default redirect limit, RFC says 5 max.
     REDIRECT_LIMIT = 5
@@ -110,6 +113,12 @@ class Request < Message
     #   Parameters which should not be encoded, by name.
     attr_accessor :raw_parameters
 
+    # @return   [Response]
+    attr_accessor :response
+
+    # @private
+    attr_accessor :response_body_buffer
+
     # @param  [Hash]  options
     #   Request options.
     # @option options [String] :url
@@ -139,7 +148,12 @@ class Request < Message
         @update_cookies  = false if @update_cookies.nil?
         @follow_location = false if @follow_location.nil?
         @max_redirects   = (Options.http.request_redirect_limit || REDIRECT_LIMIT)
-        @on_complete     = []
+
+        @on_headers    = []
+        @on_body       = []
+        @on_body_line  = []
+        @on_body_lines = []
+        @on_complete   = []
 
         @raw_parameters ||= []
         @timeout        ||= Options.http.request_timeout
@@ -287,6 +301,12 @@ class Request < Message
         s << '>'
     end
 
+    def on_headers( &block )
+        fail 'Block is missing.' if !block_given?
+        @on_headers << block
+        self
+    end
+
     # @note Can be invoked multiple times.
     #
     # @param    [Block] block
@@ -297,9 +317,31 @@ class Request < Message
         self
     end
 
+    def on_body( &block )
+        fail 'Block is missing.' if !block_given?
+        @on_body << block
+        self
+    end
+
+    def on_body_line( &block )
+        fail 'Block is missing.' if !block_given?
+        @on_body_line << block
+        self
+    end
+
+    def on_body_lines( &block )
+        fail 'Block is missing.' if !block_given?
+        @on_body_lines << block
+        self
+    end
+
     # Clears {#on_complete} callbacks.
     def clear_callbacks
         @on_complete.clear
+        @on_body.clear
+        @on_headers.clear
+        @on_body_line.clear
+        @on_body_lines.clear
     end
 
     # @return   [Bool]
@@ -322,6 +364,10 @@ class Request < Message
         @train
     end
 
+    def buffered?
+        @on_body.any? || @on_body_line.any? || @on_body_lines.any?
+    end
+
     # Flags that the response should be analyzed by the {Trainer} for new
     # elements.
     def train
@@ -346,13 +392,7 @@ class Request < Message
     #
     # @return   [Response]
     def run
-        client_run.tap { |r| r.request = self }
-    end
-
-    def handle_response( response )
-        response.request = self
-        @on_complete.each { |b| b.call response }
-        response
+        client_run
     end
 
     # @return   [Typhoeus::Response]
@@ -432,7 +472,7 @@ class Request < Message
             options[:userpwd]  = ':'
             options[:httpauth] = :gssnegotiate
         else
-            options[:httpauth] = :auto
+            options[:httpauth] = Options.http.authentication_type.to_sym
         end
 
         if proxy
@@ -459,27 +499,172 @@ class Request < Message
 
         typhoeus_request = Typhoeus::Request.new( url.split( '?').first, options )
 
+        aborted = nil
+
+        # Always set this because we'll be streaming most of the time, so we
+        # should set @response so that there'll be a response available for the
+        # #on_body and #on_body_line callbacks.
+        typhoeus_request.on_headers do |typhoeus_response|
+            next aborted if aborted
+
+            set_response_data typhoeus_response
+
+            @on_headers.each do |on_header|
+                exception_jail false do
+                    if on_header.call( self.response ) == :abort
+                        break aborted = :abort
+                    end
+                end
+
+                next aborted if aborted
+            end
+        end
+
+        if @on_body.any?
+            typhoeus_request.on_body do |chunk|
+                next aborted if aborted
+
+                @on_body.each do |b|
+                    exception_jail false do
+                        chunk.recode!
+                        if b.call( chunk, self.response ) == :abort
+                            break aborted = :abort
+                        end
+                    end
+                end
+
+                next aborted if aborted
+            end
+        end
+
+        if @on_body_line.any?
+            line_buffer = ''
+            typhoeus_request.on_body do |chunk|
+                next aborted if aborted
+
+                chunk.recode!
+                line_buffer << chunk
+
+                lines = line_buffer.lines
+
+                @response_body_buffer = nil
+
+                # Incomplete last line, we've either read everything of were cut
+                # short, but we can't know which.
+                if !lines.last.index( /[\n\r]/, -1 )
+                    last_line = lines.pop
+
+                    # Set it as the generic body buffer in order to be accessible
+                    # via #on_complete in case this was indeed the end of the
+                    # response.
+                    @response_body_buffer = last_line.dup
+
+                    # Also push it back to out own buffer in case there's more
+                    # to read in order to complete the line.
+                    line_buffer = last_line
+                end
+
+                lines.each do |line|
+                    @on_body_line.each do |b|
+                        exception_jail false do
+                            if b.call( line, self.response ) == :abort
+                                break aborted = :abort
+                            end
+                        end
+                    end
+
+                    break aborted if aborted
+                end
+
+                line_buffer.clear
+
+                next aborted if aborted
+            end
+        end
+
+        if @on_body_lines.any?
+            lines_buffer = ''
+            typhoeus_request.on_body do |chunk|
+                next aborted if aborted
+
+                chunk.recode!
+                lines_buffer << chunk
+
+                lines, middle, remnant = lines_buffer.rpartition( /[\r\n]/ )
+                lines << middle
+
+                @response_body_buffer = nil
+
+                # Incomplete last line, we've either read everything of were cut
+                # short, but we can't know which.
+                if !remnant.empty?
+                    # Set it as the generic body buffer in order to be accessible
+                    # via #on_complete in case this was indeed the end of the
+                    # response.
+                    @response_body_buffer = remnant.dup
+
+                    # Also push it back to out own buffer in case there's more
+                    # to read in order to complete the line.
+                    lines_buffer = remnant
+                end
+
+                @on_body_lines.each do |b|
+                    exception_jail false do
+                        if b.call( lines, self.response ) == :abort
+                            break aborted = :abort
+                        end
+                    end
+                end
+
+                next aborted if aborted
+            end
+        end
+
         if @on_complete.any?
-            response_body_buffer = ''
-            set_body_reader( typhoeus_request, response_body_buffer )
+            # No need to set our own reader in order to enforce max response size
+            # if the response is already been read bit by bit via other callbacks.
+            if typhoeus_request.options[:maxfilesize] && @on_body.empty? &&
+                @on_body_line.empty? && @on_body_lines.empty?
+
+                @response_body_buffer = ''
+                set_body_reader( typhoeus_request, @response_body_buffer )
+            end
 
             typhoeus_request.on_complete do |typhoeus_response|
-                if typhoeus_request.options[:maxfilesize]
+                next aborted if aborted
+
+                # Set either by the default body reader or is a remnant from
+                # a user specified callback like #on_body, #on_body_line, etc.
+                if @response_body_buffer
                     typhoeus_response.options[:response_body] =
-                        response_body_buffer
+                        @response_body_buffer
                 end
 
-                fill_in_data_from_typhoeus_response typhoeus_response
-                handle_response Response.from_typhoeus(
-                    typhoeus_response,
-                    normalize_url: @normalize_url
-                )
+                set_response_data typhoeus_response
+
+                @on_complete.each do |b|
+                    exception_jail false do
+                        b.call self.response
+                    end
+                end
             end
         end
 
         typhoeus_request
     end
 
+    def set_response_data( typhoeus_response )
+        fill_in_data_from_typhoeus_response typhoeus_response
+
+        self.response = Response.from_typhoeus(
+            typhoeus_response,
+            normalize_url: @normalize_url,
+            request:       self
+        )
+
+        self.response.update_from_typhoeus typhoeus_response
+    end
+
     def to_h
         {
             url:            url,
@@ -501,13 +686,23 @@ class Request < Message
     end
 
     def marshal_dump
-        raw_cookies = @raw_cookies.dup
-        callbacks   = @on_complete.dup
-        performer   = @performer
-
-        @performer   = nil
-        @raw_cookies = []
-        @on_complete = []
+        raw_cookies   = @raw_cookies.dup
+        callbacks     = @on_complete.dup
+        on_body       = @on_body.dup
+        on_headers    = @on_headers.dup
+        on_body_line  = @on_body_line.dup
+        on_body_lines = @on_body_lines.dup
+        performer     = @performer
+        response      = @response
+
+        @performer     = nil
+        @response      = nil
+        @raw_cookies   = []
+        @on_complete   = []
+        @on_body       = []
+        @on_body_line  = []
+        @on_body_lines = []
+        @on_headers    = []
 
         instance_variables.inject( {} ) do |h, iv|
             next h if iv == :@scope
@@ -515,9 +710,14 @@ class Request < Message
             h
         end
     ensure
-        @raw_cookies = raw_cookies
-        @on_complete = callbacks
-        @performer   = performer
+        @response      = response
+        @raw_cookies   = raw_cookies
+        @on_complete   = callbacks
+        @on_body       = on_body
+        @on_body_line  = on_body_line
+        @on_body_lines = on_body_lines
+        @on_headers    = on_headers
+        @performer     = performer
     end
 
     def marshal_load( h )
@@ -630,20 +830,16 @@ class Request < Message
     private
 
     def client_run
-        typhoeus_request = to_typhoeus
+        # Set #on_complete so that the #response will be set.
+        on_complete {}
 
-        response_body_buffer = ''
-        set_body_reader( typhoeus_request, response_body_buffer )
+        treq = self.to_typhoeus
 
-        typhoeus_response = typhoeus_request.run
+        hydra = (Thread.current[:client_run_hydra] ||= Typhoeus::Hydra.new)
+        hydra.queue treq
+        hydra.run
 
-        if typhoeus_request.options[:maxfilesize]
-            typhoeus_response.options[:response_body] = response_body_buffer
-        end
-
-        fill_in_data_from_typhoeus_response typhoeus_response
-
-        Response.from_typhoeus( typhoeus_response )
+        self.response
     end
 
     def fill_in_data_from_typhoeus_response( response )
@@ -667,6 +863,8 @@ class Request < Message
             end
 
             buffer << chunk
+
+            true
         end
     end