arachni 1.4 → 1.5

data/lib/arachni/http/cookie_jar.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -86,11 +86,7 @@ class CookieJar
 
             self << case c
                         when String
-                            begin
-                                Cookie.from_string( ::Arachni::Options.url.to_s, c )
-                            rescue
-                                Cookie.from_set_cookie( ::Arachni::Options.url.to_s, c )
-                            end
+                            Cookie.from_set_cookie( ::Arachni::Options.url.to_s, c )
 
                         when Hash
                             next if c.empty?
@@ -207,8 +203,17 @@ class CookieJar
     end
 
     def to_uri( url )
-        u = url.is_a?( ::URI ) || url.is_a?( ::Arachni::URI ) ? url : uri_parse( url.to_s )
-        fail ArgumentError, 'Complete absolute URL required.' if u.relative?
+        u = url.is_a?( Arachni::URI ) ? url : Arachni::URI( url.to_s )
+
+        if !u
+            fail "Failed to parse: #{url}"
+        end
+
+        if !u.absolute?
+            fail ArgumentError,
+                 "Complete absolute URL required, got: #{url} (#{u})"
+        end
+
         u
     end
 

data/lib/arachni/http/headers.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -18,7 +18,7 @@ module HTTP
 # @author Tasos Laskos <tasos.laskos@arachni-scanner.com>
 class Headers < Hash
 
-    FORMATTED_NAMES_CACHE = Support::Cache::LeastRecentlyPushed.new( 10_000 )
+    FORMATTED_NAMES_CACHE = Support::Cache::LeastRecentlyPushed.new( 1_000 )
 
     CONTENT_TYPE = 'content-type'
     SET_COOKIE   = 'set-cookie'
@@ -29,11 +29,11 @@ class Headers < Hash
         merge!( headers || {} )
     end
 
-    def merge!( headers )
+    def merge!( headers, convert_to_array = true )
         headers.each do |k, v|
             # Handle headers with identical normalized names, like a mixture of
             # Set-Cookie and SET-COOKIE.
-            if include? k
+            if convert_to_array && include?( k )
                 self[k] = [self[k]].flatten
                 self[k] << v
             else
@@ -42,6 +42,12 @@ class Headers < Hash
         end
     end
 
+    def merge( headers, convert_to_array = true )
+        d = dup
+        d.merge! headers, convert_to_array
+        d
+    end
+
     # @note `field` will be capitalized appropriately before storing.
     #
     # @param    [String]  field
@@ -92,7 +98,7 @@ class Headers < Hash
     # @return   [String, nil]
     #   Value of the `Content-Type` field.
     def content_type
-        self[CONTENT_TYPE]
+        (ct = self[CONTENT_TYPE]).is_a?( Array ) ? ct.first : ct
     end
 
     # @return   [String, nil]

data/lib/arachni/http/message.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -37,13 +37,16 @@ class Message
     # @option   options [String]    :body
     #   Body.
     def initialize( options = {} )
-        options = options.dup
+        update( options )
 
-        @normalize_url = options.delete( :normalize_url )
-        @normalize_url = true if @normalize_url.nil?
+        fail ArgumentError, 'Missing :url.' if url.to_s.empty?
+    end
+
+    def update( options )
+        @normalize_url = options[:normalize_url]
 
         # Headers are necessary for subsequent operations to set them first.
-        @headers = Headers.new( options.delete( :headers ) || {} )
+        @headers = Headers.new( options[:headers] || {} )
 
         options.each do |k, v|
             begin
@@ -52,8 +55,6 @@ class Message
                 instance_variable_set( "@#{k}".to_sym, v )
             end
         end
-
-        fail ArgumentError, 'Missing :url.' if url.to_s.empty?
     end
 
     def headers=( h )
@@ -71,7 +72,7 @@ class Message
     end
 
     def url=( url )
-        if @normalize_url
+        if @normalize_url || @normalize_url.nil?
             @url = URI.normalize_url( url ).to_s.freeze
         else
             @url = url.to_s.freeze

data/lib/arachni/http/message/scope.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/http/proxy_server.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -21,6 +21,8 @@ class ProxyServer
     include Arachni::UI::Output
     personalize_output
 
+    DEFAULT_CONCURRENCY = 4
+
     # @param   [Hash]  options
     # @option options   [String]    :address    ('0.0.0.0')
     #   Address to bind to.
@@ -28,6 +30,8 @@ class ProxyServer
     #   Port number to listen on -- defaults to a random port.
     # @option options   [Integer]    :timeout
     #   HTTP time-out for each request in milliseconds.
+    # @option options   [Integer]    :concurrency   (DEFAULT_CONCURRENCY)
+    #   Amount of origin requests to be active at any given time.
     # @option options   [Block]    :response_handler
     #   Block to be called to handle each response as it arrives -- will be
     #   passed the request and response.
@@ -35,42 +39,67 @@ class ProxyServer
     #   Block to be called to handle each request as it arrives -- will be
     #   passed the request and response.
     def initialize( options = {} )
-        @reactor = Arachni::Reactor.new
+        @reactor = Arachni::Reactor.new(
+            # Higher than the defaults to keep object allocations down.
+            select_timeout:    0.1,
+            max_tick_interval: 0.1
+        )
         @options = options
 
-        @active_connections = Set.new
+        @active_connections = Concurrent::Map.new
 
-        @options[:address] ||= '127.0.0.1'
-        @options[:port]    ||= Utilities.available_port
+        @options[:concurrency] ||= DEFAULT_CONCURRENCY
+        @options[:address]     ||= '127.0.0.1'
+        @options[:port]        ||= Utilities.available_port
+
+        @concurrency_control_tokens = @reactor.create_queue
     end
 
     # Starts the server without blocking, it'll only block until the server is
     # up and running and ready to accept connections.
     def start_async
-        print_debug_level_2 'Starting'
+        print_debug_level_2 'Starting...'
 
         @reactor.run_in_thread
 
+        @options[:concurrency].times do |i|
+            @concurrency_control_tokens << i
+        end
+
         @reactor.on_error do |_, e|
             print_exception e
         end
 
-        listener = @reactor.listen(
+        @reactor.listen(
             @options[:address], @options[:port], Connection,
             @options.merge( parent: self )
         )
 
-        print_debug_level_2 'Started'
+        print_debug_level_2 "...started at: #{url}"
         nil
     end
 
+    def get_request_token( &block )
+        @concurrency_control_tokens.pop( &block )
+    end
+
+    def return_request_token( token )
+        @concurrency_control_tokens << token
+    end
+
+    def has_available_request_tokens?
+        @concurrency_control_tokens.empty?
+    end
+
     def shutdown
-        print_debug_level_2 'Shutting down..'
+        print_debug_level_2 'Shutting down...'
+
+        @thread_pool.kill if @thread_pool
 
         @reactor.stop
         @reactor.wait
 
-        print_debug_level_2 'Shutdown.'
+        print_debug_level_2 '...shutdown.'
     end
 
     # @return   [Bool]
@@ -97,8 +126,12 @@ class ProxyServer
         @active_connections.size
     end
 
+    def active_connections
+        @active_connections.keys
+    end
+
     def mark_connection_active( connection )
-        @active_connections << connection
+        @active_connections.put_if_absent( connection, nil )
     end
 
     def mark_connection_inactive( connection )

data/lib/arachni/http/proxy_server/connection.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2016 Tasos Laskos <tasos.laskos@arachni-scanner.com>
+    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -14,17 +14,19 @@ class Connection < Arachni::Reactor::Connection
     include Arachni::UI::Output
     personalize_output
 
-    SKIP_HEADERS = %w(transfer-encoding connection proxy-connection
-        content-encoding te trailers accept-encoding)
+    SKIP_HEADERS = %w(transfer-encoding connection proxy-connection keep-alive
+        content-encoding te trailers accept-encoding accept-ranges vary)
 
     attr_reader :parent
+    attr_reader :request
 
     def initialize( options = {} )
         @options = options
         @parent  = options[:parent]
 
-        @body   = ''
-        @parser = ::HTTP::Parser.new
+        @body     = ''
+        @parser   = ::HTTP::Parser.new
+        @raw_request = ''
 
         @parser.on_message_begin = proc do
             if @reused
@@ -50,26 +52,67 @@ class Connection < Arachni::Reactor::Connection
 
             print_debug_level_3 "Request received: #{@parser.http_method} #{@parser.request_url}"
 
+            if headers['upgrade']
+                handle_upgrade( headers )
+                next
+            end
+
             if method == :connect
                 handle_connect( headers )
                 next
             end
 
-            request_url = sanitize_url( @parser.request_url, headers )
-
-            handle_request(
-                Arachni::HTTP::Request.new(
-                    http_opts.merge(
-                        url:     request_url,
-                        method:  method,
-                        body:    @body,
-                        headers: headers
-                    )
-                )
-            )
+            if !@parent.has_available_request_tokens?
+                print_debug_level_3 'Waiting for a request token.'
+            end
+
+            @parent.get_request_token do |token|
+                print_debug_level_3 "Got request token ##{token}."
+
+                if closed?
+                    print_debug_level_3 'Connection closed while waiting for a request token.'
+                    @parent.return_request_token( token )
+                    print_debug_level_3 "Returned request token ##{token}."
+
+                    next
+                end
+
+                Thread.new do
+                    begin
+                        @request = Arachni::HTTP::Request.new(
+                            http_opts.merge(
+                                url:     sanitize_url( @parser.request_url, headers ),
+                                method:  method,
+                                body:    @body,
+                                headers: headers
+                            )
+                        )
+
+                        handle_request( @request )
+                    rescue => e
+                        close e
+                    ensure
+                        @parent.return_request_token( token )
+                        print_debug_level_3 "Returned request token ##{token}."
+                    end
+                end
+            end
         end
     end
 
+    def handle_upgrade( headers )
+        print_debug_level_3 'Preparing to upgrade.'
+
+        host = (headers['Host'] || @parser.request_url).split( ':', 2 ).first
+
+        @tunnel = reactor.connect( host, 80, Tunnel, @options.merge( client: self ) )
+
+        # This is our last HTTP message, from this point on we'll only be
+        # tunnelling to the origin server.
+        @last_http = true
+        @tunnel.write @raw_request
+    end
+
     def handle_connect( headers )
         print_debug_level_3 'Preparing to intercept.'
 
@@ -85,42 +128,40 @@ class Connection < Arachni::Reactor::Connection
     def handle_request( request )
         print_debug_level_3 'Processing request.'
 
-        Thread.new do
-            if @options[:request_handler]
-                print_debug_level_3 "-- Has special handler: #{@options[:request_handler]}"
-
-                # Provisional empty, response in case the request_handler wants us to
-                # skip performing the request.
-                response = Response.new( url: request.url )
-                response.request = request
+        if @options[:request_handler]
+            print_debug_level_3 "-- Has special handler: #{@options[:request_handler]}"
 
-                # If the handler returns false then don't perform the HTTP request.
-                if @options[:request_handler].call( request, response )
-                    print_debug_level_3 '-- Handler approves, running...'
+            # Provisional empty, response in case the request_handler wants us to
+            # skip performing the request.
+            response = Response.new( url: request.url )
+            response.request = request
 
-                    # Even though it's a blocking request, force it to go through
-                    # the HTTP::Client in order to handle cookie update and
-                    # fingerprinting handlers.
-                    HTTP::Client.queue( request )
-                    response = request.run
-
-                    print_debug_level_3 "-- ...completed in #{response.time}: #{response.status_line}"
-                else
-                    print_debug_level_3 '-- Handler did not approve, will not run.'
-                end
-            else
-                print_debug_level_3 '-- Running...'
+            # If the handler returns false then don't perform the HTTP request.
+            if @options[:request_handler].call( request, response )
+                print_debug_level_3 '-- Handler approves, running...'
 
+                # Even though it's a blocking request, force it to go through
+                # the HTTP::Client in order to handle cookie update and
+                # fingerprinting handlers.
                 HTTP::Client.queue( request )
                 response = request.run
 
                 print_debug_level_3 "-- ...completed in #{response.time}: #{response.status_line}"
+            else
+                print_debug_level_3 '-- Handler did not approve, will not run.'
             end
+        else
+            print_debug_level_3 '-- Running...'
 
-            print_debug_level_3 'Processed request.'
+            HTTP::Client.queue( request )
+            response = request.run
 
-            handle_response( response )
+            print_debug_level_3 "-- ...completed in #{response.time}: #{response.status_line}"
         end
+
+        print_debug_level_3 'Processed request.'
+
+        reactor.schedule { handle_response( response ) }
     end
 
     def http_version
@@ -152,6 +193,11 @@ class Connection < Arachni::Reactor::Connection
         headers = cleanup_response_headers( response.headers )
         headers['Content-Length'] = response.body.bytesize
 
+        if response.text? && headers.content_type
+            headers['Content-Type'] =
+                "#{headers.content_type.split( ';' ).first}; charset=utf-8"
+        end
+
         headers.each do |k, v|
             if v.is_a?( Array )
                 v.flatten.each do |h|
@@ -166,12 +212,9 @@ class Connection < Arachni::Reactor::Connection
 
         res << "\r\n"
 
-        print_debug_level_3 'Sending response.'
+        print_debug_level_3 "Sending response for: #{@request.url}"
 
         write (res << response.body)
-    rescue => e
-        ap e
-        ap e.backtrace
     end
 
     def on_close( reason = nil )
@@ -179,33 +222,35 @@ class Connection < Arachni::Reactor::Connection
 
         @parent.mark_connection_inactive self
 
-        return if !@ssl_tunnel
-
-        @ssl_interceptor.close_without_callback
-        @ssl_tunnel.close_without_callback
+        if @ssl_interceptor
+            @ssl_interceptor.close( reason )
+            @ssl_interceptor = nil
+        end
 
-        @ssl_tunnel      = nil
-        @ssl_interceptor = nil
+        if @tunnel
+            @tunnel.close_without_callback
+            @tunnel = nil
+        end
     end
 
     def on_flush
-        @body = ''
-        @parser.reset!
-
-        if !@ssl_tunnel || @last_http
+        if !@tunnel || @last_http
 
             if @last_http
                 print_debug_level_3 'Last response sent, switching to tunnel.'
-            else
-                print_debug_level_3 'Response sent.'
+            elsif @request
+                print_debug_level_3 "Response sent for: #{@request.url}"
             end
 
-            @parent.mark_connection_inactive self
             @last_http = false
         end
-    rescue => e
-        ap e
-        ap e.backtrace
+
+        @body        = ''
+        @raw_request = ''
+        @request     = nil
+
+        @parser.reset!
+        @parent.mark_connection_inactive self
     end
 
     def write( data )
@@ -214,8 +259,11 @@ class Connection < Arachni::Reactor::Connection
     end
 
     def on_read( data )
-        if @ssl_tunnel
-            @ssl_tunnel.write( data )
+        # We need this in case we need to establish a tunnel for an "Upgrade".
+        @raw_request << data
+
+        if @tunnel
+            @tunnel.write( data )
             return
         end
 
@@ -223,12 +271,6 @@ class Connection < Arachni::Reactor::Connection
         @parser << data
     rescue ::HTTP::Parser::Error => e
         close e
-
-    # TODO: While in dev only of course.
-    rescue => e
-        ap e
-        ap e.backtrace
-        close e
     end
 
     def start_interceptor( origin_host )
@@ -241,11 +283,10 @@ class Connection < Arachni::Reactor::Connection
             @options.merge( origin_host: origin_host )
         )
 
-        @ssl_tunnel = reactor.connect(
+        @tunnel = reactor.connect(
             @options[:address], @interceptor_port, Tunnel,
             @options.merge( client: self )
         )
-
     end
 
     def cleanup_request_headers( headers )
@@ -262,14 +303,6 @@ class Connection < Arachni::Reactor::Connection
         SKIP_HEADERS.each do |name|
             headers.delete name
         end
-
-        # headers['Connection']       = 'close'
-
-        # Keep alive is on by default for HTTP/1.1 but leave this here as a
-        # reminder.
-        #
-        # headers['Connection']       = 'keep-alive'
-        # headers['Proxy-Connection'] = 'keep-alive'
         headers
     end