arachni 1.0.4 → 1.0.5

data/lib/arachni/http/headers.rb

@@ -20,7 +20,11 @@ class Headers < Hash
 
     # @param  [Headers, Hash] headers
     def initialize( headers = {} )
-        (headers || {}).each { |k, v| self[k] = v }
+        merge!( headers || {} )
+    end
+
+    def merge!( headers )
+        headers.each { |k, v| self[k] = v }
     end
 
     # @note `field` will be capitalized appropriately before storing.

data/lib/arachni/http/message.rb

@@ -38,7 +38,6 @@ class Message
     #   Body.
     def initialize( options = {} )
         options.each do |k, v|
-            v = my_dup( v )
             begin
                 send( "#{k}=", v )
             rescue NoMethodError
@@ -65,12 +64,6 @@ class Message
         @url = Arachni::URI( url ).to_s.freeze
     end
 
-    private
-
-    def my_dup( value )
-        value.dup rescue value
-    end
-
 end
 end
 end

data/lib/arachni/http/request.rb

@@ -127,10 +127,10 @@ class Request < Message
         @train           = false if @train.nil?
         @update_cookies  = false if @update_cookies.nil?
         @follow_location = false if @follow_location.nil?
-        @max_redirects   = (Arachni::Options.http.request_redirect_limit || REDIRECT_LIMIT)
+        @max_redirects   = (Options.http.request_redirect_limit || REDIRECT_LIMIT)
         @on_complete     = []
 
-        @timeout       ||= Arachni::Options.http.request_timeout
+        @timeout       ||= Options.http.request_timeout
         @mode          ||= :async
         @parameters    ||= {}
         @cookies       ||= {}
@@ -213,6 +213,10 @@ class Request < Message
         end.merge( cookies )
     end
 
+    def effective_parameters
+        Utilities.uri_parse_query( url ).merge( parameters || {} )
+    end
+
     def body_parameters
         return {} if method != :post
         parameters.any? ? parameters : self.class.parse_body( body )
@@ -276,17 +280,10 @@ class Request < Message
     #
     # @return   [Response]
     def run
-        response = to_typhoeus.run
-        fill_in_data_from_typhoeus_response response
-
-        Response.from_typhoeus( response ).tap { |r| r.request = self }
+        client_run.tap { |r| r.request = self }
     end
 
-    def handle_response( response, typhoeus_response = nil )
-        if typhoeus_response
-            fill_in_data_from_typhoeus_response typhoeus_response
-        end
-
+    def handle_response( response )
         response.request = self
         @on_complete.each { |b| b.call response }
         response
@@ -295,24 +292,15 @@ class Request < Message
     # @return   [Typhoeus::Response]
     #   `self` converted to a `Typhoeus::Request`.
     def to_typhoeus
-        headers['Cookie'] = effective_cookies.
-            map { |k, v| "#{Cookie.encode( k )}=#{Cookie.encode( v )}" }.
-            join( ';' )
-
-        headers['User-Agent'] ||= Arachni::Options.http.user_agent
-        headers['Accept']     ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
-        headers['From']       ||= Arachni::Options.authorized_by if Arachni::Options.authorized_by
-
-        headers.delete( 'Cookie' ) if headers['Cookie'].empty?
-        headers.each { |k, v| headers[k] = Header.encode( v ) if v }
+        prepare_headers
 
-        if (userpwd = (@username || Arachni::Options.http.authentication_username))
-            if (passwd = (@password || Arachni::Options.http.authentication_password))
+        if (userpwd = (@username || Options.http.authentication_username))
+            if (passwd = (@password || Options.http.authentication_password))
                 userpwd += ":#{passwd}"
             end
         end
 
-        max_size = @response_max_size || Arachni::Options.http.response_max_size
+        max_size = @response_max_size || Options.http.response_max_size
         # Weird I know, for some reason 0 gets ignored.
         max_size = 1 if max_size == 0
 
@@ -320,8 +308,7 @@ class Request < Message
             method:          method,
             headers:         headers,
             body:            body,
-            params:          Arachni::Utilities.uri_parse_query( url ).
-                                 merge( parameters || {} ),
+            params:          effective_parameters,
             userpwd:         userpwd,
             followlocation:  follow_location?,
             maxredirs:       @max_redirects,
@@ -350,15 +337,15 @@ class Request < Message
                 options[:proxyuserpwd] = proxy_user_password
             end
 
-        elsif Arachni::Options.http.proxy_host && Arachni::Options.http.proxy_port
+        elsif Options.http.proxy_host && Options.http.proxy_port
             options.merge!(
-                proxy:     "#{Arachni::Options.http.proxy_host}:#{Arachni::Options.http.proxy_port}",
-                proxytype: (Arachni::Options.http.proxy_type || :http).to_sym
+                proxy:     "#{Options.http.proxy_host}:#{Options.http.proxy_port}",
+                proxytype: (Options.http.proxy_type || :http).to_sym
             )
 
-            if Arachni::Options.http.proxy_username && Arachni::Options.http.proxy_password
+            if Options.http.proxy_username && Options.http.proxy_password
                 options[:proxyuserpwd] =
-                    "#{Arachni::Options.http.proxy_username}:#{Arachni::Options.http.proxy_password}"
+                    "#{Options.http.proxy_username}:#{Options.http.proxy_password}"
             end
         end
 
@@ -367,7 +354,8 @@ class Request < Message
 
         if @on_complete.any?
             r.on_complete do |typhoeus_response|
-                handle_response Response.from_typhoeus( typhoeus_response ), typhoeus_response
+                fill_in_data_from_typhoeus_response typhoeus_response
+                handle_response Response.from_typhoeus( typhoeus_response )
             end
         end
 
@@ -461,11 +449,31 @@ class Request < Message
 
     private
 
+    def prepare_headers
+        headers['Cookie'] = effective_cookies.
+            map { |k, v| "#{Cookie.encode( k )}=#{Cookie.encode( v )}" }.
+            join( ';' )
+
+        headers['User-Agent'] ||= Options.http.user_agent
+        headers['Accept']     ||= 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+        headers['From']       ||= Options.authorized_by if Options.authorized_by
+
+        headers.delete( 'Cookie' ) if headers['Cookie'].empty?
+        headers.each { |k, v| headers[k] = Header.encode( v ) if v }
+    end
+
     def fill_in_data_from_typhoeus_response( response )
         @headers_string = response.debug_info.header_out.first
         @effective_body = response.debug_info.data_out.first
     end
 
+    def client_run
+        response = to_typhoeus.run
+        fill_in_data_from_typhoeus_response response
+
+        Response.from_typhoeus( response )
+    end
+
 end
 end
 end

data/lib/arachni/http/response.rb

@@ -197,8 +197,15 @@ class Response < Message
 
     def self.from_typhoeus( response )
         redirections = response.redirections.map do |redirect|
+            rurl   = URI.to_absolute( redirect.headers['Location'],
+                                      response.effective_url )
+            rurl ||= response.effective_url
+
+            # Broken redirection, skip it...
+            next if !rurl
+
             new(
-                url:     redirect.headers['Location'] || response.effective_url,
+                url:     rurl,
                 code:    redirect.code,
                 headers: redirect.headers
             )

data/lib/arachni/platform/manager.rb

@@ -294,6 +294,13 @@ class Manager
     # @return   [Manager]
     #   Platform for the given `uri`
     def self.[]( uri )
+        # If fingerprinting is disabled there's no point in filling the cache
+        # with the same object over and over, create an identical one for all
+        # URLs and return that always.
+        if !Options.fingerprint?
+            return @default ||= new( Options.platforms )
+        end
+
         return new if !(key = make_key( uri ))
         synchronize { @platforms[key] ||= new }
     end

data/lib/arachni/rpc/server/framework/multi_instance.rb

@@ -44,7 +44,7 @@ module MultiInstance
     def errors( starting_line = 0, &block )
         return [] if !File.exists? error_logfile
 
-        error_strings = IO.read( error_logfile ).split( "\n" )
+        error_strings = error_buffer.dup
 
         if starting_line != 0
             error_strings = error_strings[starting_line..-1]

data/lib/arachni/session.rb

@@ -51,6 +51,9 @@ class Session
     # @return   [Browser]
     attr_reader :browser
 
+    # @return   [Block]
+    attr_reader :login_sequence
+
     def clean_up
         configuration.clear
         shutdown_browser
@@ -98,7 +101,7 @@ class Session
     # @return   [Bool]
     #   `true` if {#configure configured}, `false` otherwise.
     def configured?
-        configuration.any?
+        !!@login_sequence || configuration.any?
     end
 
     # Finds a login forms based on supplied location, collection and criteria.
@@ -201,16 +204,95 @@ class Session
         end
     end
 
-    # Uses the information provided by {#configure} to login.
+    # @param    [Block] block
+    #   Login sequence. Must return the resulting {Page}.
+    #
+    #   If a {#browser} is {#has_browser? available} it will be passed to the
+    #   block.
+    def record_login_sequence( &block )
+        @login_sequence = block
+    end
+
+    # Uses the information provided by {#configure} or {#login_sequence} to login.
     #
     # @return   [Page, nil]
-    #   {HTTP::Response} if the login form was submitted successfully,
-    #   `nil` if not {#configured?}.
+    #   {Page} if the login was successful, `nil` otherwise.
     #
+    # @raise    [Error::NotConfigured]
+    #   If not {#configured?}.
     # @raise    [Error::FormNotFound]
     #   If the form could not be found.
     def login
-        fail Error::NotConfigured, 'Please #configure the session first.' if !configured?
+        fail Error::NotConfigured, 'Please configure the session first.' if !configured?
+
+        refresh_browser
+
+        page = @login_sequence ? login_from_sequence : login_from_configuration
+
+        if has_browser?
+            http.update_cookies browser.cookies
+        end
+
+        page
+    end
+
+    # @param    [Block] block
+    #   Block to be passed the {#browser}.
+    def with_browser( &block )
+        block.call browser
+    end
+
+    # @param    [Hash]   http_options
+    #   HTTP options to use for the check.
+    # @param    [Block]  block
+    #   If a block has been provided the check will be async and the result will
+    #   be passed to it, otherwise the method will return the result.
+    #
+    # @return   [Bool, nil]
+    #   `true` if we're logged-in, `false` otherwise.
+    #
+    # @raise    [Error::NoLoginCheck]
+    #   If no login-check has been configured.
+    def logged_in?( http_options = {}, &block )
+        fail Error::NoLoginCheck if !has_login_check?
+
+        http_options = http_options.merge(
+            mode:            block_given? ? :async : :sync,
+            follow_location: true
+        )
+
+        bool = nil
+        http.get( Options.session.check_url, http_options ) do |response|
+            bool = !!response.body.match( Options.session.check_pattern )
+            block.call( bool ) if block
+        end
+        bool
+    end
+
+    # @return   [Bool]
+    #   `true` if a login check exists, `false` otherwise.
+    def has_login_check?
+        !!@login_check || !!(Options.session.check_url && Options.session.check_pattern)
+    end
+
+    # @return   [HTTP::Client]
+    def http
+        HTTP::Client
+    end
+
+    def has_browser?
+        Browser.has_executable? && Options.scope.dom_depth_limit > 0
+    end
+
+    private
+
+    def login_from_sequence
+        print_debug "Logging in via sequence: #{@login_sequence}"
+        @login_sequence.call browser
+    end
+
+    def login_from_configuration
+        print_debug 'Logging in via configuration.'
 
         if has_browser?
             print_debug 'Logging in using browser.'
@@ -222,7 +304,7 @@ class Session
 
         # Revert to the Framework DOM Level 1 page handling if no browser
         # is available.
-        page = refresh_browser ?
+        page = has_browser? ?
             browser.load( configuration[:url], take_snapshot: false ).to_page :
             Page.from_url( configuration[:url], precision: 1, http: {
                 update_cookies: true
@@ -261,8 +343,6 @@ class Session
             print_debug 'Submitting form.'
             form.submit { |p| page = p }
             print_debug 'Form submitted.'
-
-            http.update_cookies browser.cookies
         else
             page = form.submit(
                 mode:            :sync,
@@ -281,56 +361,6 @@ class Session
         page
     end
 
-    # @param    [Block] block
-    #   Block to be passed the {#browser}.
-    def with_browser( &block )
-        block.call browser
-    end
-
-    # @param    [Hash]   http_options
-    #   HTTP options to use for the check.
-    # @param    [Block]  block
-    #   If a block has been provided the check will be async and the result will
-    #   be passed to it, otherwise the method will return the result.
-    #
-    # @return   [Bool, nil]
-    #   `true` if we're logged-in, `false` otherwise.
-    #
-    # @raise    [Error::NoLoginCheck]
-    #   If no login-check has been configured.
-    def logged_in?( http_options = {}, &block )
-        fail Error::NoLoginCheck if !has_login_check?
-
-        http_options = http_options.merge(
-            mode:            block_given? ? :async : :sync,
-            follow_location: true
-        )
-
-        bool = nil
-        http.get( Options.session.check_url, http_options ) do |response|
-            bool = !!response.body.match( Options.session.check_pattern )
-            block.call( bool ) if block
-        end
-        bool
-    end
-
-    # @return   [Bool]
-    #   `true` if a login check exists, `false` otherwise.
-    def has_login_check?
-        !!(Options.session.check_url && Options.session.check_pattern)
-    end
-
-    # @return   [HTTP::Client]
-    def http
-        HTTP::Client
-    end
-
-    def has_browser?
-        Browser.has_executable? && Options.scope.dom_depth_limit > 0
-    end
-
-    private
-
     def shutdown_browser
         return if !@browser
 

data/lib/arachni/state/framework.rb

@@ -44,6 +44,9 @@ class Framework
     # @return     [Support::LookUp::HashSet]
     attr_reader   :url_queue_filter
 
+    # @return     [Support::LookUp::HashSet]
+    attr_reader   :element_pre_check_filter
+
     # @return     [Set]
     attr_reader   :browser_skip_states
 
@@ -71,6 +74,8 @@ class Framework
         @page_queue_filter = Support::LookUp::HashSet.new( hasher: :persistent_hash )
         @url_queue_filter  = Support::LookUp::HashSet.new( hasher: :persistent_hash )
 
+        @element_pre_check_filter = Support::LookUp::HashSet.new( hasher: :coverage_hash )
+
         @running = false
         @pre_pause_status = nil
 
@@ -185,6 +190,25 @@ class Framework
         @browser_skip_states.merge states
     end
 
+    # @param    [#coverage_hash]  e
+    #
+    # @return    [Bool]
+    #   `true` if the element has already been seen (based on the
+    #   {#element_pre_check_filter}), `false` otherwise.
+    #
+    # @see #element_checked
+    def element_checked?( e )
+        @element_pre_check_filter.include? e
+    end
+
+    # @param    [Page]  e
+    #   Element to mark as seen.
+    #
+    # @see #element_checked?
+    def element_checked( e )
+        @element_pre_check_filter << e
+    end
+
     def running?
         !!@running
     end
@@ -377,9 +401,8 @@ class Framework
 
         rpc.dump( "#{directory}/rpc/" )
 
-        %w(page_queue_filter url_queue_filter browser_skip_states
-            audited_page_count).each do |attribute|
-
+        %w(element_pre_check_filter page_queue_filter url_queue_filter
+            browser_skip_states audited_page_count).each do |attribute|
             IO.binwrite( "#{directory}/#{attribute}", Marshal.dump( send(attribute) ) )
         end
     end
@@ -389,8 +412,12 @@ class Framework
 
         framework.rpc = RPC.load( "#{directory}/rpc/" )
 
-        %w(page_queue_filter url_queue_filter browser_skip_states).each do |attribute|
-            framework.send(attribute).merge Marshal.load( IO.binread( "#{directory}/#{attribute}" ) )
+        %w(element_pre_check_filter page_queue_filter url_queue_filter
+            browser_skip_states).each do |attribute|
+            path = "#{directory}/#{attribute}"
+            next if !File.exist?( path )
+
+            framework.send(attribute).merge Marshal.load( IO.binread( path ) )
         end
 
         framework.audited_page_count = Marshal.load( IO.binread( "#{directory}/audited_page_count" ) )
@@ -400,6 +427,8 @@ class Framework
     def clear
         rpc.clear
 
+        @element_pre_check_filter.clear
+
         @page_queue_filter.clear
         @url_queue_filter.clear