arachni 1.1 → 1.2

data/lib/arachni/framework/parts/browser.rb

@@ -30,18 +30,20 @@ module Browser
                 state.set_status_message :browser_cluster_startup
             end
 
-            @browser_cluster ||= BrowserCluster.new
+            @browser_cluster ||= BrowserCluster.new(
+                on_pop: proc do
+                    next if !pause?
+
+                    print_debug 'Blocking browser cluster on pop.'
+                    wait_if_paused
+                end
+            )
             state.clear_status_messages
+
             @browser_cluster
         end
     end
 
-    def browser
-        return if !use_browsers?
-
-        @browser ||= Arachni::Browser.new( store_pages: false )
-    end
-
     # @return   [Bool]
     #   `true` if the environment has a browser, `false` otherwise.
     def host_has_browser?
@@ -58,36 +60,6 @@ module Browser
         browser_cluster.skip_states( browser_job.id )
     end
 
-    # @private
-    def apply_dom_metadata( page )
-        return false if page.dom.depth > 0 || !page.has_script? ||
-            !browser
-
-        # This optimization only affects Form::DOM elements, so don't bother
-        # if none of the checks are interested in any of them.
-        return false if !checks.values.find do |c|
-            c.check? page, [Element::Form::DOM, Element::Cookie::DOM]
-        end
-
-        begin
-            bp = browser.load( page ).to_page
-        rescue Selenium::WebDriver::Error::WebDriverError,
-            Watir::Exception::Error => e
-            print_debug "Could not apply metadata to '#{page.dom.url}'" <<
-                                    " because: #{e} [#{e.class}"
-            return
-        end
-
-        # Request timeout or some other failure...
-        return if bp.code == 0
-
-        page.import_metadata( bp, :skip_dom )
-
-        true
-    ensure
-        browser.clear_buffers if browser
-    end
-
     def use_browsers?
         options.browser_cluster.pool_size > 0 &&
             options.scope.dom_depth_limit > 0 && host_has_browser?
@@ -95,13 +67,6 @@ module Browser
 
     private
 
-    def shutdown_browser
-        return if !@browser
-
-        @browser.shutdown
-        @browser = nil
-    end
-
     def shutdown_browser_cluster
         return if !@browser_cluster
 
@@ -125,11 +90,6 @@ module Browser
         synchronize do
             return if !push_to_page_queue page
 
-            pushed_paths = nil
-            if crawl?
-                pushed_paths = push_paths_from_page( page ).size
-            end
-
             print_status "Got new page from the browser-cluster: #{page.dom.url}"
             print_info "DOM depth: #{page.dom.depth} (Limit: #{options.scope.dom_depth_limit})"
 
@@ -137,10 +97,6 @@ module Browser
                 print_info '  Transitions:'
                 page.dom.print_transitions( method(:print_info), '    ' )
             end
-
-            if pushed_paths
-                print_info "  -- Analysis resulted in #{pushed_paths} usable paths."
-            end
         end
     end
 
@@ -154,13 +110,52 @@ module Browser
             Options.scope.dom_depth_limit.to_i < page.dom.depth + 1 ||
             !page.has_script?
 
-        browser_cluster.queue( browser_job.forward( resource: page ) ) do |response|
-            handle_browser_page response.page
+        # We need to schedule a separate job for applying metadata because it
+        # needs to have a clean state.
+        schedule_dom_metadata_application( page )
+
+        browser_cluster.queue( browser_job.forward( resource: page ) ) do |result|
+            handle_browser_page result.page
         end
 
         true
     end
 
+    def schedule_dom_metadata_application( page )
+        return if page.dom.depth > 0
+        return if page.metadata.map { |_, data| data['skip_dom'].values }.
+            flatten.compact.any?
+
+        # This optimization only affects Form & Cookie DOM elements,
+        # so don't bother if none of the checks are interested in them.
+        return if !checks.values.
+            find { |c| c.check? page, [Element::Form::DOM, Element::Cookie::DOM], true }
+
+        page.clear_cache
+
+        browser_cluster.with_browser do |browser|
+            apply_dom_metadata( browser, page )
+        end
+    end
+
+    def apply_dom_metadata( browser, page )
+        bp = nil
+
+        begin
+            bp = browser.load( page ).to_page
+        rescue Selenium::WebDriver::Error::WebDriverError,
+            Watir::Exception::Error => e
+            print_debug "Could not apply metadata to '#{page.dom.url}'" <<
+                            " because: #{e} [#{e.class}"
+            return
+        end
+
+        # Request timeout or some other failure...
+        return if bp.code == 0
+
+        handle_browser_page bp
+    end
+
     def browser_job
         # We'll recycle the same job since all of them will have the same
         # callback. This will force the BrowserCluster to use the same block

data/lib/arachni/framework/parts/check.rb

@@ -67,11 +67,12 @@ module Check
     #   Check to run.
     # @param    [Page]    page
     def check_page( check, page )
+        ps = page.platforms.to_a
+
         # If we've been given platforms which the check doesn't support don't
         # even bother running it.
-        if !check.supports_platforms?( Options.platforms )
-            print_info "Check #{check.shortname} does not support: " <<
-                           Options.platforms.join( ' + ' )
+        if !check.supports_platforms?( ps )
+            print_info "Check #{check.shortname} does not support: #{ps.join( ' + ' )}"
             return false
         end
 

data/lib/arachni/framework/parts/data.rb

@@ -27,8 +27,9 @@ module Data
     #   `true` if push was successful, `false` if the `page` matched any
     #   exclusion criteria or has already been seen.
     def push_to_page_queue( page, force = false )
-        return false if !force && (!accepts_more_pages? || state.page_seen?( page ) ||
-            page.scope.out? || page.scope.redundant?( true ))
+        return false if !force && (!accepts_more_pages? ||
+            state.page_seen?( page ) || page.scope.out? ||
+            page.scope.redundant?( true ))
 
         # We want to update from the already loaded page cache (if there is one)
         # as we have to store the page anyways (needs to go through Browser analysis)
@@ -40,6 +41,7 @@ module Data
         # some other component; however, it wouldn't be the end of the world if
         # that were to happen.
         ElementFilter.update_from_page_cache page
+        page.clear_cache
 
         data.push_to_page_queue page
         state.page_seen page
@@ -99,12 +101,24 @@ module Data
         !url_queue.empty? || !page_queue.empty?
     end
 
+    # @return   [Page, nil]
+    #   A page if the queues aren't empty, `nil` otherwise.
+    def pop_page
+        pop_page_from_queue || pop_page_from_url_queue
+    end
+
+    # @return   [Page, nil]
+    #   A page if the queue wasn't empty, `nil` otherwise.
     def pop_page_from_url_queue( &block )
         return if url_queue.empty?
 
         grabbed_page = nil
-        Page.from_url( url_queue.pop,
-                       http: { update_cookies: true, performer: self }
+        Page.from_url(
+            url_queue.pop,
+           http: {
+               update_cookies: true,
+               performer:      self
+           }
         ) do |page|
             @retries[page.url.hash] ||= 0
 
@@ -125,7 +139,8 @@ module Data
                 @failures << page.url
 
                 print_error "Giving up trying to audit: #{page.url}"
-                print_error "Couldn't get a response after #{AUDIT_PAGE_MAX_TRIES} tries: #{page.response.return_message}."
+                print_error "Couldn't get a response after #{AUDIT_PAGE_MAX_TRIES}" +
+                                " tries: #{page.response.return_message}."
             else
                 print_bad "Retrying for: #{page.url} [#{page.response.return_message}]"
                 @retries[page.url.hash] += 1
@@ -135,16 +150,36 @@ module Data
             grabbed_page = nil
             block.call grabbed_page if block_given?
         end
+
         http.run if !block_given?
         grabbed_page
     end
 
-    # @return   [Page]
+    # @return   [Page, nil]
+    #   A page if the queue wasn't empty, `nil` otherwise.
     def pop_page_from_queue
         return if page_queue.empty?
         page_queue.pop
     end
 
+    def replenish_page_queue_from_url_queue
+        return if !page_queue.empty?
+
+        # Number pulled out of my ass, low enough to not add any noticeable
+        # stress, hopefully high enough to grab us at least one page that has
+        # some workload which will result in HTTP requests which will mask the
+        # next replenishing operation.
+        [10, page_queue.free_buffer_size].min.times do
+            return if url_queue.empty?
+
+            # We push directly to the queue instead of using #push_to_page_queue
+            # because it's too early to deduplicate.
+            pop_page_from_url_queue { |p| page_queue << p }
+        end
+
+        !url_queue.empty?
+    end
+
     def add_to_sitemap( page )
         data.add_page_to_sitemap( page )
     end

data/lib/arachni/framework/parts/state.rb

@@ -65,6 +65,14 @@ module State
     def initialize
         super
 
+        Element::Capabilities::Auditable.skip_like do |element|
+            if pause?
+                print_debug "Blocking on element audit: #{element.audit_id}"
+            end
+
+            wait_if_paused
+        end
+
         state.status = :ready
     end
 
@@ -96,13 +104,14 @@ module State
         return if @cleaned_up
         @cleaned_up = true
 
+        state.force_resume
+
         state.status = :cleanup
 
         sitemap.merge!( browser_sitemap )
 
         if shutdown_browsers
             state.set_status_message :browser_cluster_shutdown
-            shutdown_browser
             shutdown_browser_cluster
         end
 
@@ -128,6 +137,11 @@ module State
         true
     end
 
+    # @private
+    def reset_trainer
+        @trainer = Trainer.new( self )
+    end
+
     # @note Prefer this from {.reset} if you already have an instance.
     # @note You should first reset {Arachni::Options}.
     #
@@ -319,11 +333,6 @@ module State
         state.suspended?
     end
 
-    # @private
-    def reset_trainer
-        @trainer = Trainer.new( self )
-    end
-
     private
 
     # @note Must be called before calling any audit methods.
@@ -370,7 +379,7 @@ module State
                 new_element = true
             end
 
-            if e.respond_to?( :dom ) && e.dom
+            if page.dom.depth > 0 && e.respond_to?( :dom ) && e.dom
                 if !state.element_checked?( e.dom )
                     state.element_checked e.dom
                     new_element = true

data/lib/arachni/http/client.rb

@@ -140,7 +140,11 @@ class Client
         headers.merge!( Options.http.request_headers )
 
         cookie_jar.load( Options.http.cookie_jar_filepath ) if Options.http.cookie_jar_filepath
-        update_cookies( Options.http.cookies )
+
+        Options.http.cookies.each do |name, value|
+            update_cookies( name => value )
+        end
+
         update_cookies( Options.http.cookie_string ) if Options.http.cookie_string
 
         reset_burst_info
@@ -465,14 +469,30 @@ class Client
         @running = true
 
         reset_burst_info
+
+        # Lots of new objects are about to be generated, make sure that old ones
+        # have been collected to prevent RAM spikes.
+        gc
+
         client_run
 
+        # Collect the new objects as well.
+        gc
+
         @queue_size = 0
         @running    = false
 
         @burst_runtime += Time.now - @burst_runtime_start
         @total_runtime += @burst_runtime
     end
+
+    def gc
+        # Don't GC after every little run, only do it when we're past the
+        # maximum queue size.
+        return if @queue_size < Options.http.request_queue_size
+
+        GC.start
+    end
     
     def reset_burst_info
         @burst_response_time_sum = 0
@@ -501,46 +521,12 @@ class Client
             print_debug_level_3 "Headers: #{request.headers}"
             print_debug_level_3 "Cookies: #{request.cookies}"
             print_debug_level_3 "Train?: #{request.train?}"
+            print_debug_level_3 "Fingerprint?: #{request.fingerprint?}"
             print_debug_level_3  '------------'
         end
 
         if add_callbacks
-            request.on_complete do |response|
-                synchronize do
-                    @response_count          += 1
-                    @burst_response_count    += 1
-                    @burst_response_time_sum += response.time
-                    @total_response_time_sum += response.time
-
-                    if Platform::Manager.fingerprint?( response )
-                        # Force a fingerprint by converting the Response to a Page object.
-                        response.to_page
-                    end
-
-                    notify_on_complete( response )
-
-                    parse_and_set_cookies( response ) if request.update_cookies?
-
-                    if debug_level_3?
-                        print_debug_level_3 '------------'
-                        print_debug_level_3 "Got response for request ID#: #{response.request.id}"
-                        print_debug_level_3 "Performer: #{response.request.performer.inspect}"
-                        print_debug_level_3 "Status: #{response.code}"
-                        print_debug_level_3 "Code: #{response.return_code}"
-                        print_debug_level_3 "Message: #{response.return_message}"
-                        print_debug_level_3 "URL: #{response.url}"
-                        print_debug_level_3 "Headers:\n#{response.headers_string}"
-                        print_debug_level_3 "Parsed headers: #{response.headers}"
-                    end
-
-                    if response.timed_out?
-                        print_debug_level_3 "Request timed-out! -- ID# #{response.request.id}"
-                        @time_out_count += 1
-                    end
-
-                    print_debug_level_3 '------------'
-                end
-            end
+            request.on_complete( &method(:global_on_complete) )
         end
 
         synchronize { @request_count += 1 }
@@ -559,6 +545,47 @@ class Client
         request
     end
 
+    def global_on_complete( response )
+        request = response.request
+
+        synchronize do
+            @response_count          += 1
+            @burst_response_count    += 1
+            @burst_response_time_sum += response.time
+            @total_response_time_sum += response.time
+
+            if response.request.fingerprint? &&
+                Platform::Manager.fingerprint?( response )
+
+                # Force a fingerprint by converting the Response to a Page object.
+                response.to_page
+            end
+
+            notify_on_complete( response )
+
+            parse_and_set_cookies( response ) if request.update_cookies?
+
+            if debug_level_3?
+                print_debug_level_3 '------------'
+                print_debug_level_3 "Got response for request ID#: #{response.request.id}\n#{response.request}"
+                print_debug_level_3 "Performer: #{response.request.performer.inspect}"
+                print_debug_level_3 "Status: #{response.code}"
+                print_debug_level_3 "Code: #{response.return_code}"
+                print_debug_level_3 "Message: #{response.return_message}"
+                print_debug_level_3 "URL: #{response.url}"
+                print_debug_level_3 "Headers:\n#{response.headers_string}"
+                print_debug_level_3 "Parsed headers: #{response.headers}"
+            end
+
+            if response.timed_out?
+                print_debug_level_3 "Request timed-out! -- ID# #{response.request.id}"
+                @time_out_count += 1
+            end
+
+            print_debug_level_3 '------------'
+        end
+    end
+
     def client_initialize
         @hydra = Typhoeus::Hydra.new(
             max_concurrency: Options.http.request_concurrency || MAX_CONCURRENCY
@@ -566,7 +593,8 @@ class Client
     end
 
     def client_run
-        @hydra.run
+        # Can get Ethon select errors.
+        exception_jail( false ) { @hydra.run }
     end
 
     def client_abort