arachni 1.0.4 → 1.0.5

data/lib/arachni/browser/javascript.rb

@@ -29,6 +29,10 @@ class Javascript
     #   Filesystem directory containing the JS scripts.
     SCRIPT_LIBRARY  = "#{File.dirname( __FILE__ )}/javascript/scripts/"
 
+    SCRIPT_SOURCES = Dir.glob("#{SCRIPT_LIBRARY}*.js").inject({}) do |h, path|
+        h.merge!( path => IO.read(path) )
+    end
+
     NO_EVENTS_FOR_ELEMENTS = Set.new([
         :base, :bdo, :br, :head, :html, :iframe, :meta, :param, :script, :style,
         :title, :link
@@ -198,6 +202,17 @@ class Javascript
         @browser.watir.execute_script script
     end
 
+    # Executes the given code but unwraps Watir elements.
+    #
+    # @param    [String]    script
+    #   JS code to execute.
+    #
+    # @return   [Object]
+    #   Result of `script`.
+    def run_without_elements( script )
+        unwrap_elements run( script )
+    end
+
     # @return   (see TaintTracer#debug)
     def debugging_data
         return [] if !supported?
@@ -361,18 +376,18 @@ class Javascript
     def read_script( filename )
         @scripts ||= {}
         @scripts[filename] ||=
-            IO.read( filesystem_path_for_script( filename ) ).
-                gsub( '_token', "_#{token}" ).freeze
+            SCRIPT_SOURCES[filesystem_path_for_script(filename)].
+                gsub( '_token', "_#{token}" )
     end
 
     def script_exists?( filename )
-        (!!read_script( filename )) rescue false
+        SCRIPT_SOURCES.include? filesystem_path_for_script( filename )
     end
 
     def filesystem_path_for_script( filename )
         name = "#{SCRIPT_LIBRARY}#{filename}"
         name << '.js' if !name.end_with?( '.js')
-        name
+        File.expand_path( name )
     end
 
     def script_url_for( filename )
@@ -384,6 +399,27 @@ class Javascript
         "#{SCRIPT_BASE_URL}#{filename}.js"
     end
 
+    def unwrap_elements( obj )
+        case obj
+            when Watir::Element
+                unwrap_element( obj )
+
+            when Array
+                obj.map { |e| unwrap_elements( e ) }
+
+            when Hash
+                obj.each { |k, v| obj[k] = unwrap_elements( v ) }
+                obj
+
+            else
+                obj
+        end
+    end
+
+    def unwrap_element( element )
+        element.html
+    end
+
 end
 end
 end

data/lib/arachni/browser/javascript/proxy.rb

@@ -70,7 +70,7 @@ class Proxy < BasicObject
     #   Javascript property/function.
     # @param    [Array]    arguments
     def call( function, *arguments )
-        @javascript.run "return #{stub.write( function, *arguments )}"
+        @javascript.run_without_elements "return #{stub.write( function, *arguments )}"
     end
     alias :method_missing :call
 

data/lib/arachni/browser_cluster/worker.rb

@@ -82,7 +82,8 @@ class Worker < Arachni::Browser
 
         # PhantomJS may have crashed (it happens sometimes) so make sure that
         # we've got a live one before running the job.
-        browser_respawn_if_necessary
+        # If we can't respawn, then bail out.
+        return if browser_respawn_if_necessary.nil?
 
         begin
             with_timeout @job_timeout do
@@ -234,7 +235,7 @@ class Worker < Arachni::Browser
     end
 
     def browser_respawn_if_necessary
-        return if !time_to_die? && browser_alive? &&
+        return false if !time_to_die? && browser_alive? &&
             watir.windows.size < RESPAWN_WHEN_WINDOW_COUNT_REACHES
 
         browser_respawn
@@ -257,9 +258,18 @@ class Worker < Arachni::Browser
         @watir    = nil
         @selenium = nil
 
-        @watir = ::Watir::Browser.new( selenium )
+        # Browser may fail to respawn but there's nothing we can do about
+        # that, just leave it dead and try again at the next job.
+        begin
+            @watir = ::Watir::Browser.new( selenium )
+
+            ensure_open_window
 
-        ensure_open_window
+            true
+        rescue Browser::Error::Spawn => e
+            print_error 'Could not respawn the browser, will try again at the next job.'
+            nil
+        end
     end
 
     def time_to_die?

data/lib/arachni/check/auditor.rb

@@ -380,7 +380,8 @@ module Auditor
     #
     # @see  Page#audit?
     def skip?( element )
-        return true if !page.audit_element?( element )
+        return true if audited?( element.coverage_id ) ||
+            !page.audit_element?( element )
 
         # Don't audit elements which have been already logged as vulnerable
         # either by us or preferred checks.
@@ -390,6 +391,10 @@ module Auditor
             klass = framework.checks[check]
             next if !klass.info.include?(:issue)
 
+            # No point in doing the following heavy deduplication check if there
+            # are no issues logged to begin with.
+            next if klass.issue_counter == 0
+
             if Data.issues.include?( klass.create_issue( vector: element ) )
                 return true
             end
@@ -499,7 +504,10 @@ module Auditor
         if !block_given?
             audit_taint( payloads, opts )
         else
-            each_candidate_element( opts[:elements] ) { |e| e.audit( payloads, opts, &block ) }
+            each_candidate_element( opts[:elements] ) do |e|
+                e.audit( payloads, opts, &block )
+                audited( e.coverage_id )
+            end
         end
     end
 
@@ -512,7 +520,10 @@ module Auditor
     # @see Arachni::Element::Capabilities::Analyzable::Taint
     def audit_taint( payloads, opts = {} )
         opts = OPTIONS.merge( opts )
-        each_candidate_element( opts[:elements] ) { |e| e.taint_analysis( payloads, opts ) }
+        each_candidate_element( opts[:elements] )do |e|
+            e.taint_analysis( payloads, opts )
+            audited( e.coverage_id )
+        end
     end
 
     # Audits elements using differential analysis and automatically logs results.
@@ -523,7 +534,10 @@ module Auditor
     # @see Arachni::Element::Capabilities::Analyzable::Differential
     def audit_differential( opts = {}, &block )
         opts = OPTIONS.merge( opts )
-        each_candidate_element( opts[:elements] ) { |e| e.differential_analysis( opts, &block ) }
+        each_candidate_element( opts[:elements] ) do |e|
+            e.differential_analysis( opts, &block )
+            audited( e.coverage_id )
+        end
     end
 
     # Audits elements using timing attacks and automatically logs results.
@@ -534,7 +548,10 @@ module Auditor
     # @see Arachni::Element::Capabilities::Analyzable::Timeout
     def audit_timeout( payloads, opts = {} )
         opts = OPTIONS.merge( opts )
-        each_candidate_element( opts[:elements] ) { |e| e.timeout_analysis( payloads, opts ) }
+        each_candidate_element( opts[:elements] ) do |e|
+            e.timeout_analysis( payloads, opts )
+            audited( e.coverage_id )
+        end
     end
 
     # Traces the taint in the given `resource` and passes each page to the
@@ -581,7 +598,7 @@ module Auditor
 
     def prepare_each_element( elements, &block )
         elements.each do |e|
-            next if e.inputs.empty?
+            next if skip?( e ) || e.inputs.empty?
 
             d = e.dup
             d.auditor = self
@@ -591,7 +608,7 @@ module Auditor
 
     def prepare_each_dom_element( elements, &block )
         elements.each do |e|
-            next if !e.dom || e.dom.inputs.empty?
+            next if skip?( e ) || !e.dom || e.dom.inputs.empty?
 
             d = e.dup
             d.dom.auditor = self

data/lib/arachni/check/manager.rb

@@ -116,6 +116,10 @@ class Manager < Arachni::Component::Manager
     #   Check to run as a class.
     # @param    [Page]   page
     #   Page to audit.
+    #
+    # @return   [Bool]
+    #   `true` if the check was ran (based on {Check::Auditor.check?}),
+    #   `false` otherwise.
     def run_one( check, page )
         return false if !check.check?( page )
 
@@ -123,6 +127,8 @@ class Manager < Arachni::Component::Manager
         check_new.prepare
         check_new.run
         check_new.clean_up
+
+        true
     end
 
     def self.reset

data/lib/arachni/framework.rb

@@ -272,8 +272,6 @@ class Framework
 
         print_line
         print_status "[HTTP: #{page.code}] #{page.dom.url}"
-        # print_object_space
-        # print_with_statistics
 
         if page.platforms.any?
             print_info "Identified as: #{page.platforms.to_a.join( ', ' )}"
@@ -293,6 +291,9 @@ class Framework
             end
         end
 
+        # Aside from plugins and whatnot, the Trainer hooks here to update the
+        # ElementFilter so that it'll know if new elements appear during the
+        # audit, so it's a big deal.
         notify_on_page_audit( page )
 
         @current_url = page.dom.url.to_s
@@ -300,22 +301,23 @@ class Framework
         http.update_cookies( page.cookie_jar )
         perform_browser_analysis( page )
 
+        # Remove elements which have already passed through here.
+        pre_audit_element_filter( page )
+
         # Run checks which **don't** benefit from fingerprinting first, so that
         # we can use the responses of their HTTP requests to fingerprint the
         # webapp platforms, so that the checks which **do** benefit from knowing
         # the remote platforms can run more efficiently.
         ran = false
         @checks.without_platforms.values.each do |check|
-            ran = true
-            check_page( check, page )
+            ran = true if check_page( check, page )
         end
         harvest_http_responses if ran
         run_http = ran
 
         ran = false
         @checks.with_platforms.values.each do |check|
-            ran = true
-            check_page( check, page )
+            ran = true if check_page( check, page )
         end
         harvest_http_responses if ran
         run_http ||= ran
@@ -1201,7 +1203,53 @@ class Framework
         rescue => e
             print_error "Error in #{check.to_s}: #{e.to_s}"
             print_error_backtrace e
+            false
+        end
+    end
+
+    # Small but (sometimes) important optimization:
+    #
+    # Keep track of page elements which have already been passed to checks,
+    # in order to filter them out and hopefully even avoid running checks
+    # against pages with no new elements.
+    #
+    # It's not like there were going to be redundant audits anyways, because
+    # each layer of the audit performs its own redundancy checks, but those
+    # redundancy checks can introduce significant latencies when dealing
+    # with pages with lots of elements.
+    def pre_audit_element_filter( page )
+        redundant_elements  = {}
+        page.elements.each do |e|
+            next if !Options.audit.element?( e.type )
+            next if e.is_a?( Cookie ) || e.is_a?( Header )
+
+            new_element                  = false
+            redundant_elements[e.type] ||= []
+
+            if !state.element_checked?( e )
+                state.element_checked e
+                new_element = true
+            end
+
+            if e.respond_to?( :dom ) && e.dom
+                if !state.element_checked?( e.dom )
+                    state.element_checked e.dom
+                    new_element = true
+                end
+            end
+
+            next if new_element
+
+            redundant_elements[e.type] << e
+        end
+
+        # Remove redundant elements from the page cache, if there are thousands
+        # of them then just skipping them during the audit will introduce latency.
+        redundant_elements.each do |type, elements|
+            page.send( "#{type}s=", page.send( "#{type}s" ) - elements )
         end
+
+        page
     end
 
     def add_to_sitemap( page )

data/lib/arachni/http/client.rb

@@ -131,23 +131,21 @@ class Client
         clear_observers if hooks_too
         State.http.clear
 
-        opts = Options
-
-        @url = opts.url.to_s
+        @url = Options.url.to_s
         @url = nil if @url.empty?
 
-        @hydra = Typhoeus::Hydra.new( max_concurrency: opts.http.request_concurrency || MAX_CONCURRENCY )
+        client_initialize
 
         headers.merge!(
             'Accept'     => 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-            'User-Agent' => opts.http.user_agent
+            'User-Agent' => Options.http.user_agent
         )
-        headers['From'] = opts.authorized_by if opts.authorized_by
-        headers.merge!( opts.http.request_headers )
+        headers['From'] = Options.authorized_by if Options.authorized_by
+        headers.merge!( Options.http.request_headers )
 
-        cookie_jar.load( opts.http.cookie_jar_filepath ) if opts.http.cookie_jar_filepath
-        update_cookies( opts.http.cookies )
-        update_cookies( opts.http.cookie_string ) if opts.http.cookie_string
+        cookie_jar.load( Options.http.cookie_jar_filepath ) if Options.http.cookie_jar_filepath
+        update_cookies( Options.http.cookies )
+        update_cookies( Options.http.cookie_string ) if Options.http.cookie_string
 
         reset_burst_info
 
@@ -203,7 +201,7 @@ class Client
             @burst_runtime = nil
 
             begin
-                hydra_run
+                run_and_update_statistics
 
                 duped_after_run = observers_for( :after_run ).dup
                 observers_for( :after_run ).clear
@@ -258,7 +256,7 @@ class Client
 
     # Aborts the running requests on a best effort basis.
     def abort
-        exception_jail { @hydra.abort }
+        exception_jail { client_abort }
     end
 
     # @return   [Integer]
@@ -688,11 +686,11 @@ class Client
         false
     end
 
-    def hydra_run
+    def run_and_update_statistics
         @running = true
 
         reset_burst_info
-        @hydra.run
+        client_run
 
         @queue_size = 0
         @running    = false
@@ -700,7 +698,7 @@ class Client
         @burst_runtime += Time.now - @burst_runtime_start
         @total_runtime += @burst_runtime
     end
-
+    
     def reset_burst_info
         @burst_response_time_sum = 0
         @burst_response_count    = 0
@@ -771,20 +769,40 @@ class Client
 
         return if request.blocking?
 
+        if client_queue( request )
+            @queue_size += 1
+
+            if emergency_run?
+                print_info 'Request queue reached its maximum size, performing an emergency run.'
+                run_and_update_statistics
+            end
+        end
+
+        request
+    end
+
+    def client_initialize
+        @hydra = Typhoeus::Hydra.new(
+            max_concurrency: Options.http.request_concurrency || MAX_CONCURRENCY
+        )
+    end
+
+    def client_run
+        @hydra.run
+    end
+
+    def client_abort
+        @hydra.abort
+    end
+
+    def client_queue( request )
         if request.high_priority?
             @hydra.queue_front( request.to_typhoeus )
         else
             @hydra.queue( request.to_typhoeus )
         end
 
-        @queue_size += 1
-
-        if emergency_run?
-            print_info 'Request queue reached its maximum size, performing an emergency run.'
-            hydra_run
-        end
-
-        request
+        true
     end
 
     def emergency_run?