arachni 1.1 → 1.2

data/lib/arachni/browser_cluster.rb

@@ -6,8 +6,6 @@
     web site for more information on licensing and terms of use.
 =end
 
-require 'monitor'
-
 module Arachni
 
 # Real browser driver providing DOM/JS/AJAX support.
@@ -63,10 +61,6 @@ class BrowserCluster
     #   List of crawled URLs with their HTTP codes.
     attr_reader :sitemap
 
-    # @return   [String]
-    #   Javascript token used to namespace the custom JS environment.
-    attr_reader :javascript_token
-
     # @return   [Array<Worker>]
     #   Worker pool.
     attr_reader :workers
@@ -87,6 +81,8 @@ class BrowserCluster
     #
     # @raise    ArgumentError   On missing `:handler` option.
     def initialize( options = {} )
+        super()
+
         {
             pool_size: Options.browser_cluster.pool_size
         }.merge( options ).each do |k, v|
@@ -122,15 +118,16 @@ class BrowserCluster
         @mutex       = Monitor.new
         @done_signal = Queue.new
 
-        # Javascript token to share across all workers, this needs to be static
-        # because of the browsers' disk-cache which leads to cached responses
-        # being used between runs.
-        @javascript_token = 'arachni_js_namespace'
-
         @consumed_pids = []
         initialize_workers
     end
 
+    # @return   [String]
+    #   Javascript token used to namespace the custom JS environment.
+    def javascript_token
+        Browser::Javascript::TOKEN
+    end
+
     # @note Operates in non-blocking mode.
     #
     # @param    [Block] block
@@ -154,6 +151,8 @@ class BrowserCluster
         synchronize do
             print_debug "Queueing: #{job}"
 
+            notify_on_queue job
+
             @pending_job_counter  += 1
             @pending_jobs[job.id] += 1
             @job_callbacks[job.id] = block if block
@@ -168,6 +167,14 @@ class BrowserCluster
         nil
     end
 
+    # def on_queue( &block )
+        # synchronize { @on_queue << block }
+    # end
+
+    # def on_job_done( &block )
+        # synchronize { @on_job_done << block }
+    # end
+
     # @param    [Page, String, HTTP::Response]  resource
     #   Resource to explore, if given a `String` it will be treated it as a URL
     #   and will be loaded.
@@ -206,6 +213,8 @@ class BrowserCluster
         synchronize do
             print_debug "Job done: #{job}"
 
+            notify_on_job_done job
+
             if !job.never_ending?
                 @skip_states_per_job.delete job.id
                 @job_callbacks.delete job.id
@@ -299,6 +308,7 @@ class BrowserCluster
     # @private
     def pop
         {} while job_done?( job = @jobs.pop )
+        notify_on_pop job
         job
     end
 
@@ -365,6 +375,23 @@ class BrowserCluster
 
     private
 
+    def notify_on_queue( job )
+        return if !@on_queue
+        @on_queue.call job
+    end
+
+    def notify_on_job_done( job )
+        return if !@on_job_done
+
+        @on_job_done.call job
+    end
+
+    def notify_on_pop( job )
+        return if !@on_pop
+
+        @on_pop.call job
+    end
+
     def fail_if_shutdown
         fail Error::AlreadyShutdown, 'Cluster has been shut down.' if @shutdown
     end
@@ -389,10 +416,9 @@ class BrowserCluster
         @workers = []
         pool_size.times do |i|
             worker = Worker.new(
-                javascript_token: @javascript_token,
-                master:           self,
-                width:            Options.browser_cluster.screen_width,
-                height:           Options.browser_cluster.screen_height
+                master: self,
+                width:  Options.browser_cluster.screen_width,
+                height: Options.browser_cluster.screen_height
             )
             @workers << worker
             @consumed_pids << worker.pid

data/lib/arachni/browser_cluster/job.rb

@@ -101,6 +101,10 @@ class Job
     # @param    [Hash]  data
     #   Used to initialize the {Result}.
     def save_result( data )
+        # Results coming in after the job has already finished won't have a
+        # browser.
+        return if !browser
+
         browser.master.handle_job_result(
             self.class::Result.new( data.merge( job: self.clean_copy ) )
         )

data/lib/arachni/browser_cluster/jobs/resource_exploration.rb

@@ -36,15 +36,6 @@ class ResourceExploration < Job
         browser.trigger_events
     end
 
-    def resource=( resource )
-        # Get a copy of the page with the caches cleared, this way when the
-        # modules (or anything else) lazy-load elements and populate the caches
-        # there won't be any lingering references to them from the more time
-        # consuming browser analysis.
-        resource = resource.dup if resource.is_a? Page
-        @resource = resource
-    end
-
     def dup
         super.tap { |j| j.resource = resource }
     end

data/lib/arachni/browser_cluster/worker.rb

@@ -45,7 +45,6 @@ class Worker < Arachni::Browser
     attr_reader   :time_to_live
 
     def initialize( options = {} )
-        javascript_token  = options.delete( :javascript_token )
         @master           = options.delete( :master )
 
         @max_time_to_live = options.delete( :max_time_to_live ) ||
@@ -59,8 +58,6 @@ class Worker < Arachni::Browser
         # as soon as they're logged.
         super options.merge( store_pages: false )
 
-        @javascript.token = javascript_token
-
         @done_signal = Queue.new
 
         start_capture
@@ -90,13 +87,19 @@ class Worker < Arachni::Browser
                 exception_jail false do
                     begin
                         @job.configure_and_run( self )
-                    rescue Selenium::WebDriver::Error::WebDriverError
+                    rescue Selenium::WebDriver::Error::WebDriverError,
+                        Watir::Exception::Error => e
+
+                        print_debug "Error while processing job: #{@job}"
+                        print_debug
+                        print_debug_exception e
+
                         browser_respawn
                     end
                 end
             end
         rescue TimeoutError => e
-            print_debug "Job timed-out after #{@job_timeout} seconds: #{@job}"
+            print_bad "Job timed-out after #{@job_timeout} seconds: #{@job}"
 
             # Could have left us with a broken browser.
             browser_respawn

data/lib/arachni/check/auditor.rb

@@ -80,16 +80,21 @@ module Auditor
             #   Element types to check for.
             #
             # @return   [Bool]
-            def self.check?( page, restrict_to_elements = nil )
+            def self.check?( page, restrict_to_elements = nil, ignore_dom_depth = false )
                 return false if issue_limit_reached?
                 return true  if elements.empty?
 
                 audit                = Arachni::Options.audit
                 restrict_to_elements = [restrict_to_elements].flatten.compact
 
+                # We use procs to make the decisions to avoid loading the page
+                # element caches unless it's absolutely necessary.
+                #
+                # Also, it's better to audit Form & Cookie DOM elements only
+                # after the page has gone through the browser, because then
+                # we'll have some context in the metadata which can help us
+                # optimize DOM audits.
                 {
-                    # We use procs to make the decision, to avoid loading the page
-                    # element caches unless it's absolutely necessary.
                     Element::Link              =>
                         proc { audit.links?     && !!page.links.find { |e| e.inputs.any? } },
                     Element::Link::DOM         =>
@@ -97,20 +102,22 @@ module Auditor
                     Element::Form              =>
                         proc { audit.forms? && !!page.forms.find { |e| e.inputs.any? } },
                     Element::Form::DOM         =>
-                        proc { audit.form_doms? && page.has_script? && !!page.forms.find(&:dom) },
+                        proc { (ignore_dom_depth || page.dom.depth > 0) &&
+                            audit.form_doms? && page.has_script? && !!page.forms.find(&:dom) },
                     Element::Cookie            =>
                         proc { audit.cookies? && page.cookies.any? },
                     Element::Cookie::DOM       =>
-                        proc { audit.cookie_doms? && page.has_script? && page.cookies.any? },
+                        proc { (ignore_dom_depth || page.dom.depth > 0) &&
+                            audit.cookie_doms? && page.has_script? && page.cookies.any? },
                     Element::Header            =>
                         proc { audit.headers? && page.headers.any? },
                     Element::LinkTemplate      =>
                         proc { audit.link_templates? && page.link_templates.find { |e| e.inputs.any? } },
                     Element::LinkTemplate::DOM =>
                         proc { audit.link_template_doms? && !!page.link_templates.find(&:dom) },
-                    Element::JSON      =>
+                    Element::JSON              =>
                         proc { audit.jsons? && page.jsons.find { |e| e.inputs.any? } },
-                    Element::XML      =>
+                    Element::XML               =>
                         proc { audit.xmls? && page.xmls.find { |e| e.inputs.any? } },
                     Element::Body              => !page.body.empty?,
                     Element::GenericDOM        => page.has_script?,
@@ -348,17 +355,22 @@ module Auditor
     #       If `false`, a message will be printed to stdout containing the status of
     #       the operation.
     #
+    # @return   [Issue]
+    #
     # @see #log_issue
     def log_remote_file( page_or_response, silent = false )
         page = page_or_response.is_a?( Page ) ?
             page_or_response : page_or_response.to_page
 
-        log_issue(
+        issue = log_issue(
             vector: Element::Server.new( page.url ),
+            proof:  page.response.status_line,
             page:   page
         )
 
         print_ok( "Found #{page.url}" ) if !silent
+
+        issue
     end
     alias :log_remote_directory :log_remote_file
 

data/lib/arachni/check/base.rb

@@ -152,18 +152,50 @@ class Base < Component::Base
             @platforms ||= [info[:platforms]].flatten.compact
         end
 
-        # @param    [Array<Symbol, String>]     platforms
+        # @return   [Bool]
+        #   `true` if the check has specified platforms for which it does not apply.
+        #
+        # @see .platforms
+        def has_exempt_platforms?
+            exempt_platforms.any?
+        end
+
+        # @return   [Array<Symbol>]
+        #   Platforms not applicable to this check.
+        #
+        # @see .info
+        def exempt_platforms
+            @exempt_platforms ||= [info[:exempt_platforms]].flatten.compact
+        end
+
+        # @param    [Array<Symbol, String>]     resource_platforms
         #   List of platforms to check for support.
         #
         # @return   [Boolean]
         #   `true` if any of the given platforms are supported, `false` otherwise.
-        def supports_platforms?( platforms )
-            return true if platforms.empty? || !has_platforms?
+        def supports_platforms?( resource_platforms )
+            if resource_platforms.any? && has_exempt_platforms?
+                manager = Platform::Manager.new( exempt_platforms )
+
+                resource_platforms.each do |p|
+
+                    # When we check for exempt platforms we're looking for info
+                    # from the same type.
+                    ptype = Platform::Manager.find_type( p )
+                    type_manager = manager.send( ptype )
+
+                    return false if type_manager.pick( p => true ).any?
+                end
+            end
+
+            return true if resource_platforms.empty? || !has_platforms?
 
             # Determine if we've got anything for the given platforms, the same
             # way payloads are picked.
-            foo_data = self.platforms.inject({}) { |h, platform| h.merge!( platform => true ) }
-            Platform::Manager.new( platforms ).pick( foo_data ).any?
+            foo_data = self.platforms.
+                inject({}) { |h, platform| h.merge!( platform => true ) }
+
+            Platform::Manager.new( resource_platforms ).pick( foo_data ).any?
         end
 
         # @return   [Array<Symbol>]
@@ -193,7 +225,7 @@ class Base < Component::Base
 
         # @private
         def clear_info_cache
-            @elements = @platforms = nil
+            @elements = @exempt_platforms = @platforms = nil
         end
     end
 

data/lib/arachni/element/base.rb

@@ -38,6 +38,19 @@ class Base
 
     include Capabilities::WithScope
 
+    # Maximum element size in bytes.
+    # Anything larger than this should be exempt from parse and storage or have
+    # its value ignored.
+    #
+    # During the audit, thousands of copies will be generated and the same
+    # amount of HTP requests will be stored in the HTTP::Client queue.
+    # Thus, elements with inputs of excessive size will lead to excessive RAM
+    # consumption.
+    #
+    # This will almost never be necessary, but there have been cases of
+    # buggy `_VIEWSTATE` inputs that grow infinitely.
+    MAX_SIZE = 10_000
+
     # @return     [Page]
     #   Page this element belongs to.
     attr_accessor :page
@@ -124,7 +137,7 @@ class Base
     # @return   [Symbol]
     #   Element type.
     def self.type
-        name.split( ':' ).last.downcase.to_sym
+        @type ||= name.split( ':' ).last.downcase.to_sym
     end
 
     def dup
@@ -189,6 +202,10 @@ class Base
         instance
     end
 
+    def self.too_big?( element )
+        (element.is_a?( Numeric ) ? element : element.to_s.size) >= MAX_SIZE
+    end
+
 end
 end
 end

data/lib/arachni/element/capabilities/analyzable/differential.rb

@@ -28,7 +28,6 @@ module Differential
     end
 
     DIFFERENTIAL_OPTIONS =  {
-        # Append our seeds to the default values.
         format:         [Mutable::Format::STRAIGHT],
 
         # Amount of refinement operations to remove context-irrelevant dynamic

data/lib/arachni/element/capabilities/analyzable/taint.rb

@@ -15,6 +15,10 @@ module Analyzable
 # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
 module Taint
 
+    TAINT_CACHE = {
+        match: Support::Cache::LeastRecentlyPushed.new( 10_000 )
+    }
+
     TAINT_OPTIONS = {
         # The regular expression to match against the response body.
         #
@@ -99,28 +103,38 @@ module Taint
     end
 
     def match_patterns( patterns, matcher, response, opts )
+        k = [patterns, response.body]
+        return if TAINT_CACHE[:match][k] == false
+
         if opts[:longest_word_optimization]
             opts[:downcased_body] = response.body.downcase
         end
 
         case patterns
             when Regexp, String, Array
-                [patterns].flatten.compact.
-                    each { |pattern| matcher.call( pattern, response, opts ) }
+                [patterns].flatten.compact.each do |pattern|
+                    res = matcher.call( pattern, response, opts )
+                    TAINT_CACHE[:match][k] ||= !!res
+                end
 
             when Hash
                 if opts[:platform] && patterns[opts[:platform]]
                     [patterns[opts[:platform]]].flatten.compact.each do |p|
-                        [p].flatten.compact.
-                            each { |pattern| matcher.call( pattern, response, opts ) }
+                        [p].flatten.compact.each do |pattern|
+                            res = matcher.call( pattern, response, opts )
+                            TAINT_CACHE[:match][k] ||= !!res
+                        end
                     end
+
                 else
                     patterns.each do |platform, p|
                         dopts = opts.dup
                         dopts[:platform] = platform
 
-                        [p].flatten.compact.
-                            each { |pattern| matcher.call( pattern, response, dopts ) }
+                        [p].flatten.compact.each do |pattern|
+                            res = matcher.call( pattern, response, dopts )
+                            TAINT_CACHE[:match][k] ||= !!res
+                        end
                     end
                 end
 
@@ -133,15 +147,22 @@ module Taint
                         dopts = opts.dup
                         dopts[:platform] = platform
 
-                        [p].flatten.compact.
-                            each { |pattern| matcher.call( pattern, response, dopts ) }
+                        [p].flatten.compact.each do |pattern|
+                            res = matcher.call( pattern, response, dopts )
+                            TAINT_CACHE[:match][k] ||= !!res
+                        end
                     end
         end
     end
 
     def match_substring_and_log( substring, response, opts )
         return if substring.to_s.empty?
-        return if !response.body.include?( substring ) || ignore?( response, opts )
+
+        k = [substring, response.body]
+        return if TAINT_CACHE[:match][k] == false
+
+        TAINT_CACHE[:match][k] = includes = response.body.include?( substring )
+        return if !includes || ignore?( response, opts )
 
         @candidate_issues << {
             response:  response,
@@ -151,9 +172,14 @@ module Taint
             vector:    response.request.performer
         }
         setup_verification_callbacks
+
+        true
     end
 
     def match_regexp_and_log( regexp, response, opts )
+        k = [regexp, response.body]
+        return if TAINT_CACHE[:match][k] == false
+
         regexp = regexp.is_a?( Regexp ) ? regexp :
             Regexp.new( regexp.to_s, Regexp::IGNORECASE )
 
@@ -166,7 +192,9 @@ module Taint
 
         match_data = match_data[0].to_s
 
-        return if match_data.to_s.empty? || ignore?( response, opts )
+        TAINT_CACHE[:match][k] = !match_data.empty?
+
+        return if match_data.empty? || ignore?( response, opts )
 
         @candidate_issues << {
             response:  response,
@@ -176,6 +204,8 @@ module Taint
             vector:    response.request.performer
         }
         setup_verification_callbacks
+
+        true
     end
 
     def ignore?( res, opts )