arachni 1.5.1 → 1.6.1.1

data/lib/arachni/browser.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework
@@ -7,8 +7,9 @@
 =end
 
 require 'childprocess'
-require 'watir-webdriver'
+require 'watir'
 require_relative 'selenium/webdriver/element'
+require_relative 'selenium/webdriver/remote/typhoeus'
 require_relative 'processes/manager'
 require_relative 'browser/element_locator'
 require_relative 'browser/javascript'
@@ -57,6 +58,11 @@ class Browser
         # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
         class Load < Error
         end
+
+        # @author Tasos "Zapotek" Laskos <tasos.laskos@arachni-scanner.com>
+        class MissingExecutable < Error
+        end
+
     end
 
     # How much time to wait for the PhantomJS process to spawn before respawning.
@@ -80,12 +86,6 @@ class Browser
         /src\s*=\s*['"]?(.*?)?['"]?[\s>]/i,
     ]
 
-    # Unfortunately, we can't expose the HTTP user-agent for client-side
-    # stuff, because Selenium needs to know that we're using a Webkit-based
-    # browser in order to use the right JS code to trigger events etc.
-    USER_AGENT = 'Mozilla/5.0 AppleWebKit/538.1 (KHTML, like Gecko) ' <<
-        "Arachni/#{Arachni::VERSION} Safari/538.1"
-
     # @return   [Array<Page::DOM::Transition>]
     attr_reader :transitions
 
@@ -124,14 +124,6 @@ class Browser
     # @see #skip_state?
     attr_reader :skip_states
 
-    # @return   [Integer]
-    #   PID of the lifeline process managing the browser process.
-    attr_reader :lifeline_pid
-
-    # @return   [Integer]
-    #   PID of the browser process.
-    attr_reader :browser_pid
-
     attr_reader :last_url
 
     class <<self
@@ -139,13 +131,26 @@ class Browser
         # @return   [Bool]
         #   `true` if a supported browser is in the OS PATH, `false` otherwise.
         def has_executable?
-            !!executable
+            executable
+            true
+        rescue Error::MissingExecutable
+            false
         end
 
         # @return   [String]
         #   Path to the PhantomJS executable.
         def executable
-            Selenium::WebDriver::PhantomJS.path
+            @path ||= begin
+                path = Selenium::WebDriver::Platform.find_binary('chromedriver')
+                raise Error::MissingExecutable, 'chromedriver could not be found in PATH.' unless path
+                Selenium::WebDriver::Platform.assert_executable path
+                path
+            end
+        end
+
+        # @ private
+        def reset
+            @path = nil
         end
 
         def asset_domains
@@ -345,7 +350,7 @@ class Browser
                     ).until { @selenium.find_element( :css, css ) }
 
                     print_info "#{css.inspect} appeared for: #{url}"
-                rescue Selenium::WebDriver::Error::TimeOutError
+                rescue Selenium::WebDriver::Error::TimeoutError
                     print_bad "#{css.inspect} did not appear for: #{url}"
                 end
 
@@ -370,35 +375,12 @@ class Browser
 
     def wait_till_ready
         @javascript.wait_till_ready
-        wait_for_timers
-        wait_for_pending_requests
-    end
 
-    def shutdown
-        print_debug 'Shutting down...'
-
-        print_debug_level_2 'Killing process.'
-        if @kill_process
-            begin
-                @kill_process.close
-            rescue => e
-                print_debug_exception e
-            end
+        if Options.browser_cluster.wait_for_timers?
+            wait_for_timers
         end
 
-        print_debug_level_2 'Shutting down proxy...'
-        @proxy.shutdown rescue Reactor::Error::NotRunning
-        print_debug_level_2 '...done.'
-
-        @proxy        = nil
-        @kill_process = nil
-        @watir        = nil
-        @selenium     = nil
-        @lifeline_pid = nil
-        @browser_pid  = nil
-        @browser_url  = nil
-
-        print_debug '...shutdown complete.'
+        wait_for_pending_requests
     end
 
     # @return   [String]
@@ -612,7 +594,9 @@ class Browser
 
         notify_on_fire_event( element, event )
 
-        pre_timeouts = javascript.timeouts
+        if Options.browser_cluster.wait_for_timers?
+            pre_timeouts = javascript.timeouts
+        end
 
         begin
             transition = Page::DOM::Transition.new( locator, event, options ) do
@@ -679,13 +663,15 @@ class Browser
 
             print_debug_level_2 "[done in #{transition.time}s]: #{event} (#{options}) #{locator}"
 
-            delay = (javascript.timeouts - pre_timeouts).compact.map { |t| t[1].to_i }.max
-            if delay
-                print_debug_level_2 "Found new timers with max #{delay}ms."
-                delay = [Options.http.request_timeout, delay].min / 1000.0
+            if Options.browser_cluster.wait_for_timers?
+                delay = (javascript.timeouts - pre_timeouts).compact.map { |t| t[1].to_i }.max
+                if delay
+                    print_debug_level_2 "Found new timers with max #{delay}ms."
+                    delay = [Options.http.request_timeout, delay].min / 1000.0
 
-                print_debug_level_2 "Will wait for #{delay}s."
-                sleep delay
+                    print_debug_level_2 "Will wait for #{delay}s."
+                    sleep delay
+                end
             end
 
             transition
@@ -1040,7 +1026,6 @@ class Browser
     end
 
     def load_delay
-        #(intervals + timeouts).map { |t| t[1] }.max
         @javascript.timeouts.compact.map { |t| t[1].to_i }.max
     end
 
@@ -1101,30 +1086,83 @@ class Browser
     def selenium
         return @selenium if @selenium
 
-        # For some weird reason the Typhoeus client is very slow for
-        # PhantomJS 2.1.1 and causes a boatload of time-outs.
-        client = Selenium::WebDriver::Remote::Http::Default.new
-        client.timeout = Options.browser_cluster.job_timeout
+        start_proxy
 
-        @selenium = Selenium::WebDriver.for(
-            :remote,
+        proxy_uri = URI( @proxy.url )
+
+        dir = "#{Options.paths.tmpdir}/Arachni_Chrome_#{self.object_id}/"
+        FileUtils.rm_rf dir
+        FileUtils.mkdir dir
+        at_exit do
+            FileUtils.rm_rf dir
+        end
 
-            # We need to spawn our own PhantomJS process because Selenium's
-            # way sometimes gives us zombies.
-            url:                  spawn_browser,
-            desired_capabilities: capabilities,
-            http_client:          client
+        @selenium = Selenium::WebDriver.for(
+            :chrome,
+            capabilities: Selenium::WebDriver::Chrome::Options.new(
+                emulation: {
+                    userAgent: Arachni::Options.http.user_agent
+                },
+                args: [
+                        '--allow-running-insecure-content',
+                        '--disable-web-security',
+                        '--reduce-security-for-testing',
+                        '--ignore-certificate-errors',
+                        '--no-sandbox',
+                        '--disable-plugins',
+                        "--user-data-dir=#{dir}",
+                        "--proxy-server=#{proxy_uri.host}:#{proxy_uri.port}",
+                        "--buid=46464646",
+                        "--headless"
+                ]
+            ),
+            http_client: Selenium::WebDriver::Remote::Http::Typhoeus.new
         )
+
+    rescue Selenium::WebDriver::Error::WebDriverError => e
+        print_error "Please ensure that chromedriver and Chrome are the same" <<
+                      " version and in your PATH."
+        raise e
     end
 
-    def alive?
-        @lifeline_pid && Processes::Manager.alive?( @lifeline_pid )
+    def shutdown
+        print_debug 'Shutting down...'
+
+        if @selenium
+            @selenium.close
+
+            print_debug_level_2 'Quiting Selenium...'
+            # So freaking hacky but @selenium.quit freezes if we don't detach first.
+            @selenium.instance_eval do
+                bridge.quit
+
+                @service.instance_eval do
+                    Process.detach @process.pid
+                    @process.stop
+                end
+            end
+
+            @selenium.quit rescue Errno::ECONNREFUSED
+            # @selenium.quit rescue Selenium::WebDriver::Error::WebDriverError
+            print_debug_level_2 '...done.'
+
+        end
+
+        if @proxy
+            print_debug_level_2 'Shutting down proxy...'
+            @proxy.shutdown rescue Reactor::Error::NotRunning
+            print_debug_level_2 '...done.'
+        end
+
+        @proxy    = nil
+        @watir    = nil
+        @selenium = nil
+
+        print_debug '...shutdown complete.'
     end
 
     def inspect
         s = "#<#{self.class} "
-        s << "pid=#{@lifeline_pid} "
-        s << "browser_pid=#{@browser_pid} "
         s << "last-url=#{@last_url.inspect} "
         s << "transitions=#{@transitions.size}"
         s << '>'
@@ -1213,107 +1251,6 @@ class Browser
         Options.input.value_for_name( name )
     end
 
-    def spawn_browser
-        if !spawn_phantomjs
-            fail Error::Spawn, 'Could not start the browser process.'
-        end
-
-        @browser_url
-    end
-
-    def spawn_phantomjs
-        return @browser_url if @browser_url
-
-        print_debug 'Spawning PhantomJS...'
-
-        ChildProcess.posix_spawn = true
-
-        port   = nil
-        output = ''
-
-        10.times do |i|
-            # Clear output of previous attempt.
-            output = ''
-            done   = false
-            port   = Utilities.available_port
-
-            start_proxy
-
-            print_debug_level_2 "Attempt ##{i}, chose port number #{port}"
-
-            begin
-                with_timeout BROWSER_SPAWN_TIMEOUT do
-                    print_debug_level_2 "Spawning process: #{self.class.executable}"
-
-                    r, w  = IO.pipe
-                    ri, @kill_process = IO.pipe
-
-                    @lifeline_pid = Processes::Manager.spawn(
-                        :browser,
-                        executable: self.class.executable,
-                        without_arachni: true,
-                        fork: false,
-                        new_pgroup: true,
-                        stdin: ri,
-                        stdout: w,
-                        stderr: w,
-                        port: port,
-                        proxy_url: @proxy.url
-                    )
-
-                    w.close
-                    ri.close
-
-                    print_debug_level_2 'Process spawned, waiting for WebDriver server...'
-
-                    # Wait for PhantomJS to initialize.
-                     while !output.include?( 'running on port' )
-                         begin
-                             output << r.readpartial( 8192 )
-                         # EOF or something, take a breather before retrying.
-                         rescue
-                             sleep 0.05
-                         end
-                     end
-
-                    @browser_pid = output.scan( /^PID: (\d+)/ ).flatten.first.to_i
-
-                    print_debug_level_2 '...WebDriver server is up.'
-                    done = true
-                end
-            rescue Timeout::Error
-                print_debug 'Spawn timed-out.'
-            end
-
-            if !output.empty?
-                print_debug_level_2 output
-            end
-
-            if done
-                print_debug 'PhantomJS is ready.'
-                break
-            end
-
-            print_debug_level_2 'Killing process.'
-
-            # Kill everything.
-            shutdown
-        end
-
-        # Something went really bad, the browser couldn't be spawned even
-        # after our valiant efforts.
-        #
-        # Bail out for now and count on the BrowserCluster to retry to boot
-        # another process ass needed.
-        if !@lifeline_pid
-            log_error 'Could not spawn browser process.'
-            log_error output
-            return
-        end
-
-        @browser_url = "http://127.0.0.1:#{port}"
-    end
-
     def start_proxy
         print_debug 'Booting up...'
 
@@ -1335,7 +1272,7 @@ class Browser
     def start_webdriver
         print_debug_level_2 'Starting WebDriver...'
         @watir = ::Watir::Browser.new( selenium )
-        print_debug_level_2 "... started WebDriver at: #{@browser_url}"
+        print_debug_level_2 "... started WebDriver."
 
         print_debug '...boot-up completed.'
     end
@@ -1422,16 +1359,17 @@ class Browser
         end
 
         return if set_cookies.empty? &&
-            Arachni::Options.browser_cluster.local_storage.empty?
+            Options.browser_cluster.local_storage.empty? &&
+            Options.browser_cluster.session_storage.empty?
 
         set_cookie = set_cookies.values.map(&:to_set_cookie)
         print_debug_level_2 "Setting cookies: #{set_cookie}"
 
         body = ''
-        if Arachni::Options.browser_cluster.local_storage.any?
-            body = <<EOJS
+        if Options.browser_cluster.local_storage.any?
+            body << <<EOJS
                 <script>
-                    var data = #{Arachni::Options.browser_cluster.local_storage.to_json};
+                    var data = #{Options.browser_cluster.local_storage.to_json};
 
                     for( prop in data ) {
                         localStorage.setItem( prop, data[prop] );
@@ -1440,6 +1378,18 @@ class Browser
 EOJS
         end
 
+        if Options.browser_cluster.session_storage.any?
+            body << <<EOJS
+                <script>
+                    var data = #{Options.browser_cluster.session_storage.to_json};
+
+                    for( prop in data ) {
+                        sessionStorage.setItem( prop, data[prop] );
+                    }
+                </script>
+EOJS
+        end
+
         @selenium.navigate.to preload( HTTP::Response.new(
             code:    200,
             url:     "#{url}/set-cookies-#{request_token}",
@@ -1475,43 +1425,6 @@ EOJS
         @selenium.manage.window.resize_to( @width, @height )
     end
 
-    # # Firefox driver, only used for debugging.
-    # def firefox
-    #     profile = Selenium::WebDriver::Firefox::Profile.new
-    #     profile.proxy = Selenium::WebDriver::Proxy.new http: @proxy.address,
-    #                                                    ssl: @proxy.address
-    #     [:firefox, profile: profile]
-    # end
-    #
-    # # Chrome driver, only used for debugging.
-    # def chrome
-    #     [ :chrome, switches: [ "--proxy-server=#{@proxy.address}" ] ]
-    # end
-
-    def capabilities
-        Selenium::WebDriver::Remote::Capabilities.phantomjs(
-            # Selenium tries to be helpful by including screenshots for errors
-            # in the JSON response. That's not gonna fly in this use case as
-            # parsing lots of massive JSON responses at the same time will
-            # have a significant impact on performance.
-            takes_screenshot: false,
-
-            # Needs to include the string Webkit:
-            #   https://github.com/ariya/phantomjs/issues/14198
-            #
-            # Default is:
-            #   Mozilla/5.0 (Unknown; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) PhantomJS/2.1.1 Safari/538.1
-            'phantomjs.page.settings.userAgent'                   =>
-                USER_AGENT,
-            'phantomjs.page.customHeaders.X-Arachni-Browser-Auth' =>
-                auth_token,
-            'phantomjs.page.settings.resourceTimeout'             =>
-                Options.http.request_timeout,
-            'phantomjs.page.settings.loadImages'                  =>
-                !Options.browser_cluster.ignore_images
-        )
-    end
-
     def flush_request_transitions
         @request_transitions.dup
     ensure
@@ -1529,8 +1442,8 @@ EOJS
     def request_handler( request, response )
         request.performer = self
 
-        return if request.headers['X-Arachni-Browser-Auth'] != auth_token
-        request.headers.delete 'X-Arachni-Browser-Auth'
+        # return if request.headers['X-Arachni-Browser-Auth'] != auth_token
+        # request.headers.delete 'X-Arachni-Browser-Auth'
 
         print_debug_level_2 "Request: #{request.url}"
 
@@ -1539,7 +1452,7 @@ EOJS
         #
         # Still, it's a nice feature to have when requesting assets or anything
         # else.
-        if request.url == @last_url
+        if !@last_url || request.url == @last_url
             request.headers.delete 'If-None-Match'
             request.headers.delete 'If-Modified-Since'
         end
@@ -1594,7 +1507,11 @@ EOJS
 
         # Prevent PhantomJS from caching the root page, we need to have an
         # associated response.
-        if @last_url == response.url
+        #
+        # Also don't cache when we don't have a @last_url because this could
+        # be driven directly from Selenium/Watir via a plugin and caching it
+        # can ruin the scan.
+        if !@last_url || @last_url == response.url
             response.headers.delete 'Cache-control'
             response.headers.delete 'Etag'
             response.headers.delete 'Date'
@@ -1872,7 +1789,7 @@ EOJS
     end
 
     def normalize_watir_url( url )
-        normalize_url( ::URI.encode( url, ';' ) ).gsub( '%3B', '%253B' )
+        normalize_url( url.gsub( ';', '%3B' )  ).gsub( '%3B', '%253B' )
     end
 
 end

data/lib/arachni/browser_cluster/job/result.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/job.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/jobs/browser_provider.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/jobs/dom_exploration/event_trigger/result.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/jobs/dom_exploration/event_trigger.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/jobs/dom_exploration/result.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/jobs/dom_exploration.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/jobs/taint_trace/event_trigger/result.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/jobs/taint_trace/event_trigger.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/jobs/taint_trace/result.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework

data/lib/arachni/browser_cluster/jobs/taint_trace.rb

@@ -1,5 +1,5 @@
 =begin
-    Copyright 2010-2017 Sarosys LLC <http://www.sarosys.com>
+    Copyright 2010-2022 Ecsypno <http://www.ecsypno.com>
 
     This file is part of the Arachni Framework project and is subject to
     redistribution and commercial restrictions. Please see the Arachni Framework