ferrum 0.18.0 → 0.19.1

data/lib/ferrum/client/web_socket.rb

@@ -16,6 +16,7 @@ module Ferrum
       WEBSOCKET_BUG_SLEEP = 0.05
       DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
       SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
+      ESCAPE_OR_LONE_SURROGATE = /\\u[dD][89abAB]\h{2}\\u[dD][c-fC-F]\h{2}|\\u([dD][89a-fA-F]\h{2})|\\./
 
       attr_reader :url, :messages
 
@@ -106,13 +107,26 @@ module Ferrum
       # @param [Hash] data
       #   The message to send, must include an `:id` key.
       #
+      # @param [Numeric, nil] timeout
+      #   How long the browser may go without accepting any of the frame's
+      #   bytes. `nil` waits for as long as it takes.
+      #
       # @return [void]
       #
-      def send_message(data)
+      # @raise [Ferrum::DeadBrowserError]
+      #   If the browser accepted no data for `timeout` seconds. The connection
+      #   is closed, since the frame was cut off midway.
+      #
+      def send_message(data, timeout = nil)
         @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
 
         json = data.to_json
-        @driver_mutex.synchronize { @driver.text(json) }
+        @driver_mutex.synchronize do
+          @write_timeout = timeout
+          @driver.text(json)
+        ensure
+          @write_timeout = nil
+        end
         @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
       end
 
@@ -121,13 +135,28 @@ module Ferrum
       # `websocket-driver` to emit frames. Closes {#messages} instead of
       # raising if the connection has already been torn down.
       #
+      # A browser that stops reading its socket (hung, or stopped) would block
+      # a plain write forever, holding the lock every other command needs to be
+      # sent. So the write gives up once the browser accepts no data for the
+      # timeout of the command being sent.
+      #
       # @param [String] data
       #   The raw bytes to write.
       #
       # @return [void]
       #
+      # @raise [Ferrum::DeadBrowserError]
+      #   If the browser accepted no data for the command's timeout.
+      #
       def write(data)
-        @sock.write(data)
+        until data.empty?
+          written = @sock.write_nonblock(data, exception: false)
+          if written.is_a?(Integer)
+            data = data.byteslice(written..)
+          elsif !wait_for_socket(written)
+            abandon_stalled_write
+          end
+        end
       rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
         @messages.close
       end
@@ -141,8 +170,38 @@ module Ferrum
         @driver_mutex.synchronize { @driver.close }
       end
 
+      #
+      # Closes the underlying socket whether or not the close handshake ever
+      # completed.
+      #
+      # {#close} only sends a close frame; the socket itself is closed by
+      # {#on_close}, which runs when the browser answers with a close frame of
+      # its own. A browser that is already dead never answers, so without this
+      # the socket stays open until GC finalizes it -- long enough to exhaust
+      # the process's file descriptors when browsers are restarted in a loop.
+      #
+      # @return [void]
+      #
+      def force_close
+        @sock.close unless @sock.closed?
+      rescue IOError, SystemCallError
+        # nop
+      end
+
       private
 
+      def wait_for_socket(state)
+        readers, writers = state == :wait_readable ? [[@sock], nil] : [nil, [@sock]]
+        IO.select(readers, writers, nil, @write_timeout)
+      end
+
+      def abandon_stalled_write
+        timeout = @write_timeout
+        @messages.close
+        @sock.close
+        raise DeadBrowserError, "Browser accepted no data for #{timeout}s and is likely stuck, connection closed"
+      end
+
       def start
         @thread = Utils::Thread.spawn do
           loop do
@@ -152,6 +211,10 @@ module Ferrum
             @driver_mutex.synchronize { @driver.parse(data) }
           end
         rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
+          # The browser went away, nothing to report.
+        rescue StandardError => e
+          warn("Ferrum: websocket reader stopped, #{e.class}: #{e.message}\n  #{e.backtrace&.first}")
+        ensure
           @messages.close
         end
       end
@@ -161,14 +224,12 @@ module Ferrum
       rescue JSON::NestingError
         # nop
       rescue JSON::ParserError
-        safely_parse_escaped_json(data)
+        safely_parse_lone_surrogates(data)
       end
 
-      def safely_parse_escaped_json(data)
-        unescaped_unicode =
-          data.gsub(/\\u([\da-fA-F]{4})/) { |_| [::Regexp.last_match(1)].pack("H*").unpack("n*").pack("U*") }
-        escaped_data = unescaped_unicode.encode("UTF-8", "UTF-8", undef: :replace, invalid: :replace, replace: "?")
-        JSON.parse(escaped_data, max_nesting: false)
+      def safely_parse_lone_surrogates(data)
+        replaced = data.gsub(ESCAPE_OR_LONE_SURROGATE) { |escape| ::Regexp.last_match(1) ? "\\ufffd" : escape }
+        JSON.parse(replaced, max_nesting: false)
       rescue JSON::ParserError
         # nop
       end

data/lib/ferrum/client.rb

@@ -208,25 +208,33 @@ module Ferrum
     #   Whether to return immediately instead of waiting for a response.
     #
     # @param [Numeric, nil] timeout
-    #   How long to wait for the response. Defaults to `protocol_timeout`.
+    #   How long to wait for the browser to take the message, and then for its
+    #   response. Defaults to `protocol_timeout`.
     #
     # @return [Boolean, Hash]
     #   `true` when sent asynchronously, otherwise the parsed `"result"`
     #   from the response.
     #
     def send_message(message, async:, timeout: nil)
+      timeout ||= protocol_timeout
+
       if async
-        @ws.send_message(message)
+        @ws.send_message(message, timeout)
         true
       else
+        raise DeadBrowserError if @ws.messages.closed?
+
         pending = Concurrent::IVar.new
         @pendings[message[:id]] = pending
-        @ws.send_message(message)
-        data = pending.value!(timeout || protocol_timeout)
-        @pendings.delete(message[:id])
+        begin
+          @ws.send_message(message, timeout)
+          data = pending.value!(timeout)
+        ensure
+          @pendings.delete(message[:id])
+        end
 
         raise DeadBrowserError if data.nil? && @ws.messages.closed?
-        raise TimeoutError unless data
+        raise TimeoutError.new(message[:method], session_id: message[:sessionId]) unless data
 
         error, response = data.values_at("error", "result")
         raise_browser_error(error) if error
@@ -297,6 +305,7 @@ module Ferrum
       @pendings.clear
       @thread.kill unless @thread.join(1)
       @subscriber.close
+      @ws.force_close
     end
 
     #
@@ -340,9 +349,22 @@ module Ferrum
             @pendings[message["id"]]&.set(message)
           end
         end
+      rescue StandardError => e
+        # Nothing delivers responses once this thread is gone, so the connection
+        # is done for. Close it rather than leave every command hanging.
+        warn("Ferrum: dispatch stopped, #{e.class}: #{e.message}\n  #{e.backtrace&.first}")
+        @ws.messages.close
+      ensure
+        release_pendings
       end
     end
 
+    # Nothing is going to answer the commands still in flight once the socket
+    # is gone, release them instead of making each wait out its timeout.
+    def release_pendings
+      @pendings.each_value { |pending| pending.try_set(nil) }
+    end
+
     # Locked so two concurrent commands never share an id and read each
     # other's responses from @pendings.
     def next_command_id
@@ -354,7 +376,8 @@ module Ferrum
       # Node has disappeared while we were trying to get it
       when "No node with given id found",
            "Could not find node with given id",
-           "Inspected target navigated or closed"
+           "Inspected target navigated or closed",
+           "Argument should belong to the same JavaScript world as target object"
         raise NodeNotFoundError, error
       # Context is lost, page is reloading
       when "Cannot find context with specified id"

data/lib/ferrum/context.rb

@@ -15,14 +15,25 @@ module Ferrum
 
     attr_reader :id, :targets
 
-    def initialize(client, contexts, id)
+    def initialize(client, contexts, id, implicit: false)
       @id = id
+      @implicit = implicit
       @client = client
       @contexts = contexts
       @targets = Concurrent::Map.new
       @pendings = Concurrent::Map.new
     end
 
+    # Whether this is the browser's implicit context, the one it puts its
+    # startup window in. We didn't create it, so we can't dispose it, and
+    # Chrome doesn't let us address it by id either: targets are created in
+    # it by omitting `browserContextId` altogether.
+    #
+    # @return [Boolean]
+    def implicit?
+      @implicit
+    end
+
     # The context's first known target, creating one via
     # `Target.createTarget` if none has attached yet.
     #
@@ -92,7 +103,8 @@ module Ferrum
     #
     # @raise [NoSuchTargetError]
     def create_target
-      target_id = @client.command("Target.createTarget", browserContextId: @id, url: "about:blank")["targetId"]
+      options = { browserContextId: @id } unless implicit?
+      target_id = @client.command("Target.createTarget", url: "about:blank", **Hash(options))["targetId"]
 
       new_pending = Concurrent::IVar.new
       pending = @pendings.put_if_absent(target_id, new_pending) || new_pending
@@ -114,7 +126,7 @@ module Ferrum
     #
     # @return [Target]
     def add_target(params:, session_id: nil)
-      new_target = Target.new(@client, session_id, params)
+      new_target = Target.new(@client, session_id, params, implicit: implicit?)
       # `put_if_absent` returns nil if added a new value or existing if there was one already
       target = @targets.put_if_absent(new_target.id, new_target) || new_target
       # on first iteration session_id may be nil, then if session is present here we must set it to the target
@@ -184,9 +196,10 @@ module Ferrum
       end
     end
 
-    # Disposes this browser context and all of its targets.
+    # Disposes this browser context and all of its targets. The browser's
+    # implicit context cannot be disposed, see {#implicit?}.
     #
-    # @return [Boolean]
+    # @return [Boolean, nil]
     def dispose
       @contexts.dispose(@id)
     end

data/lib/ferrum/contexts.rb

@@ -22,6 +22,7 @@ module Ferrum
       @client = client
       @contexts = Concurrent::Map.new
       @manually_attached = Concurrent::Map.new
+      @default_context = find_implicit_context
       subscribe
       auto_attach
       discover
@@ -33,7 +34,9 @@ module Ferrum
       @manually_attached[target_id] = true
     end
 
-    # The browser's first context, created lazily.
+    # The context we work in unless told otherwise: the browser's implicit
+    # context when it came up with a startup window (see {#find_implicit_context}),
+    # otherwise one of our own, created lazily.
     #
     # @return [Context]
     def default_context
@@ -84,18 +87,29 @@ module Ferrum
       context
     end
 
-    # Disposes a browser context and all of its targets.
+    # Disposes a browser context and all of its targets. The browser's implicit
+    # context is not ours to dispose, see {Context#implicit?}.
+    #
+    # The context is forgotten even if Chrome fails to dispose it in time: its
+    # targets are disconnected by then, and asking again only gets
+    # `Disposal of browser context ... is already pending` back.
     #
     # @param [String] context_id
     #
-    # @return [Boolean]
+    # @return [Boolean, nil]
+    #
+    # @raise [TimeoutError, BrowserError]
     def dispose(context_id)
       context = @contexts[context_id]
       return unless context
+      return if context.implicit?
 
-      context.close_targets_connection
-      @client.command("Target.disposeBrowserContext", browserContextId: context.id)
-      @contexts.delete(context_id)
+      begin
+        context.close_targets_connection
+        @client.command("Target.disposeBrowserContext", browserContextId: context.id)
+      ensure
+        @contexts.delete(context_id)
+      end
       true
     end
 
@@ -107,13 +121,23 @@ module Ferrum
       @contexts.each_value(&:close_targets_connection)
     end
 
-    # Disposes every context still known to the browser.
+    # Disposes every context still known to the browser. A context that fails
+    # to dispose doesn't stop the others from being disposed; the first error
+    # is raised once all of them have been tried.
     #
     # @return [void]
+    #
+    # @raise [TimeoutError, BrowserError]
     def reset
+      error = nil
       context_ids = @client.command("Target.getBrowserContexts")["browserContextIds"]
       @default_context = nil if context_ids.include?(@default_context&.id)
-      @contexts.each_key { |id| dispose(id) if context_ids.include?(id) }
+      (@contexts.keys & context_ids).each do |id|
+        dispose(id)
+      rescue BrowserError, TimeoutError => e
+        error ||= e
+      end
+      raise error if error
     end
 
     # Number of known contexts.
@@ -135,7 +159,7 @@ module Ferrum
     def subscribe_attached_target(client)
       client.on("Target.attachedToTarget") do |params|
         info, session_id = params.values_at("targetInfo", "sessionId")
-        next unless ALLOWED_TARGET_TYPES.include?(info["type"])
+        next detach_untracked(session_id) unless ALLOWED_TARGET_TYPES.include?(info["type"])
 
         context_id = info["browserContextId"]
         add_context(context_id)
@@ -154,8 +178,7 @@ module Ferrum
         context_id = info["browserContextId"]
         add_context(context_id)
 
-        if info["type"] == "iframe" &&
-           (target = @contexts[context_id]&.find_target { |t| t.connected? && t.page.frame_by(id: info["targetId"]) })
+        if info["type"] == "iframe" && (target = find_frame_owner(context_id, info["targetId"]))
           @contexts[context_id]&.add_target(session_id: target.session_id, params: info)
         else
           @contexts[context_id]&.add_target(params: info)
@@ -181,6 +204,12 @@ module Ferrum
       end
     end
 
+    def find_frame_owner(context_id, frame_id)
+      @contexts[context_id]&.find_target do |target|
+        (target.page? || target.iframe?) && target.connected? && target.page.frame_by(id: frame_id)
+      end
+    end
+
     def rearm_auto_attach(session_id, type)
       return unless RECURSIVE_AUTO_ATTACH_TYPES.include?(type)
 
@@ -201,6 +230,16 @@ module Ferrum
       end
     end
 
+    # Auto-attach pauses every target it attaches to, including the types
+    # we don't track: Chrome's own `browser_ui` targets, for instance. We
+    # never resume those, so left alone they stay attached and paused for
+    # the lifetime of the browser. Resume them and drop the session.
+    def detach_untracked(session_id)
+      detach(session_id)
+    rescue BrowserError, TimeoutError
+      nil
+    end
+
     # Attaching keeps a service worker alive forever, so unless the caller
     # explicitly asked to connect to it (via Context#attach_target), we
     # just resume it and let go.
@@ -208,7 +247,7 @@ module Ferrum
       return if @manually_attached.delete(target.id)
 
       detach(session_id)
-    rescue BrowserError
+    rescue BrowserError, TimeoutError
       nil
     end
 
@@ -217,7 +256,7 @@ module Ferrum
     # only then resumes the debugger itself.
     def connect_worker(target)
       target.worker
-    rescue BrowserError
+    rescue BrowserError, TimeoutError
       nil
     end
 
@@ -241,9 +280,29 @@ module Ferrum
     def add_context(context_id)
       return if @contexts[context_id]
 
-      context = Context.new(@client, self, context_id)
-      @contexts[context_id] = context
-      @default_context ||= context # rubocop:disable Naming/MemoizedInstanceVariableName
+      @contexts[context_id] = Context.new(@client, self, context_id)
+    end
+
+    # The browser's implicit context, the context Chrome uses for its startup window.
+    # We register it so events for that window are routed to it.
+    # `Target.getBrowserContexts` reports every context created with
+    # `Target.createBrowserContext`, ours and those of anyone else driving the same
+    # browser, so the context holding targets that it doesn't report is the implicit
+    # one. Chrome also reports its id as `defaultBrowserContextId`, but only since
+    # 145 and only as an experimental field, so we don't rely on it.
+    #
+    # Returns nil if the browser has no startup window, which happens when it is
+    # launched with `--no-startup-window` (used for `incognito: true`). In that case,
+    # we create our own context instead.
+    #
+    # @return [Context, nil]
+    def find_implicit_context
+      created_ids = @client.command("Target.getBrowserContexts")["browserContextIds"]
+      target_infos = @client.command("Target.getTargets")["targetInfos"]
+      implicit_id = (target_infos.filter_map { |t| t["browserContextId"] } - created_ids).first
+      return unless implicit_id
+
+      @contexts[implicit_id] = Context.new(@client, self, implicit_id, implicit: true)
     end
   end
 end

data/lib/ferrum/downloads.rb

@@ -62,8 +62,9 @@ module Ferrum
       raise ArgumentError unless VALID_BEHAVIOR.include?(behavior.to_sym)
       raise Error, "supply absolute path for `:save_path` option" unless Pathname.new(save_path.to_s).absolute?
 
+      options = { browserContextId: @page.context_id } if @page.context_id
       @page.command("Browser.setDownloadBehavior",
-                    browserContextId: @page.context_id,
+                    **Hash(options),
                     downloadPath: save_path,
                     behavior: behavior,
                     eventsEnabled: true)

data/lib/ferrum/errors.rb

@@ -40,18 +40,50 @@ module Ferrum
 
   # Raised when waiting for a response from the browser times out.
   class TimeoutError < Error
+    # The CDP method that didn't answer in time, `nil` when the wait wasn't for a command.
     #
-    # Explains that waiting for a response timed out.
+    # @return [String, nil]
+    attr_reader :command
+
+    # The session the command was sent to, `nil` for the browser-wide session.
+    #
+    # @return [String, nil]
+    attr_reader :session_id
+
+    #
+    # @param [String, nil] command
+    #   The CDP method that timed out.
+    #
+    # @param [String, nil] session_id
+    #   The session it was sent to.
+    #
+    def initialize(command = nil, session_id: nil)
+      @command = command
+      @session_id = session_id
+      super()
+    end
+
+    #
+    # Explains that waiting for a response timed out, naming the command when
+    # there was one.
     #
     # @return [String]
     #
     def message
-      "Timed out waiting for response. It's possible that this happened " \
+      "Timed out waiting for #{awaited}. It's possible that this happened " \
         "because something took a very long time (for example a page load " \
         "was slow). If so, setting the :timeout option to a higher value might " \
         "help. If this happened on an internal protocol call instead, try " \
         "raising :protocol_timeout."
     end
+
+    private
+
+    def awaited
+      return "response" unless @command
+
+      @session_id ? "a response to #{@command} (session #{@session_id})" : "a response to #{@command}"
+    end
   end
 
   # Raised when an evaluated script takes too long to return a value.
@@ -77,6 +109,18 @@ module Ferrum
     end
   end
 
+  # Raised when connecting to an already running browser, but its
+  # `/json/version` endpoint doesn't answer with the websocket url.
+  class NoWebSocketUrlError < Error
+    attr_reader :url
+
+    def initialize(url)
+      @url = url
+      super("Couldn't get a websocket url out of #{url}, `/json/version` there didn't answer with JSON. " \
+            "Check the browser is reachable at that url and that any credentials it needs are part of it")
+    end
+  end
+
   # Raised when trying to interact with a browser process that has died or
   # a window that has already been closed.
   class DeadBrowserError < Error

data/lib/ferrum/frame/dom.rb

@@ -116,7 +116,8 @@ module Ferrum
       # Returns the element in which the window is embedded.
       #
       # @return [Node, nil]
-      #   The element in which the window is embedded.
+      #   The element in which the window is embedded, `nil` for the main frame
+      #   and for a frame from a different origin than its parent.
       #
       # @example
       #   browser.go_to("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
@@ -243,6 +244,54 @@ module Ferrum
         evaluate_func(expr, selector, within)
       end
 
+      #
+      # Finds a node by using a CSS path or XPath selector, retrying until
+      # it appears or `timeout` elapses. Pass exactly one of `css`/`xpath`;
+      # each attempt delegates to {#at_css}/{#at_xpath} respectively, so it
+      # matches their behavior exactly.
+      #
+      # @param [String, nil] css
+      #   The CSS path selector. Mutually exclusive with `xpath`.
+      #
+      # @param [String, nil] xpath
+      #   The XPath selector. Mutually exclusive with `css`.
+      #
+      # @param [Node, nil] within
+      #   The parent node to search within.
+      #
+      # @param [Float, Integer] timeout
+      #   How long to keep retrying, in seconds.
+      #
+      # @param [Float, Integer] interval
+      #   How long to sleep between attempts, in seconds.
+      #
+      # @return [Node, nil]
+      #   The matching node, or `nil` if `timeout` elapses without a match.
+      #
+      # @raise [ArgumentError]
+      #   If neither or both of `css`/`xpath` are given.
+      #
+      # @example
+      #   browser.go_to("https://github.com/")
+      #   browser.wait_for_selector(css: "a[aria-label='Issues you created']") # => Node
+      #
+      def wait_for_selector(css: nil, xpath: nil, within: nil, timeout: @page.timeout, interval: 0.05)
+        raise ArgumentError, "pass either `css:` or `xpath:`" if css && xpath
+
+        selector, method_name = css ? [css, :at_css] : [xpath, :at_xpath]
+        raise ArgumentError, "pass either `css:` or `xpath:`" unless selector
+
+        start = Utils::ElapsedTime.monotonic_time
+
+        until (node = send(method_name, selector, within: within))
+          return nil if Utils::ElapsedTime.timeout?(start, timeout)
+
+          sleep(interval)
+        end
+
+        node
+      end
+
       #
       # Adds a `<script>` tag to the document.
       #

data/lib/ferrum/frame/runtime.rb

@@ -146,7 +146,8 @@ module Ferrum
       private
 
       def call(expression:, arguments: [], on: nil, wait: 0, handle: true, **options)
-        errors = [NodeNotFoundError, NoExecutionContextError]
+        # do not rescue -> retry if we operate on an existing node
+        errors = on ? [] : [NodeNotFoundError, NoExecutionContextError]
 
         Utils::Attempt.with_retry(errors: errors, max: INTERMITTENT_ATTEMPTS, wait: INTERMITTENT_SLEEP) do
           params = options.dup
@@ -185,7 +186,7 @@ module Ferrum
         end
       end
 
-      def handle_response(response)
+      def handle_response(response, check_cyclic: true)
         case response["type"]
         when "boolean", "number", "string"
           response["value"]
@@ -201,10 +202,10 @@ module Ferrum
             description = @page.command("DOM.describeNode", objectId: object_id)["node"]
             Node.new(self, @page.target_id, description, object_id: object_id)
           when "array"
-            reduce_props(object_id, []) do |memo, key, value|
+            reduce_props(object_id, [], check_cyclic: check_cyclic) do |memo, key, value|
               next(memo) unless Integer(key, exception: false)
 
-              value = value["objectId"] ? handle_response(value) : value["value"]
+              value = value["objectId"] ? handle_response(value, check_cyclic: false) : value["value"]
               memo.insert(key.to_i, value)
             end.compact
           when "date"
@@ -212,8 +213,8 @@ module Ferrum
           when "null"
             nil
           else
-            reduce_props(object_id, {}) do |memo, key, value|
-              value = value["objectId"] ? handle_response(value) : value["value"]
+            reduce_props(object_id, {}, check_cyclic: check_cyclic) do |memo, key, value|
+              value = value["objectId"] ? handle_response(value, check_cyclic: false) : value["value"]
               memo.merge(key => value)
             end
           end
@@ -233,8 +234,8 @@ module Ferrum
         end
       end
 
-      def reduce_props(object_id, to)
-        if cyclic?(object_id).dig("result", "value")
+      def reduce_props(object_id, to, check_cyclic: true)
+        if check_cyclic && cyclic?(object_id).dig("result", "value")
           to.is_a?(Array) ? [cyclic_object] : cyclic_object
         else
           props = @page.command("Runtime.getProperties", ownProperties: true, objectId: object_id)