puppeteer-ruby 0.45.6 → 0.50.0.alpha5

data/lib/puppeteer/browser_runner.rb

@@ -1,7 +1,5 @@
 require 'fileutils'
 require 'open3'
-require 'timeout'
-
 # https://github.com/puppeteer/puppeteer/blob/master/lib/Launcher.js
 class Puppeteer::BrowserRunner
   include Puppeteer::DebugPrint
@@ -9,8 +7,7 @@ class Puppeteer::BrowserRunner
   # @param {string} executablePath
   # @param {!Array<string>} processArguments
   # @param {string=} tempDirectory
-  def initialize(for_firefox, executable_path, process_arguments, user_data_dir, using_temp_user_data_dir)
-    @for_firefox = for_firefox
+  def initialize(executable_path, process_arguments, user_data_dir, using_temp_user_data_dir)
     @executable_path = executable_path
     @process_arguments = process_arguments
     @user_data_dir = user_data_dir
@@ -54,7 +51,7 @@ class Puppeteer::BrowserRunner
     attr_reader :stdout, :stderr, :spawnargs
   end
 
-  class LaunchError < StandardError
+  class LaunchError < Puppeteer::Error
     def initialize(reason)
       super("Failed to launch browser! #{reason}")
     end
@@ -159,13 +156,18 @@ class Puppeteer::BrowserRunner
   end
 
 
-  # @param {!({usePipe?: boolean, timeout: number, slowMo: number, preferredRevision: string})} options
+  # @param {!({usePipe?: boolean, timeout: number, slowMo: number, preferredRevision: string, protocolTimeout: number?})} options
   # @return {!Promise<!Connection>}
-  def setup_connection(use_pipe:, timeout:, slow_mo:, preferred_revision:)
+  def setup_connection(use_pipe:, timeout:, slow_mo:, preferred_revision:, protocol_timeout: nil)
     if !use_pipe
       browser_ws_endpoint = wait_for_ws_endpoint(@proc, timeout, preferred_revision)
       transport = Puppeteer::WebSocketTransport.create(browser_ws_endpoint)
-      @connection = Puppeteer::Connection.new(browser_ws_endpoint, transport, slow_mo)
+      @connection = Puppeteer::Connection.new(
+        browser_ws_endpoint,
+        transport,
+        slow_mo,
+        protocol_timeout: protocol_timeout,
+      )
     else
       raise NotImplementedError.new('PipeTransport is not yet implemented')
     end
@@ -175,7 +177,7 @@ class Puppeteer::BrowserRunner
 
   private def wait_for_ws_endpoint(browser_process, timeout, preferred_revision)
     lines = []
-    Timeout.timeout(timeout / 1000.0) do
+    wait_for_endpoint = lambda do
       loop do
         line = browser_process.stderr.readline
         /^WebDriver BiDi listening on (ws:\/\/.*)$/.match(line) do |m|
@@ -188,9 +190,15 @@ class Puppeteer::BrowserRunner
         lines << line
       end
     end
+
+    if timeout && timeout > 0
+      Puppeteer::AsyncUtils.async_timeout(timeout, wait_for_endpoint).wait
+    else
+      wait_for_endpoint.call
+    end
   rescue EOFError
     raise LaunchError.new("\n#{lines.join("\n")}\nTROUBLESHOOTING: https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md")
-  rescue Timeout::Error
+  rescue Async::TimeoutError
     raise Puppeteer::TimeoutError.new("Timed out after #{timeout} ms while trying to connect to the browser! Only Chrome at revision r#{preferred_revision} is guaranteed to work.")
   end
 end

data/lib/puppeteer/cdp_session.rb

@@ -1,56 +1,91 @@
+# rbs_inline: enabled
+
 class Puppeteer::CDPSession
   include Puppeteer::DebugPrint
   include Puppeteer::EventCallbackable
   using Puppeteer::DefineAsyncMethod
 
-  class Error < StandardError; end
+  class Error < Puppeteer::Error; end
 
-  # @param {!Connection} connection
-  # @param {string} targetType
-  # @param {string} sessionId
-  def initialize(connection, target_type, session_id)
-    @callbacks = Concurrent::Hash.new
+  # @rbs connection: Puppeteer::Connection -- CDP connection
+  # @rbs target_type: String -- Target type
+  # @rbs session_id: String -- CDP session id
+  # @rbs parent_session: Puppeteer::CDPSession? -- Parent CDP session
+  # @rbs return: void -- No return value
+  def initialize(connection, target_type, session_id, parent_session: nil)
+    @callbacks = {}
+    @callbacks_mutex = Mutex.new
     @connection = connection
     @target_type = target_type
     @session_id = session_id
+    @parent_session = parent_session
+    @ready_promise = Async::Promise.new
+    @target = nil
   end
 
-  # @internal
+  # @rbs return: String -- CDP session id
   def id
     @session_id
   end
 
-  attr_reader :connection
+  attr_reader :connection #: Puppeteer::Connection?
+  attr_reader :parent_session #: Puppeteer::CDPSession?
+  attr_accessor :target #: Puppeteer::Target?
+
+  # @rbs return: void -- Resolve session readiness
+  def mark_ready
+    @ready_promise.resolve(true) unless @ready_promise.resolved?
+  end
+
+  # @rbs return: bool -- True when session is ready
+  def wait_for_ready
+    @ready_promise.wait
+  end
 
-  # @param method [String]
-  # @param params [Hash]
-  # @returns [Hash]
+  # @rbs method: String -- CDP method name
+  # @rbs params: Hash[String, untyped] -- CDP parameters
+  # @rbs return: Hash[String, untyped] -- CDP response
   def send_message(method, params = {})
-    await async_send_message(method, params)
+    protocol_timeout = @connection&.protocol_timeout
+    if protocol_timeout && protocol_timeout > 0
+      begin
+        Puppeteer::AsyncUtils.async_timeout(protocol_timeout, async_send_message(method, params)).wait
+      rescue Async::TimeoutError
+        raise Puppeteer::Connection::ProtocolError.new(
+          method: method,
+          error_message: "Timeout #{protocol_timeout}ms exceeded",
+        )
+      end
+    else
+      async_send_message(method, params).wait
+    end
   end
 
-  # @param method [String]
-  # @param params [Hash]
-  # @returns [Future<Hash>]
+  # @rbs method: String -- CDP method name
+  # @rbs params: Hash[String, untyped] -- CDP parameters
+  # @rbs return: Async::Promise[Hash[String, untyped]] -- Async CDP response
   def async_send_message(method, params = {})
     if !@connection
       raise Error.new("Protocol error (#{method}): Session closed. Most likely the #{@target_type} has been closed.")
     end
 
-    promise = resolvable_future
+    promise = Async::Promise.new
 
     @connection.generate_id do |id|
-      @callbacks[id] = Puppeteer::Connection::MessageCallback.new(method: method, promise: promise)
+      @callbacks_mutex.synchronize do
+        @callbacks[id] = Puppeteer::Connection::MessageCallback.new(method: method, promise: promise)
+      end
       @connection.raw_send(id: id, message: { sessionId: @session_id, method: method, params: params })
     end
 
     promise
   end
 
-  # @param {{id?: number, method: string, params: Object, error: {message: string, data: any}, result?: *}} object
+  # @rbs message: Hash[String, untyped] -- Raw CDP message
+  # @rbs return: void -- No return value
   def handle_message(message)
     if message['id']
-      if callback = @callbacks.delete(message['id'])
+      if callback = @callbacks_mutex.synchronize { @callbacks.delete(message['id']) }
         callback_with_message(callback, message)
       else
         raise Error.new("unknown id: #{message['id']}")
@@ -72,6 +107,7 @@ class Puppeteer::CDPSession
     end
   end
 
+  # @rbs return: void -- Detach the session
   def detach
     if !@connection
       raise Error.new("Session already detarched. Most likely the #{@target_type} has been closed.")
@@ -79,24 +115,32 @@ class Puppeteer::CDPSession
     @connection.send_message('Target.detachFromTarget',  sessionId: @session_id)
   end
 
+  # @rbs return: void -- Close the session and reject pending callbacks
   def handle_closed
-    @callbacks.each_value do |callback|
+    callbacks = @callbacks_mutex.synchronize do
+      @callbacks.values.tap { @callbacks.clear }
+    end
+    callbacks.each do |callback|
       callback.reject(
         Puppeteer::Connection::ProtocolError.new(
           method: callback.method,
           error_message: 'Target Closed.'))
     end
-    @callbacks.clear
+    @ready_promise.reject(Error.new("Session closed")) unless @ready_promise.resolved?
     @connection = nil
     emit_event(CDPSessionEmittedEvents::Disconnected)
   end
 
-  # @param event_name [String]
+  # @rbs event_name: String -- CDP event name
+  # @rbs &block: ^(untyped) -> void -- Event handler
+  # @rbs return: String -- Listener id
   def on(event_name, &block)
     add_event_listener(event_name, &block)
   end
 
-  # @param event_name [String]
+  # @rbs event_name: String -- CDP event name
+  # @rbs &block: ^(untyped) -> void -- Event handler
+  # @rbs return: String -- Listener id
   def once(event_name, &block)
     observe_first(event_name, &block)
   end

data/lib/puppeteer/chrome_target_manager.rb

@@ -8,12 +8,13 @@ class Puppeteer::ChromeTargetManager
     @attached_targets_by_session_id = {}
     @ignored_targets = Set.new
     @target_ids_for_init = Set.new
+    @service_worker_detach_promises = {}
 
     @connection = connection
     @target_filter_callback = target_filter_callback
     @target_factory = target_factory
     @target_interceptors = {}
-    @initialize_promise = resolvable_future
+    @initialize_promise = Async::Promise.new
 
     @connection_event_listeners = []
     @connection_event_listeners << @connection.add_event_listener(
@@ -34,15 +35,17 @@ class Puppeteer::ChromeTargetManager
     )
 
     setup_attachment_listeners(@connection)
-    @connection.async_send_message('Target.setDiscoverTargets', {
-      discover: true,
-      filter: [
-        { type: 'tab', exclude: true },
-        {},
-      ],
-    }).then do
+
+    Async do
+      @connection.async_send_message('Target.setDiscoverTargets', {
+        discover: true,
+        filter: [
+          { type: 'tab', exclude: true },
+          {},
+        ],
+      }).wait
       store_existing_targets_for_init
-    end.rescue do |err|
+    rescue => err
       debug_puts(err)
     end
   end
@@ -62,7 +65,7 @@ class Puppeteer::ChromeTargetManager
       autoAttach: true,
     })
     finish_initialization_if_ready
-    @initialize_promise.value!
+    @initialize_promise.wait
   end
 
   def dispose
@@ -74,6 +77,14 @@ class Puppeteer::ChromeTargetManager
     @attached_targets_by_target_id
   end
 
+  def wait_for_service_worker_detach(target_id)
+    promise = @service_worker_detach_promises[target_id]
+    return unless promise
+    promise.wait
+  ensure
+    @service_worker_detach_promises.delete(target_id)
+  end
+
   def add_target_interceptor(client, interceptor)
     interceptors = @target_interceptors[client] || []
     interceptors << interceptor
@@ -157,7 +168,7 @@ class Puppeteer::ChromeTargetManager
     emit_event(TargetManagerEmittedEvents::TargetChanged, original_target, target_info)
   end
 
-  class SessionNotCreatedError < StandardError ; end
+  class SessionNotCreatedError < Puppeteer::Error ; end
 
   private def handle_attached_to_target(parent_session, event)
     target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
@@ -167,17 +178,27 @@ class Puppeteer::ChromeTargetManager
       raise SessionNotCreatedError.new("Session #{session_id} was not created.")
     end
 
-    silent_detach = -> {
-      session.async_send_message('Runtime.runIfWaitingForDebugger').rescue do |err|
-        Logger.new($stderr).warn(err)
-      end
+    silent_detach = ->(detached_promise = nil) {
+      Async do
+        begin
+          Puppeteer::AsyncUtils.await(session.async_send_message('Runtime.runIfWaitingForDebugger'))
+        rescue => err
+          Logger.new($stderr).warn(err)
+        end
 
-      # We don't use `session.detach()` because that dispatches all commands on
-      # the connection instead of the parent session.
-      parent_session.async_send_message('Target.detachFromTarget', {
-        sessionId: session.id,
-      }).rescue do |err|
-        Logger.new($stderr).warn(err)
+        # We don't use `session.detach()` because that dispatches all commands on
+        # the connection instead of the parent session.
+        begin
+          Puppeteer::AsyncUtils.await(parent_session.async_send_message('Target.detachFromTarget', {
+            sessionId: session.id,
+          }))
+        rescue => err
+          Logger.new($stderr).warn(err)
+        ensure
+          if detached_promise && !detached_promise.resolved?
+            detached_promise.resolve(true)
+          end
+        end
       end
     }
 
@@ -192,12 +213,13 @@ class Puppeteer::ChromeTargetManager
     # CDP.
     if target_info.type == 'service_worker' && @connection.auto_attached?(target_info.target_id)
       finish_initialization_if_ready(target_info.target_id)
-      silent_detach.call
-      if parent_session.is_a?(Puppeteer::CDPSession)
-        target = @target_factory.call(target_info, parent_session)
-        @attached_targets_by_target_id[target_info.target_id] = target
-        emit_event(TargetManagerEmittedEvents::TargetAvailable, target)
-      end
+      @service_worker_detach_promises[target_info.target_id] ||= Async::Promise.new
+      silent_detach.call(@service_worker_detach_promises[target_info.target_id])
+      return if @attached_targets_by_target_id.has_key?(target_info.target_id)
+
+      target = @target_factory.call(target_info, nil)
+      @attached_targets_by_target_id[target_info.target_id] = target
+      emit_event(TargetManagerEmittedEvents::TargetAvailable, target)
 
       return
     end
@@ -214,6 +236,7 @@ class Puppeteer::ChromeTargetManager
 
     target = @attached_targets_by_target_id[target_info.target_id] || @target_factory.call(target_info, session)
     setup_attachment_listeners(session)
+    session.target = target
 
     @attached_targets_by_target_id[target_info.target_id] ||= target
     @attached_targets_by_session_id[session.id] = target
@@ -234,29 +257,31 @@ class Puppeteer::ChromeTargetManager
 
     @target_ids_for_init.delete(target.target_id)
     unless is_existing_target
-      future { emit_event(TargetManagerEmittedEvents::TargetAvailable, target) }
+      Async do
+        Puppeteer::AsyncUtils.future_with_logging { emit_event(TargetManagerEmittedEvents::TargetAvailable, target) }.call
+      end
     end
     finish_initialization_if_ready
-
-    future do
-      # TODO: the browser might be shutting down here. What do we do with the error?
-      await_all(
-        session.async_send_message('Target.setAutoAttach', {
-          waitForDebuggerOnStart: true,
-          flatten: true,
-          autoAttach: true,
-        }),
-        session.async_send_message('Runtime.runIfWaitingForDebugger'),
-      )
+    parent_session.emit_event(CDPSessionEmittedEvents::Ready, session)
+
+    Async do
+      Puppeteer::AsyncUtils.await(session.async_send_message('Target.setAutoAttach', {
+        waitForDebuggerOnStart: true,
+        flatten: true,
+        autoAttach: true,
+      }))
+      Puppeteer::AsyncUtils.await(session.async_send_message('Runtime.runIfWaitingForDebugger'))
     rescue => err
       Logger.new($stderr).warn(err)
+    ensure
+      session.mark_ready
     end
   end
 
   private def finish_initialization_if_ready(target_id = nil)
     @target_ids_for_init.delete(target_id) if target_id
     if @target_ids_for_init.empty?
-      @initialize_promise.fulfill(nil) unless @initialize_promise.resolved?
+      @initialize_promise.resolve(nil) unless @initialize_promise.resolved?
     end
   end
 

data/lib/puppeteer/connection.rb

@@ -5,7 +5,7 @@ class Puppeteer::Connection
   include Puppeteer::EventCallbackable
   using Puppeteer::DefineAsyncMethod
 
-  class ProtocolError < StandardError
+  class ProtocolError < Puppeteer::Error
     def initialize(method:, error_message:, error_data: nil)
       msg = "Protocol error (#{method}): #{error_message}"
       if error_data
@@ -19,14 +19,14 @@ class Puppeteer::Connection
   # callback object stored in @callbacks.
   class MessageCallback
     # @param method [String]
-    # @param promise [Concurrent::Promises::ResolvableFuture]
+    # @param promise [Async::Promise]
     def initialize(method:, promise:)
       @method = method
       @promise = promise
     end
 
     def resolve(result)
-      @promise.fulfill(result)
+      @promise.resolve(result)
     end
 
     def reject(error)
@@ -36,11 +36,13 @@ class Puppeteer::Connection
     attr_reader :method
   end
 
-  def initialize(url, transport, delay = 0)
+  def initialize(url, transport, delay = 0, protocol_timeout: nil)
     @url = url
     @last_id = 0
-    @callbacks = Concurrent::Hash.new
+    @callbacks = {}
+    @callbacks_mutex = Mutex.new
     @delay = delay
+    @protocol_timeout = protocol_timeout
 
     @transport = transport
     @transport.on_message do |data|
@@ -56,24 +58,26 @@ class Puppeteer::Connection
       handle_close
     end
 
-    @sessions = Concurrent::Hash.new
+    @sessions = {}
+    @sessions_mutex = Mutex.new
     @closed = false
     @manually_attached = Set.new
   end
 
+  attr_reader :protocol_timeout
+
   # used only in Browser#connected?
   def closed?
     @closed
   end
 
   private def sleep_before_handling_message(message)
-    # Puppeteer doesn't handle any Network monitoring responses.
-    # So we don't have to sleep.
+    # Keep network events ordered without extra delay.
     return if message['method']&.start_with?('Network.')
 
     # For some reasons, sleeping a bit reduces trivial errors...
     # 4ms is an interval of internal shared timer of WebKit.
-    sleep 0.004
+    Puppeteer::AsyncUtils.sleep_seconds(0.004)
   end
 
   private def should_handle_synchronously?(message)
@@ -83,9 +87,7 @@ class Puppeteer::Connection
     when nil
       false
     when /^Network\./
-      # Puppeteer doesn't handle any Network monitoring responses.
-      # So we don't care their handling order.
-      false
+      true
     when /^Page\.frame/
       # Page.frameAttached
       # Page.frameNavigated
@@ -117,7 +119,7 @@ class Puppeteer::Connection
   # @param {string} sessionId
   # @return {?CDPSession}
   def session(session_id)
-    @sessions[session_id]
+    @sessions_mutex.synchronize { @sessions[session_id] }
   end
 
   def url
@@ -127,14 +129,24 @@ class Puppeteer::Connection
   # @param {string} method
   # @param {!Object=} params
   def send_message(method, params = {})
-    await async_send_message(method, params)
+    if @protocol_timeout && @protocol_timeout > 0
+      begin
+        Puppeteer::AsyncUtils.async_timeout(@protocol_timeout, async_send_message(method, params)).wait
+      rescue Async::TimeoutError
+        raise ProtocolError.new(method: method, error_message: "Timeout #{@protocol_timeout}ms exceeded")
+      end
+    else
+      async_send_message(method, params).wait
+    end
   end
 
   def async_send_message(method, params = {})
-    promise = resolvable_future
+    promise = Async::Promise.new
 
     generate_id do |id|
-      @callbacks[id] = MessageCallback.new(method: method, promise: promise)
+      @callbacks_mutex.synchronize do
+        @callbacks[id] = MessageCallback.new(method: method, promise: promise)
+      end
       raw_send(id: id, message: { method: method, params: params })
     end
 
@@ -163,7 +175,7 @@ class Puppeteer::Connection
     #
     # So we have to know the message id in advance before send_text.
     #
-    payload = JSON.fast_generate(message.compact.merge(id: id))
+    payload = JSON.generate(message.compact.merge(id: id))
     @transport.send_text(payload)
     request_debug_printer.handle_payload(payload)
   end
@@ -194,7 +206,6 @@ class Puppeteer::Connection
       'Network.responseReceived',
       'Network.responseReceivedExtraInfo',
       'Page.lifecycleEvent',
-      'Target.receivedMessageFromTarget', # only Firefox
     ]
 
     def handle_message(message)
@@ -236,7 +247,7 @@ class Puppeteer::Connection
 
   private def handle_message(message)
     if @delay > 0
-      sleep(@delay / 1000.0)
+      Puppeteer::AsyncUtils.sleep_seconds(@delay / 1000.0)
     end
 
     response_debug_printer.handle_message(message)
@@ -244,22 +255,28 @@ class Puppeteer::Connection
     case message['method']
     when 'Target.attachedToTarget'
       session_id = message['params']['sessionId']
-      session = Puppeteer::CDPSession.new(self, message['params']['targetInfo']['type'], session_id)
-      @sessions[session_id] = session
+      parent_session =
+        if message['sessionId']
+          @sessions_mutex.synchronize { @sessions[message['sessionId']] }
+        end
+      session = Puppeteer::CDPSession.new(
+        self,
+        message['params']['targetInfo']['type'],
+        session_id,
+        parent_session: parent_session,
+      )
+      @sessions_mutex.synchronize { @sessions[session_id] = session }
       emit_event('sessionattached', session)
-      if message['sessionId']
-        parent_session = @sessions[message['sessionId']]
-        parent_session&.emit_event('sessionattached', session)
-      end
+      parent_session&.emit_event('sessionattached', session)
     when 'Target.detachedFromTarget'
       session_id = message['params']['sessionId']
-      session = @sessions[session_id]
+      session = @sessions_mutex.synchronize { @sessions[session_id] }
       if session
         session.handle_closed
-        @sessions.delete(session_id)
+        @sessions_mutex.synchronize { @sessions.delete(session_id) }
         emit_event('sessiondetached', session)
         if message['sessionId']
-          parent_session = @sessions[message['sessionId']]
+          parent_session = @sessions_mutex.synchronize { @sessions[message['sessionId']] }
           parent_session&.emit_event('sessiondetached', session)
         end
       end
@@ -267,10 +284,10 @@ class Puppeteer::Connection
 
     if message['sessionId']
       session_id = message['sessionId']
-      @sessions[session_id]&.handle_message(message)
+      @sessions_mutex.synchronize { @sessions[session_id] }&.handle_message(message)
     elsif message['id']
       # Callbacks could be all rejected if someone has called `.dispose()`.
-      if callback = @callbacks.delete(message['id'])
+      if callback = @callbacks_mutex.synchronize { @callbacks.delete(message['id']) }
         if message['error']
           callback.reject(
             ProtocolError.new(
@@ -293,17 +310,19 @@ class Puppeteer::Connection
     @closed = true
     @transport.on_message
     @transport.on_close
-    @callbacks.each_value do |callback|
+    callbacks = @callbacks_mutex.synchronize do
+      @callbacks.values.tap { @callbacks.clear }
+    end
+    callbacks.each do |callback|
       callback.reject(
         ProtocolError.new(
           method: callback.method,
           error_message: 'Target Closed.'))
     end
-    @callbacks.clear
-    @sessions.each_value do |session|
-      session.handle_closed
+    sessions = @sessions_mutex.synchronize do
+      @sessions.values.tap { @sessions.clear }
     end
-    @sessions.clear
+    sessions.each(&:handle_closed)
     emit_event(ConnectionEmittedEvents::Disconnected)
   end
 
@@ -333,6 +352,6 @@ class Puppeteer::Connection
     result = send_message('Target.attachToTarget', targetId: target_info.target_id, flatten: true)
     session_id = result['sessionId']
     @manually_attached.delete(target_info.target_id)
-    @sessions[session_id]
+    @sessions_mutex.synchronize { @sessions[session_id] }.tap { |session| session&.mark_ready }
   end
 end

data/lib/puppeteer/console_message.rb

@@ -17,7 +17,15 @@ class Puppeteer::ConsoleMessage
     @log_type = log_type
     @text = text
     @args = args
-    @stack_trace_locations = stack_trace_locations
+    @stack_trace_locations =
+      case stack_trace_locations
+      when nil
+        []
+      when Array
+        stack_trace_locations
+      else
+        [stack_trace_locations]
+      end
   end
 
   attr_reader :log_type, :text, :args