puppeteer-ruby 0.42.0 → 0.43.0

data/lib/puppeteer/chrome_target_manager.rb

@@ -0,0 +1,256 @@
+class Puppeteer::ChromeTargetManager
+  include Puppeteer::EventCallbackable
+
+  def initialize(connection:, target_factory:, target_filter_callback:)
+    @discovered_targets_by_target_id = {}
+    @attached_targets_by_target_id = {}
+    @attached_targets_by_session_id = {}
+    @ignored_targets = Set.new
+    @target_ids_for_init = Set.new
+
+    @connection = connection
+    @target_filter_callback = target_filter_callback
+    @target_factory = target_factory
+    @target_interceptors = {}
+    @initialize_promise = resolvable_future
+
+    @connection_event_listeners = []
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetCreated',
+      &method(:handle_target_created)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetDestroyed',
+      &method(:handle_target_destroyed)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetInfoChanged',
+      &method(:handle_target_info_changed)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'sessiondetached',
+      &method(:handle_session_detached)
+    )
+
+    setup_attachment_listeners(@connection)
+    @connection.async_send_message('Target.setDiscoverTargets', discover: true)
+  end
+
+  def init
+    @discovered_targets_by_target_id.each do |target_id, target_info|
+      if @target_filter_callback.call(target_info)
+        @target_ids_for_init << target_id
+      end
+    end
+    @connection.send_message('Target.setAutoAttach', {
+      waitForDebuggerOnStart: true,
+      flatten: true,
+      autoAttach: true,
+    })
+    @initialize_promise.value!
+  end
+
+  def dispose
+    @connection.remove_event_listener(*@connection_event_listeners)
+    remove_attachment_listeners(@connection)
+  end
+
+  def available_targets
+    @attached_targets_by_target_id
+  end
+
+  def add_target_interceptor(client, interceptor)
+    interceptors = @target_interceptors[client] || []
+    interceptors << interceptor
+    @target_interceptors[client] = interceptors
+  end
+
+  def remove_target_interceptor(client, interceptor)
+    @target_interceptors[client]&.delete_if { |current| current == interceptor }
+  end
+
+  private def setup_attachment_listeners(session)
+    @attachment_listener_ids ||= {}
+    @attachment_listener_ids[session] ||= []
+
+    @attachment_listener_ids[session] << session.add_event_listener('Target.attachedToTarget') do |event|
+      handle_attached_to_target(session, event)
+    end
+
+    @attachment_listener_ids[session] << session.add_event_listener('Target.detachedFromTarget') do |event|
+      handle_detached_from_target(session, event)
+    end
+  end
+
+  private def remove_attachment_listeners(session)
+    return unless @attachment_listener_ids
+    listener_ids = @attachment_listener_ids.delete(session)
+    return if !listener_ids || listener_ids.empty?
+    session.remove_event_listener(*listener_ids)
+  end
+
+  private def handle_session_detached(session)
+    remove_attachment_listeners(session)
+    @target_interceptors.delete(session)
+  end
+
+  private def handle_target_created(event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    @discovered_targets_by_target_id[target_info.target_id] = target_info
+
+    emit_event(TargetManagerEmittedEvents::TargetDiscovered, target_info)
+
+    # The connection is already attached to the browser target implicitly,
+    # therefore, no new CDPSession is created and we have special handling
+    # here.
+    if target_info.type == 'browser' && target_info.attached
+      return if @attached_targets_by_target_id[target_info.target_id]
+
+      target = @target_factory.call(target_info, nil)
+      @attached_targets_by_target_id[target_info.target_id] = target
+    end
+
+    if target_info.type == 'shared_worker'
+      # Special case (https://crbug.com/1338156): currently, shared_workers
+      # don't get auto-attached. This should be removed once the auto-attach
+      # works.
+      @connection.create_session(target_info)
+    end
+  end
+
+  private def handle_target_destroyed(event)
+    target_id = event['targetId']
+    target_info = @discovered_targets_by_target_id.delete(target_id)
+    finish_initialization_if_ready(target_id)
+
+    if target_info.type == 'service_worker' && @attached_targets_by_target_id.has_key?(target_id)
+      # Special case for service workers: report TargetGone event when
+      # the worker is destroyed.
+      target = @attached_targets_by_target_id.delete(target_id)
+      emit_event(TargetManagerEmittedEvents::TargetGone, target)
+    end
+  end
+
+  private def handle_target_info_changed(event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    @discovered_targets_by_target_id[target_info.target_id] = target_info
+
+    if @ignored_targets.include?(target_info.target_id) || !@attached_targets_by_target_id.has_key?(target_info.target_id) || !target_info.attached
+      return
+    end
+    original_target = @attached_targets_by_target_id[target_info.target_id]
+    emit_event(TargetManagerEmittedEvents::TargetChanged, original_target, target_info)
+  end
+
+  class SessionNotCreatedError < StandardError ; end
+
+  private def handle_attached_to_target(parent_session, event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    session_id = event['sessionId']
+    session = @connection.session(session_id)
+    unless session
+      raise SessionNotCreatedError.new("Session #{session_id} was not created.")
+    end
+
+    silent_detach = -> {
+      begin
+        session.send_message('Runtime.runIfWaitingForDebugger')
+      rescue => err
+        Logger.new($stderr).warn(err)
+      end
+
+      # We don't use `session.detach()` because that dispatches all commands on
+      # the connection instead of the parent session.
+      begin
+        parent_session.send_message('Target.detachFromTarget', {
+          sessionId: session.id,
+        })
+      rescue => err
+        Logger.new($stderr).warn(err)
+      end
+    }
+
+    # Special case for service workers: being attached to service workers will
+    # prevent them from ever being destroyed. Therefore, we silently detach
+    # from service workers unless the connection was manually created via
+    # `page.worker()`. To determine this, we use
+    # `this.#connection.isAutoAttached(targetInfo.targetId)`. In the future, we
+    # should determine if a target is auto-attached or not with the help of
+    # CDP.
+    if target_info.type == 'service_worker' && @connection.auto_attached?(target_info.target_id)
+      finish_initialization_if_ready(target_info.target_id)
+      silent_detach.call
+      if parent_session.is_a?(Puppeteer::CDPSession)
+        target = @target_factory.call(target_info, parent_session)
+        @attached_targets_by_target_id[target_info.target_id] = target
+        emit_event(TargetManagerEmittedEvents::TargetAvailable, target)
+      end
+
+      return
+    end
+
+    unless @target_filter_callback.call(target_info)
+      @ignored_targets << target_info.target_id
+      finish_initialization_if_ready(target_info.target_id)
+      silent_detach.call
+
+      return
+    end
+
+    target = @attached_targets_by_target_id[target_info.target_id] || @target_factory.call(target_info, session)
+    setup_attachment_listeners(session)
+
+    @attached_targets_by_target_id[target_info.target_id] ||= target
+    @attached_targets_by_session_id[session.id] = target
+
+    @target_interceptors[parent_session]&.each do |interceptor|
+      if parent_session.is_a?(Puppeteer::Connection)
+        interceptor.call(target, nil)
+      else
+        # Sanity check: if parent session is not a connection, it should be
+        # present in #attachedTargetsBySessionId.
+        attached_target = @attached_targets_by_session_id[parent_session.id]
+        unless attached_target
+          raise "No target found for the parent session: #{parent_session.id}"
+        end
+        interceptor.call(target, attached_target)
+      end
+    end
+
+    @target_ids_for_init.delete(target.target_id)
+    future { emit_event(TargetManagerEmittedEvents::TargetAvailable, target) }
+
+    if @target_ids_for_init.empty?
+      @initialize_promise.fulfill(nil) unless @initialize_promise.resolved?
+    end
+
+    future do
+      # TODO: the browser might be shutting down here. What do we do with the error?
+      await_all(
+        session.async_send_message('Target.setAutoAttach', {
+          waitForDebuggerOnStart: true,
+          flatten: true,
+          autoAttach: true,
+        }),
+        session.async_send_message('Runtime.runIfWaitingForDebugger'),
+      )
+    rescue => err
+      Logger.new($stderr).warn(err)
+    end
+  end
+
+  private def finish_initialization_if_ready(target_id)
+    @target_ids_for_init.delete(target_id)
+    if @target_ids_for_init.empty?
+      @initialize_promise.fulfill(nil) unless @initialize_promise.resolved?
+    end
+  end
+
+  private def handle_detached_from_target(parent_session, event)
+    session_id = event['sessionId']
+    target = @attached_targets_by_session_id.delete(session_id)
+    return unless target
+    @attached_targets_by_target_id.delete(target.target_id)
+    emit_event(TargetManagerEmittedEvents::TargetGone, target)
+  end
+end

data/lib/puppeteer/connection.rb

@@ -58,6 +58,7 @@ class Puppeteer::Connection
 
     @sessions = Concurrent::Hash.new
     @closed = false
+    @manually_attached = Set.new
   end
 
   # used only in Browser#connected?
@@ -243,12 +244,22 @@ class Puppeteer::Connection
       session_id = message['params']['sessionId']
       session = Puppeteer::CDPSession.new(self, message['params']['targetInfo']['type'], session_id)
       @sessions[session_id] = session
+      emit_event('sessionattached', session)
+      if message['sessionId']
+        parent_session = @sessions[message['sessionId']]
+        parent_session&.emit_event('sessionattached', session)
+      end
     when 'Target.detachedFromTarget'
       session_id = message['params']['sessionId']
       session = @sessions[session_id]
       if session
         session.handle_closed
         @sessions.delete(session_id)
+        emit_event('sessiondetached', session)
+        if message['sessionId']
+          parent_session = @sessions[message['sessionId']]
+          parent_session&.emit_event('sessiondetached', session)
+        end
       end
     end
 
@@ -307,9 +318,14 @@ class Puppeteer::Connection
     @transport.close
   end
 
+  def auto_attached?(target_id)
+    @manually_attached.include?(target_id)
+  end
+
   # @param {Protocol.Target.TargetInfo} targetInfo
   # @return [CDPSession]
   def create_session(target_info)
+    @manually_attached << target_info.target_id
     result = send_message('Target.attachToTarget', targetId: target_info.target_id, flatten: true)
     session_id = result['sessionId']
     @sessions[session_id]

data/lib/puppeteer/events.rb

@@ -182,3 +182,12 @@ module PageEmittedEvents ; end
   # {https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API WebWorker} is destroyed by the page.
   WorkerDestroyed: 'workerdestroyed',
 }.define_const_into(PageEmittedEvents)
+
+module TargetManagerEmittedEvents ; end
+
+{
+  TargetDiscovered: 'targetDiscovered',
+  TargetAvailable: 'targetAvailable',
+  TargetGone: 'targetGone',
+  TargetChanged: 'targetChanged',
+}.define_const_into(TargetManagerEmittedEvents)

data/lib/puppeteer/firefox_target_manager.rb

@@ -0,0 +1,158 @@
+# FirefoxTargetManager implements target management using
+# `Target.setDiscoverTargets` without using auto-attach. It, therefore, creates
+# targets that lazily establish their CDP sessions.
+#
+# Although the approach is potentially flaky, there is no other way for Firefox
+# because Firefox's CDP implementation does not support auto-attach.
+#
+# Firefox does not support targetInfoChanged and detachedFromTarget events:
+# - https://bugzilla.mozilla.org/show_bug.cgi?id=1610855
+# - https://bugzilla.mozilla.org/show_bug.cgi?id=1636979
+class Puppeteer::FirefoxTargetManager
+  include Puppeteer::EventCallbackable
+
+  def initialize(connection:, target_factory:, target_filter_callback:)
+    @discovered_targets_by_target_id = {}
+    @available_targets_by_target_id = {}
+    @available_targets_by_session_id = {}
+    @ignored_targets = Set.new
+    @target_ids_for_init = Set.new
+
+    @connection = connection
+    @target_filter_callback = target_filter_callback
+    @target_factory = target_factory
+    @target_interceptors = {}
+    @initialize_promise = resolvable_future
+
+    @connection_event_listeners = []
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetCreated',
+      &method(:handle_target_created)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetDestroyed',
+      &method(:handle_target_destroyed)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'sessiondetached',
+      &method(:handle_session_detached)
+    )
+
+    setup_attachment_listeners(@connection)
+  end
+
+  def add_target_interceptor(client, interceptor)
+    interceptors = @target_interceptors[client] || []
+    interceptors << interceptor
+    @target_interceptors[client] = interceptors
+  end
+
+  def remove_target_interceptor(client, interceptor)
+    @target_interceptors[client]&.delete_if { |current| current == interceptor }
+  end
+
+  private def setup_attachment_listeners(session)
+    @attachment_listener_ids ||= {}
+    @attachment_listener_ids[session] ||= []
+
+    @attachment_listener_ids[session] << session.add_event_listener('Target.attachedToTarget') do |event|
+      handle_attached_to_target(session, event)
+    end
+  end
+
+  private def handle_session_detached(session)
+    remove_session_listeners(session)
+    @target_interceptors.delete(session)
+  end
+
+  private def remove_session_listeners(session)
+    return unless @attachment_listener_ids
+    listener_ids = @attachment_listener_ids.delete(session)
+    return if listener_ids.empty?
+    session.remove_event_listener(*listener_ids)
+  end
+
+  def available_targets
+    @available_targets_by_target_id
+  end
+
+  def dispose
+    @connection.remove_event_listener(*@connection_event_listeners)
+    remove_session_listeners(@connection)
+  end
+
+  def init
+    @connection.send_message('Target.setDiscoverTargets', discover: true)
+    @target_ids_for_init.merge(@discovered_targets_by_target_id.keys)
+    @initialize_promise.value!
+  end
+
+  private def handle_target_created(event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    return if @discovered_targets_by_target_id[target_info.target_id]
+    @discovered_targets_by_target_id[target_info.target_id] = target_info
+
+    if target_info.type == 'browser' && target_info.attached
+      target = @target_factory.call(target_info, nil)
+      @available_targets_by_target_id[target_info.target_id] = target
+      finish_initialization_if_ready(target.target_id)
+    end
+
+    unless @target_filter_callback.call(target_info)
+      @ignored_targets << target_info.target_id
+      finish_initialization_if_ready(target_info.target_id)
+      return
+    end
+
+    target = @target_factory.call(target_info, nil)
+    @available_targets_by_target_id[target_info.target_id] = target
+    emit_event(TargetManagerEmittedEvents::TargetAvailable, target)
+    finish_initialization_if_ready(target.target_id)
+  end
+
+  private def handle_target_destroyed(event)
+    target_id = event['targetId']
+    target_info = @discovered_targets_by_target_id.delete(target_id)
+    finish_initialization_if_ready(target_id)
+
+    target = @available_targets_by_target_id.delete(target_id)
+    emit_event(TargetManagerEmittedEvents::TargetGone, target)
+  end
+
+  class SessionNotCreatedError < StandardError ; end
+
+  private def handle_attached_to_target(parent_session, event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    session_id = event['sessionId']
+    session = @connection.session(session_id)
+    unless session
+      raise SessionNotCreatedError.new("Session #{session_id} was not created.")
+    end
+
+    target = @available_targets_by_target_id[target_info.target_id] or raise "Target #{target_info.target_id} is missing"
+    setup_attachment_listeners(session)
+
+    @available_targets_by_session_id[session_id] = target
+
+    @target_interceptors[parent_session]&.each do |hook|
+      if parent_session.is_a?(Puppeteer::Connection)
+        hook.call(target, nil)
+      else
+        # Sanity check: if parent session is not a connection, it should be
+        # present in #attachedTargetsBySessionId.
+        available_target = @available_targets_by_session_id[parent_session.id]
+        unless available_target
+          raise "No target found for the parent session: #{parent_session.id}"
+        end
+        hook.call(target, available_target)
+      end
+    end
+  end
+
+  private def finish_initialization_if_ready(target_id)
+    @target_ids_for_init.delete(target_id)
+    if @target_ids_for_init.empty?
+      @initialize_promise.fulfill(nil) unless @initialize_promise.resolved?
+    end
+  end
+end

data/lib/puppeteer/frame_manager.rb

@@ -27,6 +27,13 @@ class Puppeteer::FrameManager
     # @type {!Set<string>}
     @isolated_worlds = Set.new
 
+    # Keeps track of OOPIF targets/frames (target ID == frame ID for OOPIFs)
+    # that are being initialized.
+    @frames_pending_target_init = {}
+
+    # Keeps track of frames that are in the process of being attached in #onFrameAttached.
+    @frames_pending_attachment = {}
+
     setup_listeners(@client)
   end
 
@@ -61,27 +68,17 @@ class Puppeteer::FrameManager
     client.on_event('Page.lifecycleEvent') do |event|
       handle_lifecycle_event(event)
     end
-    client.on_event('Target.attachedToTarget') do |event|
-      handle_attached_to_target(event)
-    end
-    client.on_event('Target.detachedFromTarget') do |event|
-      handle_detached_from_target(event)
-    end
   end
 
   attr_reader :client, :timeout_settings
 
-  private def init(cdp_session = nil)
+  private def init(target_id, cdp_session = nil)
+    @frames_pending_target_init[target_id] ||= resolvable_future
     client = cdp_session || @client
 
     promises = [
       client.async_send_message('Page.enable'),
       client.async_send_message('Page.getFrameTree'),
-      cdp_session&.async_send_message('Target.setAutoAttach', {
-        autoAttach: true,
-        waitForDebuggerOnStart: false,
-        flatten: true,
-      }),
     ].compact
     results = await_all(*promises)
     frame_tree = results[1]['frameTree']
@@ -97,6 +94,8 @@ class Puppeteer::FrameManager
     return if err.message.include?('Target closed') || err.message.include?('Session closed')
 
     raise
+  ensure
+    @frames_pending_target_init.delete(target_id)&.fulfill(nil)
   end
 
   define_async_method :async_init
@@ -121,12 +120,13 @@ class Puppeteer::FrameManager
     option_timeout = timeout || @timeout_settings.navigation_timeout
 
     watcher = Puppeteer::LifecycleWatcher.new(self, frame, option_wait_until, option_timeout)
+    ensure_new_document_navigation = false
 
     begin
       navigate = future do
         result = @client.send_message('Page.navigate', navigate_params)
         loader_id = result['loaderId']
-
+        ensure_new_document_navigation = !!loader_id
         if result['errorText']
           raise NavigationError.new("#{result['errorText']} at #{url}")
         end
@@ -137,9 +137,12 @@ class Puppeteer::FrameManager
       )
 
       await_any(
-        watcher.new_document_navigation_promise,
-        watcher.same_document_navigation_promise,
         watcher.timeout_or_termination_promise,
+        if ensure_new_document_navigation
+          watcher.new_document_navigation_promise
+        else
+          watcher.same_document_navigation_promise
+        end,
       )
     rescue Puppeteer::TimeoutError => err
       raise NavigationError.new(err)
@@ -175,20 +178,19 @@ class Puppeteer::FrameManager
   end
 
   # @param event [Hash]
-  def handle_attached_to_target(event)
-    return if event['targetInfo']['type'] != 'iframe'
-
-    frame = @frames[event['targetInfo']['targetId']]
-    session = Puppeteer::Connection.from_session(@client).session(event['sessionId'])
+  def handle_attached_to_target(target)
+    return if target.target_info.type != 'iframe'
 
+    frame = @frames[target.target_info.target_id]
+    session = target.session
     frame&.send(:update_client, session)
     setup_listeners(session)
-    async_init(session)
+    async_init(target.target_info.target_id, session)
   end
 
   # @param event [Hash]
-  def handle_detached_from_target(event)
-    frame = @frames[event['targetId']]
+  def handle_detached_from_target(target)
+    frame = @frames[target.target_id]
     if frame && frame.oop_frame?
       # When an OOP iframe is removed from the page, it
       # will only get a Target.detachedFromTarget event.
@@ -256,7 +258,7 @@ class Puppeteer::FrameManager
 
   # @param session [Puppeteer::CDPSession]
   # @param frameId [String]
-  # @param parentFrameId [String|nil]
+  # @param parentFrameId [String]
   def handle_frame_attached(session, frame_id, parent_frame_id)
     if @frames.has_key?(frame_id)
       frame = @frames[frame_id]
@@ -268,28 +270,57 @@ class Puppeteer::FrameManager
       end
       return
     end
-    if !parent_frame_id
-      raise ArgymentError.new('parent_frame_id must not be nil')
-    end
     parent_frame = @frames[parent_frame_id]
-    frame = Puppeteer::Frame.new(self, parent_frame, frame_id, session)
-    @frames[frame_id] = frame
+    if parent_frame
+      attach_child_frame(parent_frame, parent_frame_id, frame_id, session)
+      return
+    end
 
+    if @frames_pending_target_init[parent_frame_id]
+      @frames_pending_attachment[frame_id] ||= resolvable_future
+      @frames_pending_target_init[parent_frame_id].then do |_|
+        attach_child_frame(@frames[parent_frame_id], parent_frame_id, frame_id, session)
+        @frames_pending_attachment.delete(frame_id)&.fulfill(nil)
+      end
+      return
+    end
+
+    raise FrameNotFoundError.new("Parent frame #{parent_frame_id} not found.")
+  end
+
+  class FrameNotFoundError < StandardError ; end
+
+  private def attach_child_frame(parent_frame, parent_frame_id, frame_id, session)
+    unless parent_frame
+      raise FrameNotFoundError.new("Parent frame #{parent_frame_id} not found.")
+    end
+
+    frame = Puppeteer::Frame.new(self, parent_frame, frame_id, session)
+    @frames[frame.id] = frame
     emit_event(FrameManagerEmittedEvents::FrameAttached, frame)
+    frame
   end
 
   # @param frame_payload [Hash]
   def handle_frame_navigated(frame_payload)
+    frame_id = frame_payload['id']
     is_main_frame = !frame_payload['parentId']
-    frame =
-      if is_main_frame
-        @main_frame
-      else
-        @frames[frame_payload['id']]
+
+
+    if @frames_pending_attachment[frame_id]
+      @frames_pending_attachment[frame_id].then do |_|
+        frame = is_main_frame ? @main_frame : @frames[frame_id]
+        reattach_frame(frame, frame_id, is_main_frame, frame_payload)
       end
+    else
+      frame = is_main_frame ? @main_frame : @frames[frame_id]
+      reattach_frame(frame, frame_id, is_main_frame, frame_payload)
+    end
+  end
 
+  private def reattach_frame(frame, frame_id, is_main_frame, frame_payload)
     if !is_main_frame && !frame
-      raise ArgumentError.new('We either navigate top level or have old version of the navigated frame')
+      raise "Missing frame isMainFrame=#{is_main_frame}, frameId=#{frame_id}"
     end
 
     # Detach all child frames first.
@@ -304,12 +335,12 @@ class Puppeteer::FrameManager
       if frame
         # Update frame id to retain frame identity on cross-process navigation.
         @frames.delete(frame.id)
-        frame.id = frame_payload['id']
+        frame.id = frame_id
       else
         # Initial main frame navigation.
-        frame = Puppeteer::Frame.new(self, nil, frame_payload['id'], @client)
+        frame = Puppeteer::Frame.new(self, nil, frame_id, @client)
       end
-      @frames[frame_payload['id']] = frame
+      @frames[frame_id] = frame
       @main_frame = frame
     end