puppeteer-ruby 0.41.0 → 0.43.0

data/lib/puppeteer/connection.rb

@@ -58,6 +58,7 @@ class Puppeteer::Connection
 
     @sessions = Concurrent::Hash.new
     @closed = false
+    @manually_attached = Set.new
   end
 
   # used only in Browser#connected?
@@ -243,12 +244,22 @@ class Puppeteer::Connection
       session_id = message['params']['sessionId']
       session = Puppeteer::CDPSession.new(self, message['params']['targetInfo']['type'], session_id)
       @sessions[session_id] = session
+      emit_event('sessionattached', session)
+      if message['sessionId']
+        parent_session = @sessions[message['sessionId']]
+        parent_session&.emit_event('sessionattached', session)
+      end
     when 'Target.detachedFromTarget'
       session_id = message['params']['sessionId']
       session = @sessions[session_id]
       if session
         session.handle_closed
         @sessions.delete(session_id)
+        emit_event('sessiondetached', session)
+        if message['sessionId']
+          parent_session = @sessions[message['sessionId']]
+          parent_session&.emit_event('sessiondetached', session)
+        end
       end
     end
 
@@ -307,9 +318,14 @@ class Puppeteer::Connection
     @transport.close
   end
 
+  def auto_attached?(target_id)
+    @manually_attached.include?(target_id)
+  end
+
   # @param {Protocol.Target.TargetInfo} targetInfo
   # @return [CDPSession]
   def create_session(target_info)
+    @manually_attached << target_info.target_id
     result = send_message('Target.attachToTarget', targetId: target_info.target_id, flatten: true)
     session_id = result['sessionId']
     @sessions[session_id]

data/lib/puppeteer/element_handle.rb

@@ -169,37 +169,25 @@ class Puppeteer::ElementHandle < Puppeteer::JSHandle
 
   def scroll_into_view_if_needed
     js = <<~JAVASCRIPT
-      async(element, pageJavascriptEnabled) => {
+      async(element) => {
         if (!element.isConnected)
           return 'Node is detached from document';
         if (element.nodeType !== Node.ELEMENT_NODE)
           return 'Node is not of type HTMLElement';
-
-        if (element.scrollIntoViewIfNeeded) {
-          element.scrollIntoViewIfNeeded({block: 'center', inline: 'center', behavior: 'instant'});
-        } else {
-          // force-scroll if page's javascript is disabled.
-          if (!pageJavascriptEnabled) {
-            element.scrollIntoView({block: 'center', inline: 'center', behavior: 'instant'});
-            return false;
-          }
-          const visibleRatio = await new Promise(resolve => {
-            const observer = new IntersectionObserver(entries => {
-              resolve(entries[0].intersectionRatio);
-              observer.disconnect();
-            });
-            observer.observe(element);
-          });
-          if (visibleRatio !== 1.0)
-            element.scrollIntoView({block: 'center', inline: 'center', behavior: 'instant'});
-        }
         return false;
       }
     JAVASCRIPT
-    error = evaluate(js, @page.javascript_enabled) # returns String or false
+    error = evaluate(js) # returns String or false
     if error
       raise ScrollIntoViewError.new(error)
     end
+    begin
+      @remote_object.scroll_into_view_if_needed(@client)
+    rescue => err
+      # Just ignore 'Node does not have a layout object' for backward-compatibility.
+      raise unless err.message =~ /Node does not have a layout object/
+    end
+
     # clickpoint is often calculated before scrolling is completed.
     # So, just sleep about 10 frames
     sleep 0.16
@@ -569,7 +557,7 @@ class Puppeteer::ElementHandle < Puppeteer::JSHandle
       capture_beyond_viewport: capture_beyond_viewport,
       from_surface: from_surface)
   ensure
-    if needs_viewport_reset
+    if needs_viewport_reset && viewport
       @page.viewport = viewport
     end
   end

data/lib/puppeteer/events.rb

@@ -182,3 +182,12 @@ module PageEmittedEvents ; end
   # {https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API WebWorker} is destroyed by the page.
   WorkerDestroyed: 'workerdestroyed',
 }.define_const_into(PageEmittedEvents)
+
+module TargetManagerEmittedEvents ; end
+
+{
+  TargetDiscovered: 'targetDiscovered',
+  TargetAvailable: 'targetAvailable',
+  TargetGone: 'targetGone',
+  TargetChanged: 'targetChanged',
+}.define_const_into(TargetManagerEmittedEvents)

data/lib/puppeteer/firefox_target_manager.rb

@@ -0,0 +1,158 @@
+# FirefoxTargetManager implements target management using
+# `Target.setDiscoverTargets` without using auto-attach. It, therefore, creates
+# targets that lazily establish their CDP sessions.
+#
+# Although the approach is potentially flaky, there is no other way for Firefox
+# because Firefox's CDP implementation does not support auto-attach.
+#
+# Firefox does not support targetInfoChanged and detachedFromTarget events:
+# - https://bugzilla.mozilla.org/show_bug.cgi?id=1610855
+# - https://bugzilla.mozilla.org/show_bug.cgi?id=1636979
+class Puppeteer::FirefoxTargetManager
+  include Puppeteer::EventCallbackable
+
+  def initialize(connection:, target_factory:, target_filter_callback:)
+    @discovered_targets_by_target_id = {}
+    @available_targets_by_target_id = {}
+    @available_targets_by_session_id = {}
+    @ignored_targets = Set.new
+    @target_ids_for_init = Set.new
+
+    @connection = connection
+    @target_filter_callback = target_filter_callback
+    @target_factory = target_factory
+    @target_interceptors = {}
+    @initialize_promise = resolvable_future
+
+    @connection_event_listeners = []
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetCreated',
+      &method(:handle_target_created)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetDestroyed',
+      &method(:handle_target_destroyed)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'sessiondetached',
+      &method(:handle_session_detached)
+    )
+
+    setup_attachment_listeners(@connection)
+  end
+
+  def add_target_interceptor(client, interceptor)
+    interceptors = @target_interceptors[client] || []
+    interceptors << interceptor
+    @target_interceptors[client] = interceptors
+  end
+
+  def remove_target_interceptor(client, interceptor)
+    @target_interceptors[client]&.delete_if { |current| current == interceptor }
+  end
+
+  private def setup_attachment_listeners(session)
+    @attachment_listener_ids ||= {}
+    @attachment_listener_ids[session] ||= []
+
+    @attachment_listener_ids[session] << session.add_event_listener('Target.attachedToTarget') do |event|
+      handle_attached_to_target(session, event)
+    end
+  end
+
+  private def handle_session_detached(session)
+    remove_session_listeners(session)
+    @target_interceptors.delete(session)
+  end
+
+  private def remove_session_listeners(session)
+    return unless @attachment_listener_ids
+    listener_ids = @attachment_listener_ids.delete(session)
+    return if listener_ids.empty?
+    session.remove_event_listener(*listener_ids)
+  end
+
+  def available_targets
+    @available_targets_by_target_id
+  end
+
+  def dispose
+    @connection.remove_event_listener(*@connection_event_listeners)
+    remove_session_listeners(@connection)
+  end
+
+  def init
+    @connection.send_message('Target.setDiscoverTargets', discover: true)
+    @target_ids_for_init.merge(@discovered_targets_by_target_id.keys)
+    @initialize_promise.value!
+  end
+
+  private def handle_target_created(event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    return if @discovered_targets_by_target_id[target_info.target_id]
+    @discovered_targets_by_target_id[target_info.target_id] = target_info
+
+    if target_info.type == 'browser' && target_info.attached
+      target = @target_factory.call(target_info, nil)
+      @available_targets_by_target_id[target_info.target_id] = target
+      finish_initialization_if_ready(target.target_id)
+    end
+
+    unless @target_filter_callback.call(target_info)
+      @ignored_targets << target_info.target_id
+      finish_initialization_if_ready(target_info.target_id)
+      return
+    end
+
+    target = @target_factory.call(target_info, nil)
+    @available_targets_by_target_id[target_info.target_id] = target
+    emit_event(TargetManagerEmittedEvents::TargetAvailable, target)
+    finish_initialization_if_ready(target.target_id)
+  end
+
+  private def handle_target_destroyed(event)
+    target_id = event['targetId']
+    target_info = @discovered_targets_by_target_id.delete(target_id)
+    finish_initialization_if_ready(target_id)
+
+    target = @available_targets_by_target_id.delete(target_id)
+    emit_event(TargetManagerEmittedEvents::TargetGone, target)
+  end
+
+  class SessionNotCreatedError < StandardError ; end
+
+  private def handle_attached_to_target(parent_session, event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    session_id = event['sessionId']
+    session = @connection.session(session_id)
+    unless session
+      raise SessionNotCreatedError.new("Session #{session_id} was not created.")
+    end
+
+    target = @available_targets_by_target_id[target_info.target_id] or raise "Target #{target_info.target_id} is missing"
+    setup_attachment_listeners(session)
+
+    @available_targets_by_session_id[session_id] = target
+
+    @target_interceptors[parent_session]&.each do |hook|
+      if parent_session.is_a?(Puppeteer::Connection)
+        hook.call(target, nil)
+      else
+        # Sanity check: if parent session is not a connection, it should be
+        # present in #attachedTargetsBySessionId.
+        available_target = @available_targets_by_session_id[parent_session.id]
+        unless available_target
+          raise "No target found for the parent session: #{parent_session.id}"
+        end
+        hook.call(target, available_target)
+      end
+    end
+  end
+
+  private def finish_initialization_if_ready(target_id)
+    @target_ids_for_init.delete(target_id)
+    if @target_ids_for_init.empty?
+      @initialize_promise.fulfill(nil) unless @initialize_promise.resolved?
+    end
+  end
+end

data/lib/puppeteer/frame.rb

@@ -41,6 +41,10 @@ class Puppeteer::Frame
     @secondary_world = Puppeteer::DOMWorld.new(@client, @frame_manager, self, @frame_manager.timeout_settings)
   end
 
+  def page
+    @frame_manager.page
+  end
+
   def oop_frame?
     @client != @frame_manager.client
   end

data/lib/puppeteer/frame_manager.rb

@@ -27,6 +27,13 @@ class Puppeteer::FrameManager
     # @type {!Set<string>}
     @isolated_worlds = Set.new
 
+    # Keeps track of OOPIF targets/frames (target ID == frame ID for OOPIFs)
+    # that are being initialized.
+    @frames_pending_target_init = {}
+
+    # Keeps track of frames that are in the process of being attached in #onFrameAttached.
+    @frames_pending_attachment = {}
+
     setup_listeners(@client)
   end
 
@@ -61,27 +68,17 @@ class Puppeteer::FrameManager
     client.on_event('Page.lifecycleEvent') do |event|
       handle_lifecycle_event(event)
     end
-    client.on_event('Target.attachedToTarget') do |event|
-      handle_attached_to_target(event)
-    end
-    client.on_event('Target.detachedFromTarget') do |event|
-      handle_detached_from_target(event)
-    end
   end
 
   attr_reader :client, :timeout_settings
 
-  private def init(cdp_session = nil)
+  private def init(target_id, cdp_session = nil)
+    @frames_pending_target_init[target_id] ||= resolvable_future
     client = cdp_session || @client
 
     promises = [
       client.async_send_message('Page.enable'),
       client.async_send_message('Page.getFrameTree'),
-      cdp_session&.async_send_message('Target.setAutoAttach', {
-        autoAttach: true,
-        waitForDebuggerOnStart: false,
-        flatten: true,
-      }),
     ].compact
     results = await_all(*promises)
     frame_tree = results[1]['frameTree']
@@ -97,6 +94,8 @@ class Puppeteer::FrameManager
     return if err.message.include?('Target closed') || err.message.include?('Session closed')
 
     raise
+  ensure
+    @frames_pending_target_init.delete(target_id)&.fulfill(nil)
   end
 
   define_async_method :async_init
@@ -121,12 +120,13 @@ class Puppeteer::FrameManager
     option_timeout = timeout || @timeout_settings.navigation_timeout
 
     watcher = Puppeteer::LifecycleWatcher.new(self, frame, option_wait_until, option_timeout)
+    ensure_new_document_navigation = false
 
     begin
       navigate = future do
         result = @client.send_message('Page.navigate', navigate_params)
         loader_id = result['loaderId']
-
+        ensure_new_document_navigation = !!loader_id
         if result['errorText']
           raise NavigationError.new("#{result['errorText']} at #{url}")
         end
@@ -137,9 +137,12 @@ class Puppeteer::FrameManager
       )
 
       await_any(
-        watcher.new_document_navigation_promise,
-        watcher.same_document_navigation_promise,
         watcher.timeout_or_termination_promise,
+        if ensure_new_document_navigation
+          watcher.new_document_navigation_promise
+        else
+          watcher.same_document_navigation_promise
+        end,
       )
     rescue Puppeteer::TimeoutError => err
       raise NavigationError.new(err)
@@ -175,20 +178,19 @@ class Puppeteer::FrameManager
   end
 
   # @param event [Hash]
-  def handle_attached_to_target(event)
-    return if event['targetInfo']['type'] != 'iframe'
-
-    frame = @frames[event['targetInfo']['targetId']]
-    session = Puppeteer::Connection.from_session(@client).session(event['sessionId'])
+  def handle_attached_to_target(target)
+    return if target.target_info.type != 'iframe'
 
+    frame = @frames[target.target_info.target_id]
+    session = target.session
     frame&.send(:update_client, session)
     setup_listeners(session)
-    async_init(session)
+    async_init(target.target_info.target_id, session)
   end
 
   # @param event [Hash]
-  def handle_detached_from_target(event)
-    frame = @frames[event['targetId']]
+  def handle_detached_from_target(target)
+    frame = @frames[target.target_id]
     if frame && frame.oop_frame?
       # When an OOP iframe is removed from the page, it
       # will only get a Target.detachedFromTarget event.
@@ -256,7 +258,7 @@ class Puppeteer::FrameManager
 
   # @param session [Puppeteer::CDPSession]
   # @param frameId [String]
-  # @param parentFrameId [String|nil]
+  # @param parentFrameId [String]
   def handle_frame_attached(session, frame_id, parent_frame_id)
     if @frames.has_key?(frame_id)
       frame = @frames[frame_id]
@@ -268,28 +270,57 @@ class Puppeteer::FrameManager
       end
       return
     end
-    if !parent_frame_id
-      raise ArgymentError.new('parent_frame_id must not be nil')
-    end
     parent_frame = @frames[parent_frame_id]
-    frame = Puppeteer::Frame.new(self, parent_frame, frame_id, session)
-    @frames[frame_id] = frame
+    if parent_frame
+      attach_child_frame(parent_frame, parent_frame_id, frame_id, session)
+      return
+    end
 
+    if @frames_pending_target_init[parent_frame_id]
+      @frames_pending_attachment[frame_id] ||= resolvable_future
+      @frames_pending_target_init[parent_frame_id].then do |_|
+        attach_child_frame(@frames[parent_frame_id], parent_frame_id, frame_id, session)
+        @frames_pending_attachment.delete(frame_id)&.fulfill(nil)
+      end
+      return
+    end
+
+    raise FrameNotFoundError.new("Parent frame #{parent_frame_id} not found.")
+  end
+
+  class FrameNotFoundError < StandardError ; end
+
+  private def attach_child_frame(parent_frame, parent_frame_id, frame_id, session)
+    unless parent_frame
+      raise FrameNotFoundError.new("Parent frame #{parent_frame_id} not found.")
+    end
+
+    frame = Puppeteer::Frame.new(self, parent_frame, frame_id, session)
+    @frames[frame.id] = frame
     emit_event(FrameManagerEmittedEvents::FrameAttached, frame)
+    frame
   end
 
   # @param frame_payload [Hash]
   def handle_frame_navigated(frame_payload)
+    frame_id = frame_payload['id']
     is_main_frame = !frame_payload['parentId']
-    frame =
-      if is_main_frame
-        @main_frame
-      else
-        @frames[frame_payload['id']]
+
+
+    if @frames_pending_attachment[frame_id]
+      @frames_pending_attachment[frame_id].then do |_|
+        frame = is_main_frame ? @main_frame : @frames[frame_id]
+        reattach_frame(frame, frame_id, is_main_frame, frame_payload)
       end
+    else
+      frame = is_main_frame ? @main_frame : @frames[frame_id]
+      reattach_frame(frame, frame_id, is_main_frame, frame_payload)
+    end
+  end
 
+  private def reattach_frame(frame, frame_id, is_main_frame, frame_payload)
     if !is_main_frame && !frame
-      raise ArgumentError.new('We either navigate top level or have old version of the navigated frame')
+      raise "Missing frame isMainFrame=#{is_main_frame}, frameId=#{frame_id}"
     end
 
     # Detach all child frames first.
@@ -304,12 +335,12 @@ class Puppeteer::FrameManager
       if frame
         # Update frame id to retain frame identity on cross-process navigation.
         @frames.delete(frame.id)
-        frame.id = frame_payload['id']
+        frame.id = frame_id
       else
         # Initial main frame navigation.
-        frame = Puppeteer::Frame.new(self, nil, frame_payload['id'], @client)
+        frame = Puppeteer::Frame.new(self, nil, frame_id, @client)
       end
-      @frames[frame_payload['id']] = frame
+      @frames[frame_id] = frame
       @main_frame = frame
     end
 

data/lib/puppeteer/launcher/chrome.rb

@@ -81,6 +81,7 @@ module Puppeteer::Launcher
           )
 
           Puppeteer::Browser.create(
+            product: product,
             connection: connection,
             context_ids: [],
             ignore_https_errors: @browser_options.ignore_https_errors?,
@@ -126,7 +127,8 @@ module Puppeteer::Launcher
           '--disable-extensions',
           # TODO: remove AvoidUnnecessaryBeforeUnloadCheckSync below
           # once crbug.com/1324138 is fixed and released.
-          '--disable-features=Translate,BackForwardCache,AvoidUnnecessaryBeforeUnloadCheckSync',
+          # AcceptCHFrame disabled because of crbug.com/1348106.
+          '--disable-features=Translate,BackForwardCache,AcceptCHFrame,AvoidUnnecessaryBeforeUnloadCheckSync',
           '--disable-hang-monitor',
           '--disable-ipc-flooding-protection',
           '--disable-popup-blocking',
@@ -181,61 +183,6 @@ module Puppeteer::Launcher
       DefaultArgs.new(ChromeArgOptions.new(options || {}))
     end
 
-    # @return [Puppeteer::Browser]
-    def connect(options = {})
-      @browser_options = BrowserOptions.new(options)
-      browser_ws_endpoint = options[:browser_ws_endpoint]
-      browser_url = options[:browser_url]
-      transport = options[:transport]
-
-      connection =
-        if browser_ws_endpoint && browser_url.nil? && transport.nil?
-          connect_with_browser_ws_endpoint(browser_ws_endpoint)
-        elsif browser_ws_endpoint.nil? && browser_url && transport.nil?
-          connect_with_browser_url(browser_url)
-        elsif browser_ws_endpoint.nil? && browser_url.nil? && transport
-          connect_with_transport(transport)
-        else
-          raise ArgumentError.new("Exactly one of browserWSEndpoint, browserURL or transport must be passed to puppeteer.connect")
-        end
-
-      result = connection.send_message('Target.getBrowserContexts')
-      browser_context_ids = result['browserContextIds']
-
-      Puppeteer::Browser.create(
-        connection: connection,
-        context_ids: browser_context_ids,
-        ignore_https_errors: @browser_options.ignore_https_errors?,
-        default_viewport: @browser_options.default_viewport,
-        process: nil,
-        close_callback: -> { connection.send_message('Browser.close') },
-        target_filter_callback: @browser_options.target_filter,
-        is_page_target_callback: @browser_options.is_page_target,
-      )
-    end
-
-    # @return [Puppeteer::Connection]
-    private def connect_with_browser_ws_endpoint(browser_ws_endpoint)
-      transport = Puppeteer::WebSocketTransport.create(browser_ws_endpoint)
-      Puppeteer::Connection.new(browser_ws_endpoint, transport, @browser_options.slow_mo)
-    end
-
-    # @return [Puppeteer::Connection]
-    private def connect_with_browser_url(browser_url)
-      require 'net/http'
-      uri = URI(browser_url)
-      uri.path = '/json/version'
-      response_body = Net::HTTP.get(uri)
-      json = JSON.parse(response_body)
-      connection_url = json['webSocketDebuggerUrl']
-      connect_with_browser_ws_endpoint(connection_url)
-    end
-
-    # @return [Puppeteer::Connection]
-    private def connect_with_transport(transport)
-      Puppeteer::Connection.new('', transport, @browser_options.slow_mo)
-    end
-
     # @return {string}
     def executable_path(channel: nil)
       if channel

data/lib/puppeteer/launcher/firefox.rb

@@ -79,6 +79,7 @@ module Puppeteer::Launcher
           )
 
           Puppeteer::Browser.create(
+            product: product,
             connection: connection,
             context_ids: [],
             ignore_https_errors: @browser_options.ignore_https_errors?,
@@ -106,61 +107,6 @@ module Puppeteer::Launcher
       browser
     end
 
-    # @return [Puppeteer::Browser]
-    def connect(options = {})
-      @browser_options = BrowserOptions.new(options)
-      browser_ws_endpoint = options[:browser_ws_endpoint]
-      browser_url = options[:browser_url]
-      transport = options[:transport]
-
-      connection =
-        if browser_ws_endpoint && browser_url.nil? && transport.nil?
-          connect_with_browser_ws_endpoint(browser_ws_endpoint)
-        elsif browser_ws_endpoint.nil? && browser_url && transport.nil?
-          connect_with_browser_url(browser_url)
-        elsif browser_ws_endpoint.nil? && browser_url.nil? && transport
-          connect_with_transport(transport)
-        else
-          raise ArgumentError.new("Exactly one of browserWSEndpoint, browserURL or transport must be passed to puppeteer.connect")
-        end
-
-      result = connection.send_message('Target.getBrowserContexts')
-      browser_context_ids = result['browserContextIds']
-
-      Puppeteer::Browser.create(
-        connection: connection,
-        context_ids: browser_context_ids,
-        ignore_https_errors: @browser_options.ignore_https_errors?,
-        default_viewport: @browser_options.default_viewport,
-        process: nil,
-        close_callback: -> { connection.send_message('Browser.close') },
-        target_filter_callback: nil,
-        is_page_target_callback: nil,
-      )
-    end
-
-    # @return [Puppeteer::Connection]
-    private def connect_with_browser_ws_endpoint(browser_ws_endpoint)
-      transport = Puppeteer::WebSocketTransport.create(browser_ws_endpoint)
-      Puppeteer::Connection.new(browser_ws_endpoint, transport, @browser_options.slow_mo)
-    end
-
-    # @return [Puppeteer::Connection]
-    private def connect_with_browser_url(browser_url)
-      require 'net/http'
-      uri = URI(browser_url)
-      uri.path = '/json/version'
-      response_body = Net::HTTP.get(uri)
-      json = JSON.parse(response_body)
-      connection_url = json['webSocketDebuggerUrl']
-      connect_with_browser_ws_endpoint(connection_url)
-    end
-
-    # @return [Puppeteer::Connection]
-    private def connect_with_transport(transport)
-      Puppeteer::Connection.new('', transport, @browser_options.slow_mo)
-    end
-
     # @return {string}
     def executable_path(channel: nil)
       if channel

data/lib/puppeteer/lifecycle_watcher.rb

@@ -65,6 +65,7 @@ class Puppeteer::LifecycleWatcher
     @expected_lifecycle = ExpectedLifecycle.new(wait_until)
     @frame_manager = frame_manager
     @frame = frame
+    @initial_loader_id = frame.loader_id
     @timeout = timeout
 
     @listener_ids = {}
@@ -162,7 +163,7 @@ class Puppeteer::LifecycleWatcher
     if @has_same_document_navigation && @same_document_navigation_promise.pending?
       @same_document_navigation_promise.fulfill(true)
     end
-    if (@swapped || @new_document_navigation) && @new_document_navigation_promise.pending?
+    if (@swapped || @frame.loader_id != @initial_loader_id) && @new_document_navigation_promise.pending?
       @new_document_navigation_promise.fulfill(true)
     end
   end