puppeteer-ruby 0.52.0 → 0.53.0

data/lib/puppeteer/emulation_manager.rb

@@ -1,4 +1,5 @@
 class Puppeteer::EmulationManager
+  include Puppeteer::DebugPrint
   using Puppeteer::DefineAsyncMethod
 
   # @param {!Puppeteer.CDPSession} client
@@ -6,11 +7,59 @@ class Puppeteer::EmulationManager
     @client = client
     @emulating_mobile = false
     @has_touch = false
+    @viewport = nil
+    @locale = nil
+    @locale_configured = false
+    @secondary_clients = Set.new
+  end
+
+  def update_client(client)
+    @client = client
+    @secondary_clients.delete(client)
+  end
+
+  def register_speculative_session(client)
+    @secondary_clients << client
+    client.once(CDPSessionEmittedEvents::Disconnected) do
+      @secondary_clients.delete(client)
+    end
+    promises = []
+    promises.concat(viewport_promises(client, @viewport)) if @viewport
+    if @locale_configured
+      promises << client.async_send_message(
+        'Emulation.setLocaleOverride',
+        { locale: @locale }.compact,
+      )
+    end
+    return if promises.empty?
+
+    Async do
+      Puppeteer::AsyncUtils.await_promise_all(*promises)
+    rescue => err
+      debug_puts(err)
+    end
   end
 
   # @param viewport [Puppeteer::Viewport]
   # @return [true|false]
   def emulate_viewport(viewport)
+    mobile = viewport.mobile?
+    has_touch = viewport.has_touch?
+
+    apply_viewport(@client, viewport)
+    @viewport = viewport
+
+    reload_needed = @emulating_mobile != mobile || @has_touch != has_touch
+    @emulating_mobile = mobile
+    @has_touch = has_touch
+    reload_needed
+  end
+
+  private def apply_viewport(client, viewport)
+    Puppeteer::AsyncUtils.await_promise_all(*viewport_promises(client, viewport))
+  end
+
+  private def viewport_promises(client, viewport)
     mobile = viewport.mobile?
     width = viewport.width
     height = viewport.height
@@ -23,24 +72,36 @@ class Puppeteer::EmulationManager
       end
     has_touch = viewport.has_touch?
 
-    Puppeteer::AsyncUtils.await_promise_all(
-      @client.async_send_message('Emulation.setDeviceMetricsOverride',
+    [
+      client.async_send_message('Emulation.setDeviceMetricsOverride',
         mobile: mobile,
         width: width,
         height: height,
         deviceScaleFactor: device_scale_factor,
         screenOrientation: screen_orientation,
       ),
-      @client.async_send_message('Emulation.setTouchEmulationEnabled',
+      client.async_send_message('Emulation.setTouchEmulationEnabled',
         enabled: has_touch,
       ),
-    )
-
-    reload_needed = @emulating_mobile != mobile || @has_touch != has_touch
-    @emulating_mobile = mobile
-    @has_touch = has_touch
-    reload_needed
+    ]
   end
 
   define_async_method :async_emulate_viewport
+
+  # @rbs locale: String? -- Locale to emulate, or nil to disable emulation
+  # @rbs return: void -- No return value
+  def emulate_locale(locale)
+    @locale = locale
+    @locale_configured = true
+    clients = [@client, *@secondary_clients]
+    promises = clients.map do |client|
+      client.async_send_message(
+        'Emulation.setLocaleOverride',
+        { locale: locale }.compact,
+      )
+    end
+    Puppeteer::AsyncUtils.await_promise_all(*promises)
+  end
+
+  define_async_method :async_emulate_locale
 end

data/lib/puppeteer/events.rb

@@ -35,6 +35,7 @@ module CDPSessionEmittedEvents ; end
 {
   Disconnected: EventsDefinitionUtils.symbol('CDPSession.Disconnected'),
   Ready: EventsDefinitionUtils.symbol('CDPSession.Ready'),
+  Swapped: EventsDefinitionUtils.symbol('CDPSession.Swapped'),
 }.define_const_into(CDPSessionEmittedEvents)
 
 # All the events a Browser may emit.
@@ -98,6 +99,7 @@ module FrameManagerEmittedEvents ; end
   FrameNavigated: EventsDefinitionUtils.symbol('FrameManager.FrameNavigated'),
   FrameDetached: EventsDefinitionUtils.symbol('FrameManager.FrameDetached'),
   FrameSwapped: EventsDefinitionUtils.symbol('FrameManager.FrameSwapped'),
+  FrameSwappedByActivation: EventsDefinitionUtils.symbol('FrameManager.FrameSwappedByActivation'),
   LifecycleEvent: EventsDefinitionUtils.symbol('FrameManager.LifecycleEvent'),
   FrameNavigatedWithinDocument: EventsDefinitionUtils.symbol('FrameManager.FrameNavigatedWithinDocument'),
   ExecutionContextCreated: EventsDefinitionUtils.symbol('FrameManager.ExecutionContextCreated'),

data/lib/puppeteer/execution_context.rb

@@ -2,6 +2,7 @@
 
 class Puppeteer::ExecutionContext
   include Puppeteer::IfPresent
+  include Puppeteer::EventCallbackable
   using Puppeteer::DefineAsyncMethod
 
   EVALUATION_SCRIPT_URL = 'pprt://__puppeteer_evaluation_script__'
@@ -19,6 +20,11 @@ class Puppeteer::ExecutionContext
 
   attr_reader :client, :world
 
+  # @rbs return: void -- Notify consumers that this context was destroyed
+  def dispose
+    emit_event('disposed')
+  end
+
   # only used in IsolaatedWorld
   private def _context_id
     @context_id
@@ -353,6 +359,7 @@ class Puppeteer::ExecutionContext
     message = error.message
     if message.end_with?('Cannot find context with specified id') ||
        message.end_with?('Inspected target navigated or closed')
+
       return Puppeteer::Error.new('Execution context was destroyed, most likely because of a navigation.')
     end
 

data/lib/puppeteer/extension.rb

@@ -38,11 +38,16 @@ class Puppeteer::Extension
     extension_targets = @browser.targets.select do |target|
       target.type == 'service_worker' && target.url.start_with?(extension_prefix)
     end
-    extension_targets.filter_map do |target|
-      target.worker
-    rescue
-      nil
+    tasks = extension_targets.map do |target|
+      proc do
+        target.worker
+      rescue => error
+        raise unless can_ignore_error?(error)
+
+        nil
+      end
     end
+    Puppeteer::AsyncUtils.await_promise_all(*tasks).compact
   end
 
   # @rbs return: Array[Puppeteer::Page] -- Extension pages
@@ -54,15 +59,24 @@ class Puppeteer::Extension
     end
     extension_targets.filter_map do |target|
       target.as_page
-    rescue
+    rescue => error
+      raise unless can_ignore_error?(error)
+
       nil
     end
   end
 
+  # @rbs error: StandardError -- Error raised while resolving an extension target
+  # @rbs return: bool -- Whether the target resolution error can be ignored
+  private def can_ignore_error?(error)
+    error.is_a?(Puppeteer::TargetCloseError) ||
+      error.message.include?('No target with given id found')
+  end
+
   # @rbs page: Puppeteer::Page -- Target page
   # @rbs return: void -- No return value
   def trigger_action(page)
-    page.browser.send(:connection).send_message('Extensions.triggerAction', {
+    @browser.send(:connection).send_message('Extensions.triggerAction', {
       id: @id,
       targetId: page._tab_id,
     })

data/lib/puppeteer/frame.rb

@@ -52,6 +52,11 @@ class Puppeteer::Frame
     @frame_manager.page
   end
 
+  # @rbs return: Puppeteer::Accessibility -- Accessibility tree for this frame
+  def accessibility
+    @accessibility ||= Puppeteer::Accessibility.new(self)
+  end
+
   # @rbs return: Numeric -- Default timeout in milliseconds
   def default_timeout
     @frame_manager.timeout_settings.timeout
@@ -284,11 +289,10 @@ class Puppeteer::Frame
   # @rbs selector: String -- CSS selector
   # @rbs delay: Numeric? -- Delay between down and up (ms)
   # @rbs button: String? -- Mouse button
-  # @rbs click_count: Integer? -- Deprecated: use count (click_count only sets clickCount)
   # @rbs count: Integer? -- Number of clicks to perform
   # @rbs return: void -- No return value
-  def click(selector, delay: nil, button: nil, click_count: nil, count: nil)
-    @puppeteer_world.click(selector, delay: delay, button: button, click_count: click_count, count: count)
+  def click(selector, delay: nil, button: nil, count: nil)
+    @puppeteer_world.click(selector, delay: delay, button: button, count: count)
   end
 
   define_async_method :async_click

data/lib/puppeteer/frame_manager.rb

@@ -40,6 +40,7 @@ class Puppeteer::FrameManager
     @frame_tree_mutex = Mutex.new
 
     setup_listeners(@client)
+    setup_client_disconnect_listener(@client)
   end
 
   private def setup_listeners(client)
@@ -97,6 +98,69 @@ class Puppeteer::FrameManager
 
   attr_reader :client, :timeout_settings
 
+  # When the main frame is replaced by another main frame, preserve the
+  # existing Frame object while replacing its ID, client, and frame tree.
+  def swap_frame_tree(client)
+    @client = client
+    frame = @main_frame
+    if frame
+      target_id = client.target.target_id
+      @frame_naviigated_received << target_id
+      @frames.delete(frame.id)
+      frame.id = target_id
+      @frames[target_id] = frame
+      frame.send(:update_client, client)
+    end
+
+    setup_listeners(client)
+    setup_client_disconnect_listener(client)
+    init(client.target.target_id, client)
+    @network_manager.add_client(client)
+    emit_event(FrameManagerEmittedEvents::FrameSwappedByActivation, frame) if frame
+  end
+
+  def register_speculative_session(client)
+    @network_manager.add_client(client)
+  end
+
+  private def setup_client_disconnect_listener(client)
+    client.once(CDPSessionEmittedEvents::Disconnected) do
+      Async do
+        handle_client_disconnect(client)
+      rescue => err
+        debug_puts(err)
+      end
+    end
+  end
+
+  # A disconnected primary client can mean either that the page closed or
+  # that a prerendered target is about to replace it. Wait for one of those
+  # events instead of guessing with a timer.
+  private def handle_client_disconnect(client)
+    frame = @main_frame
+    return unless frame
+    return unless @client == client
+
+    unless @page.browser.connected? && !@page.closed?
+      remove_frame_recursively(frame)
+      return
+    end
+
+    frame.child_frames.each { |child| remove_frame_recursively(child) }
+    swapped = Async::Promise.new
+    swap_listener_id = add_event_listener(FrameManagerEmittedEvents::FrameSwappedByActivation) do |swapped_frame|
+      swapped.resolve(true) if swapped_frame == frame && !swapped.resolved?
+    end
+    close_listener_id = @page.add_event_listener(PageEmittedEvents::Close) do
+      swapped.resolve(false) unless swapped.resolved?
+    end
+
+    remove_frame_recursively(frame) unless swapped.wait
+  ensure
+    remove_event_listener(swap_listener_id) if swap_listener_id
+    @page.remove_event_listener(close_listener_id) if close_listener_id
+  end
+
   private def init(target_id, cdp_session = nil)
     @frames_pending_target_init[target_id] ||= Async::Promise.new
     client = cdp_session || @client
@@ -115,7 +179,6 @@ class Puppeteer::FrameManager
       client.async_send_message('Runtime.enable'),
       @page.browser.issues_enabled? ? client.async_send_message('Audits.enable') : nil,
     )
-    maybe_setup_block_list(client)
     ensure_isolated_world(client, UTILITY_WORLD_NAME)
     @network_manager.init unless cdp_session
   rescue => err
@@ -143,6 +206,11 @@ class Puppeteer::FrameManager
   # @return [Puppeteer::HTTPResponse]
   def navigate_frame(frame, url, referer: nil, referrer_policy: nil, timeout: nil, wait_until: nil)
     assert_no_legacy_navigation_options(wait_until: wait_until)
+    unless @page.browser.url_allowed?(url)
+      raise Puppeteer::Error.new(
+        "Navigation to #{url} is blocked by blocklist/allowlist rules",
+      )
+    end
 
     referrer_policy ||= @network_manager.extra_http_headers['referer-policy']
     protocol_referrer_policy = referrer_policy_to_protocol(referrer_policy)
@@ -565,6 +633,7 @@ class Puppeteer::FrameManager
     context = @context_id_to_context[key]
     return unless context
     @context_id_to_context.delete(key)
+    context.dispose
     context.world&.delete_context(context)
   end
 
@@ -578,6 +647,7 @@ class Puppeteer::FrameManager
       if key_session_id != session_id
         true # keep
       else
+        context.dispose
         context.world&.delete_context(context)
         false # remove
       end
@@ -589,31 +659,6 @@ class Puppeteer::FrameManager
     @context_id_to_context[key] or raise "INTERNAL ERROR: missing context with id = #{context_id}"
   end
 
-  private def maybe_setup_block_list(client)
-    block_list = @page.browser.block_list
-    return if block_list.nil? || block_list.empty?
-
-    client.send_message('Network.enable')
-    matched_network_conditions = block_list.map do |pattern|
-      {
-        urlPattern: pattern,
-        latency: 0,
-        downloadThroughput: -1,
-        uploadThroughput: -1,
-      }
-    end
-    client.send_message('Network.emulateNetworkConditionsByRule', {
-      matchedNetworkConditions: matched_network_conditions,
-      offline: true,
-    })
-  rescue Puppeteer::Connection::ProtocolError => err
-    if err.message.include?('Method not available') || err.message.include?("wasn't found")
-      client.send_message('Network.setBlockedURLs', urls: block_list)
-    else
-      raise
-    end
-  end
-
   # @param {!Frame} frame
   private def remove_frame_recursively(frame)
     frame.child_frames.each do |child|

data/lib/puppeteer/http_response.rb

@@ -54,7 +54,8 @@ class Puppeteer::HTTPResponse
     @headers = {}
     headers = extra_info ? extra_info['headers'] : response_payload['headers']
     headers.each do |key, value|
-      @headers[key.downcase] = value
+      header_name = key.downcase
+      @headers[header_name] = normalize_header_value(header_name, value)
     end
     @security_details = if_present(response_payload['securityDetails']) do |security_payload|
       SecurityDetails.new(security_payload)
@@ -68,6 +69,19 @@ class Puppeteer::HTTPResponse
 
   attr_reader :remote_address, :url, :status, :status_text, :headers, :security_details, :request, :timing
 
+  # Multiline values represent duplicate fields in CDP. RFC 9110 allows
+  # comma-combining fields except Set-Cookie, whose lines must stay separate.
+  private def normalize_header_value(name, value)
+    return value unless value.include?("\n")
+
+    separator = name == 'set-cookie' ? "\n " : ', '
+    value
+      .split("\n")
+      .map(&:strip)
+      .reject(&:empty?)
+      .join(separator)
+  end
+
   def inspect
     values = %i[remote_address url status status_text headers security_details request].map do |sym|
       value = instance_variable_get(:"@#{sym}")

data/lib/puppeteer/isolated_world.rb

@@ -68,7 +68,7 @@ class Puppeteer::IsolaatedWorld
     @client.on_event('Runtime.bindingCalled', &method(:handle_binding_called))
   end
 
-  attr_reader :frame, :task_manager, :origin, :world_id
+  attr_reader :frame, :task_manager, :origin, :world_id, :context
 
   # only used in Puppeteer::WaitTask#initialize
   private def _bound_functions
@@ -246,6 +246,13 @@ class Puppeteer::IsolaatedWorld
   # @param wait_until [String|Array<String>]
   def set_content(html, timeout: nil, wait_until: nil)
     option_wait_until = [wait_until || 'load'].flatten
+    unsupported_events = option_wait_until & ['networkidle0', 'networkidle2']
+    unless unsupported_events.empty?
+      raise ArgumentError.new(
+        "set_content does not support #{unsupported_events.join(', ')}; " \
+        'compose set_content with wait_for_network_idle instead.',
+      )
+    end
     option_timeout = timeout || @timeout_settings.navigation_timeout
 
     # We rely upon the fact that document.open() will reset frame lifecycle with "init"
@@ -398,11 +405,10 @@ class Puppeteer::IsolaatedWorld
   # @param selector [String]
   # @param delay [Number]
   # @param button [String] "left"|"right"|"middle"
-  # @param click_count [Number] Deprecated: use count (click_count only sets clickCount)
   # @param count [Number]
-  def click(selector, delay: nil, button: nil, click_count: nil, count: nil)
+  def click(selector, delay: nil, button: nil, count: nil)
     handle = query_selector(selector) or raise ElementNotFoundError.new(selector)
-    handle.click(delay: delay, button: button, click_count: click_count, count: count)
+    handle.click(delay: delay, button: button, count: count)
     handle.dispose
   end
 

data/lib/puppeteer/js_coverage.rb

@@ -29,6 +29,10 @@ class Puppeteer::JSCoverage
     @script_parsed_tasks = []
   end
 
+  def update_client(client)
+    @client = client
+  end
+
   def start(
         reset_on_navigation: nil,
         report_anonymous_scripts: nil,

data/lib/puppeteer/keyboard.rb

@@ -16,6 +16,12 @@ class Puppeteer::Keyboard
 
   attr_reader :modifiers
 
+  # @rbs client: Puppeteer::CDPSession -- Replacement CDP session
+  # @rbs return: void -- No return value
+  def update_client(client)
+    @client = client
+  end
+
   # @rbs key: String -- Key name
   # @rbs text: String? -- Text to input
   # @rbs commands: Array[String]? -- Editing commands

data/lib/puppeteer/launcher/browser_options.rb

@@ -38,6 +38,18 @@ module Puppeteer::Launcher
       if @block_list && !@block_list.is_a?(Array)
         raise ArgumentError.new('block_list must be an Array of URL patterns')
       end
+      @allow_list = options[:allow_list]
+      if @allow_list && !@allow_list.is_a?(Array)
+        raise ArgumentError.new('allow_list must be an Array of URL patterns')
+      end
+      if @block_list && @allow_list
+        raise ArgumentError.new('Cannot specify both blocklist and allowlist')
+      end
+      [*@block_list, *@allow_list].each do |pattern|
+        URLPattern::URLPattern.new(pattern)
+      rescue URLPattern::Error => error
+        raise ArgumentError.new("Invalid URLPattern #{pattern.inspect}: #{error.message}")
+      end
 
       # only for Puppeteer.connect
       @target_filter = options[:target_filter]
@@ -51,7 +63,7 @@ module Puppeteer::Launcher
       end
     end
 
-    attr_reader :default_viewport, :slow_mo, :target_filter, :is_page_target, :network_enabled, :issues_enabled, :protocol_timeout, :block_list
+    attr_reader :default_viewport, :slow_mo, :target_filter, :is_page_target, :network_enabled, :issues_enabled, :protocol_timeout, :block_list, :allow_list
 
     def ignore_https_errors?
       @ignore_https_errors

data/lib/puppeteer/launcher/chrome.rb

@@ -3,6 +3,24 @@ require 'tmpdir'
 # https://github.com/puppeteer/puppeteer/blob/main/src/node/Launcher.ts
 module Puppeteer::Launcher
   class Chrome
+    def self.get_features(flag, options = [])
+      prefix = flag.end_with?('=') ? flag : "#{flag}="
+      options
+        .select { |option| option.start_with?(prefix) }
+        .flat_map do |option|
+          option[(option.index('=') + 1)..]
+            .strip
+            .split(',')
+            .map(&:strip)
+        end
+        .reject(&:empty?)
+    end
+
+    def self.remove_matching_flags(array, flag)
+      prefix = flag.end_with?('=') ? flag : "#{flag}="
+      array.delete_if { |option| option.start_with?(prefix) }
+    end
+
     def initialize(project_root:, preferred_revision:, is_puppeteer_core:)
       @project_root = project_root
       @preferred_revision = preferred_revision
@@ -89,6 +107,7 @@ module Puppeteer::Launcher
             network_enabled: @browser_options.network_enabled,
             issues_enabled: @browser_options.issues_enabled,
             block_list: @browser_options.block_list,
+            allow_list: @browser_options.allow_list,
             process: runner.proc,
             close_callback: -> { runner.close },
             target_filter_callback: nil,
@@ -120,6 +139,33 @@ module Puppeteer::Launcher
       # @param options [Launcher::ChromeArgOptions]
       def initialize(chrome_arg_options)
         # See https://github.com/GoogleChrome/chrome-launcher/blob/main/docs/chrome-flags-for-tools.md
+        user_disabled_features = Chrome.get_features('--disable-features', chrome_arg_options.args)
+        if user_disabled_features.any?
+          Chrome.remove_matching_flags(chrome_arg_options.args, '--disable-features')
+        end
+
+        user_enabled_features = Chrome.get_features('--enable-features', chrome_arg_options.args)
+        if user_enabled_features.any?
+          Chrome.remove_matching_flags(chrome_arg_options.args, '--enable-features')
+        end
+
+        enabled_features = [
+          'NetworkServiceInProcess2',
+          *user_enabled_features,
+        ].reject(&:empty?)
+
+        disabled_features = [
+          'Translate',
+          'BackForwardCache',
+          'AcceptCHFrame',
+          'MediaRouter',
+          'OptimizationHints',
+          'IPH_ReadingModePageActionLabel',
+          'ReadAnythingOmniboxChip',
+          'WebUIReloadButton',
+          *user_disabled_features,
+        ].reject(&:empty?).reject { |feature| enabled_features.include?(feature) }
+
         chrome_arguments = [
           '--allow-pre-commit-input',
           '--disable-background-networking',
@@ -131,8 +177,7 @@ module Puppeteer::Launcher
           '--disable-component-update',
           '--disable-default-apps',
           '--disable-dev-shm-usage',
-          # AcceptCHFrame disabled because of crbug.com/1348106.
-          '--disable-features=Translate,BackForwardCache,AcceptCHFrame,MediaRouter,OptimizationHints,IPH_ReadingModePageActionLabel,ReadAnythingOmniboxChip',
+          "--disable-features=#{disabled_features.join(',')}",
           '--disable-hang-monitor',
           '--disable-ipc-flooding-protection',
           '--disable-popup-blocking',
@@ -143,7 +188,7 @@ module Puppeteer::Launcher
           # TODO(sadym): remove '--enable-blink-features=IdleDetection' once
           # IdleDetection is turned on by default.
           '--enable-blink-features=IdleDetection',
-          '--enable-features=NetworkServiceInProcess2',
+          "--enable-features=#{enabled_features.join(',')}",
           '--export-tagged-pdf',
           '--force-color-profile=srgb',
           '--metrics-recording-only',
@@ -179,7 +224,6 @@ module Puppeteer::Launcher
         end
 
         if chrome_arg_options.enable_extensions
-          chrome_arguments << '--enable-unsafe-extension-debugging'
           if chrome_arg_options.enable_extensions.is_a?(Array) && !chrome_arg_options.enable_extensions.empty?
             extension_paths = chrome_arg_options.enable_extensions.map do |path|
               File.expand_path(path)

data/lib/puppeteer/lifecycle_watcher.rb

@@ -69,9 +69,6 @@ class Puppeteer::LifecycleWatcher
     @timeout = timeout
 
     @listener_ids = {}
-    @listener_ids['client'] = @frame_manager.client.add_event_listener(CDPSessionEmittedEvents::Disconnected) do
-      terminate(TerminatedError.new('Navigation failed because browser has disconnected!'))
-    end
     connection = @frame_manager.client.respond_to?(:connection) ? @frame_manager.client.connection : nil
     if connection
       @listener_ids['connection'] = connection.add_event_listener(ConnectionEmittedEvents::Disconnected) do
@@ -85,6 +82,7 @@ class Puppeteer::LifecycleWatcher
       @frame_manager.add_event_listener(FrameManagerEmittedEvents::FrameNavigatedWithinDocument, &method(:navigated_within_document)),
       @frame_manager.add_event_listener(FrameManagerEmittedEvents::FrameNavigated, &method(:navigated)),
       @frame_manager.add_event_listener(FrameManagerEmittedEvents::FrameSwapped, &method(:handle_frame_swapped)),
+      @frame_manager.add_event_listener(FrameManagerEmittedEvents::FrameSwappedByActivation, &method(:handle_frame_swapped)),
       @frame_manager.add_event_listener(FrameManagerEmittedEvents::FrameDetached, &method(:handle_frame_detached)),
     ]
     @listener_ids['network_manager'] = [
@@ -209,9 +207,6 @@ class Puppeteer::LifecycleWatcher
   end
 
   def dispose
-    if_present(@listener_ids['client']) do |id|
-      @frame_manager.client.remove_event_listener(id)
-    end
     if_present(@listener_ids['frame_manager']) do |ids|
       @frame_manager.remove_event_listener(*ids)
     end