puppeteer-ruby 0.50.1 → 0.52.0

data/lib/puppeteer/extension.rb

@@ -0,0 +1,70 @@
+# rbs_inline: enabled
+
+class Puppeteer::Extension
+  # @rbs id: String -- Extension id
+  # @rbs version: String -- Extension version
+  # @rbs name: String -- Extension name
+  # @rbs path: String -- Extension path
+  # @rbs enabled: bool -- Whether extension is enabled
+  # @rbs browser: Puppeteer::Browser -- Browser instance
+  # @rbs return: void -- No return value
+  def initialize(id:, version:, name:, path:, enabled:, browser:)
+    @id = id
+    @version = version
+    @name = name
+    @path = path
+    @enabled = enabled
+    @browser = browser
+  end
+
+  # @rbs return: String -- Extension id
+  attr_reader :id
+
+  # @rbs return: String -- Extension version
+  attr_reader :version
+
+  # @rbs return: String -- Extension name
+  attr_reader :name
+
+  # @rbs return: String -- Extension path
+  attr_reader :path
+
+  # @rbs return: bool -- Whether extension is enabled
+  attr_reader :enabled
+
+  # @rbs return: Array[Puppeteer::CdpWebWorker] -- Extension workers
+  def workers
+    extension_prefix = "chrome-extension://#{@id}"
+    extension_targets = @browser.targets.select do |target|
+      target.type == 'service_worker' && target.url.start_with?(extension_prefix)
+    end
+    extension_targets.filter_map do |target|
+      target.worker
+    rescue
+      nil
+    end
+  end
+
+  # @rbs return: Array[Puppeteer::Page] -- Extension pages
+  def pages
+    extension_prefix = "chrome-extension://#{@id}"
+    extension_targets = @browser.targets.select do |target|
+      target_url = target.url
+      ['page', 'background_page'].include?(target.type) && target_url.start_with?(extension_prefix)
+    end
+    extension_targets.filter_map do |target|
+      target.as_page
+    rescue
+      nil
+    end
+  end
+
+  # @rbs page: Puppeteer::Page -- Target page
+  # @rbs return: void -- No return value
+  def trigger_action(page)
+    page.browser.send(:connection).send_message('Extensions.triggerAction', {
+      id: @id,
+      targetId: page._tab_id,
+    })
+  end
+end

data/lib/puppeteer/frame.rb

@@ -19,6 +19,7 @@ class Puppeteer::Frame
     @url = 'about:blank'
     @lifecycle_events = Set.new
     @child_frames = Set.new
+    @extension_worlds = {}
     if parent_frame
       parent_frame._child_frames << self
     end
@@ -71,7 +72,7 @@ class Puppeteer::Frame
     @client != @frame_manager.client
   end
 
-  attr_accessor :frame_manager, :id, :loader_id, :lifecycle_events, :main_world, :puppeteer_world
+  attr_accessor :frame_manager, :id, :loader_id, :lifecycle_events, :main_world, :puppeteer_world, :extension_worlds
   attr_reader :client
 
   # @rbs other: Object -- Other object to compare
@@ -229,6 +230,11 @@ class Puppeteer::Frame
     @url
   end
 
+  # @rbs return: Array[untyped] -- Extension execution realms for this frame
+  def extension_realms
+    @extension_worlds.values
+  end
+
   # @rbs return: Puppeteer::Frame? -- Parent frame
   def parent_frame
     @parent_frame
@@ -425,6 +431,7 @@ class Puppeteer::Frame
     @detached = true
     @main_world.detach
     @puppeteer_world.detach
+    @extension_worlds.each_value(&:detach)
     if @parent_frame
       @parent_frame._child_frames.delete(self)
     end

data/lib/puppeteer/frame_manager.rb

@@ -5,6 +5,7 @@ class Puppeteer::FrameManager
   using Puppeteer::DefineAsyncMethod
 
   UTILITY_WORLD_NAME = '__puppeteer_utility_world__'
+  CHROME_EXTENSION_PREFIX = 'chrome-extension://'
 
   # @param {!Puppeteer.CDPSession} client
   # @param {!Puppeteer.Page} page
@@ -89,6 +90,9 @@ class Puppeteer::FrameManager
         handle_lifecycle_event(event)
       end
     end
+    client.on_event('Audits.issueAdded') do |event|
+      @page.emit_event(PageEmittedEvents::Issue, Puppeteer::Issue.new(event['issue']))
+    end
   end
 
   attr_reader :client, :timeout_settings
@@ -109,7 +113,9 @@ class Puppeteer::FrameManager
     Puppeteer::AsyncUtils.await_promise_all(
       client.async_send_message('Page.setLifecycleEventsEnabled', enabled: true),
       client.async_send_message('Runtime.enable'),
+      @page.browser.issues_enabled? ? client.async_send_message('Audits.enable') : nil,
     )
+    maybe_setup_block_list(client)
     ensure_isolated_world(client, UTILITY_WORLD_NAME)
     @network_manager.init unless cdp_session
   rescue => err
@@ -224,7 +230,11 @@ class Puppeteer::FrameManager
     session = target.session
     frame&.send(:update_client, session)
     setup_listeners(session)
-    async_init(target.target_info.target_id, session)
+    Async do
+      async_init(target.target_info.target_id, session).wait
+    rescue => err
+      debug_puts(err)
+    end
   end
 
   # @param event [Hash]
@@ -493,6 +503,7 @@ class Puppeteer::FrameManager
   # @pram session [Puppeteer::CDPSession]
   def handle_execution_context_created(context_payload, session)
     frame = if_present(context_payload.dig('auxData', 'frameId')) { |frame_id| @frames[frame_id] }
+    origin = context_payload['origin']
 
     world = nil
     if frame
@@ -508,6 +519,17 @@ class Puppeteer::FrameManager
         # connections so we might end up creating multiple isolated worlds.
         # We can use either.
         world = frame.puppeteer_world
+      elsif extension_origin?(origin)
+        extension_id = extract_extension_id(origin)
+        if extension_id
+          world = frame.extension_worlds[extension_id]
+          unless world
+            world = Puppeteer::IsolaatedWorld.new(frame._client || @client, self, frame, @timeout_settings)
+            frame.extension_worlds[extension_id] = world
+          end
+          world.origin = origin
+          world.world_id = extension_id
+        end
       end
     end
 
@@ -523,6 +545,19 @@ class Puppeteer::FrameManager
     @context_id_to_context[key] = context
   end
 
+  private def extension_origin?(origin)
+    origin.is_a?(String) && origin.start_with?(CHROME_EXTENSION_PREFIX)
+  end
+
+  private def extract_extension_id(origin)
+    return nil unless extension_origin?(origin)
+
+    path_part = origin[CHROME_EXTENSION_PREFIX.length..]
+    return nil unless path_part
+    slash_index = path_part.index('/')
+    slash_index ? path_part[0...slash_index] : path_part
+  end
+
   # @param execution_context_id [Integer]
   # @param session [Puppeteer::CDPSEssion]
   def handle_execution_context_destroyed(execution_context_id, session)
@@ -537,7 +572,7 @@ class Puppeteer::FrameManager
   def handle_execution_contexts_cleared(session)
     session_id = session.id
     @context_id_to_context.select! do |execution_context_id, context|
-      key_session_id, context_id = execution_context_id.split(':', 2)
+      key_session_id, _context_id = execution_context_id.split(':', 2)
       # Make sure to only clear execution contexts that belong
       # to the current session.
       if key_session_id != session_id
@@ -554,6 +589,31 @@ class Puppeteer::FrameManager
     @context_id_to_context[key] or raise "INTERNAL ERROR: missing context with id = #{context_id}"
   end
 
+  private def maybe_setup_block_list(client)
+    block_list = @page.browser.block_list
+    return if block_list.nil? || block_list.empty?
+
+    client.send_message('Network.enable')
+    matched_network_conditions = block_list.map do |pattern|
+      {
+        urlPattern: pattern,
+        latency: 0,
+        downloadThroughput: -1,
+        uploadThroughput: -1,
+      }
+    end
+    client.send_message('Network.emulateNetworkConditionsByRule', {
+      matchedNetworkConditions: matched_network_conditions,
+      offline: true,
+    })
+  rescue Puppeteer::Connection::ProtocolError => err
+    if err.message.include?('Method not available') || err.message.include?("wasn't found")
+      client.send_message('Network.setBlockedURLs', urls: block_list)
+    else
+      raise
+    end
+  end
+
   # @param {!Frame} frame
   private def remove_frame_recursively(frame)
     frame.child_frames.each do |child|

data/lib/puppeteer/http_request.rb

@@ -1,5 +1,7 @@
 # rbs_inline: enabled
 
+require 'base64'
+
 class Puppeteer::HTTPRequest
   include Puppeteer::DebugPrint
   include Puppeteer::IfPresent
@@ -101,7 +103,7 @@ class Puppeteer::HTTPRequest
     resource_type = event['type'] || event['resourceType'] || 'other'
     @resource_type = resource_type.downcase
     @method = event['request']['method']
-    @post_data = event['request']['postData']
+    @post_data = parse_post_data(event)
     has_post_data = event.dig('request', 'hasPostData')
     @has_post_data = has_post_data.nil? ? !@post_data.nil? : has_post_data
     @frame = frame
@@ -160,6 +162,18 @@ class Puppeteer::HTTPRequest
     end
   end
 
+  private def parse_post_data(event)
+    post_data_entries = event.dig('request', 'postDataEntries')
+    if post_data_entries && !post_data_entries.empty?
+      post_data_entries
+        .filter_map { |entry| entry['bytes'] ? Base64.decode64(entry['bytes']) : nil }
+        .join
+        .force_encoding('UTF-8')
+    else
+      event.dig('request', 'postData')
+    end
+  end
+
   private def assert_interception_not_handled
     if @interception_handled
       raise AlreadyHandledError.new

data/lib/puppeteer/http_response.rb

@@ -113,7 +113,12 @@ class Puppeteer::HTTPResponse
   # @param text [String]
   # @rbs return: String -- Response body as text
   def text
-    buffer
+    content = buffer
+    content = content.dup.force_encoding('UTF-8')
+    unless content.valid_encoding?
+      raise Puppeteer::Error.new('Could not decode response body as UTF-8')
+    end
+    content
   end
 
   # @param json [Hash]

data/lib/puppeteer/isolated_world.rb

@@ -62,11 +62,13 @@ class Puppeteer::IsolaatedWorld
     @ctx_bindings = Set.new
     @detached = false
     @context = nil
+    @origin = nil
+    @world_id = nil
 
     @client.on_event('Runtime.bindingCalled', &method(:handle_binding_called))
   end
 
-  attr_reader :frame, :task_manager
+  attr_reader :frame, :task_manager, :origin, :world_id
 
   # only used in Puppeteer::WaitTask#initialize
   private def _bound_functions
@@ -109,6 +111,25 @@ class Puppeteer::IsolaatedWorld
     @task_manager.terminate_all(Puppeteer::WaitTask::TerminatedError.new('waitForFunction failed: frame got detached.'))
   end
 
+  # @rbs origin: String -- Origin for this realm
+  # @rbs return: String -- Origin
+  def origin=(origin)
+    @origin = origin
+  end
+
+  # @rbs world_id: String -- World id for this realm
+  # @rbs return: String -- World id
+  def world_id=(world_id)
+    @world_id = world_id
+  end
+
+  # @rbs return: Puppeteer::Extension? -- Owning extension for this realm
+  def extension
+    return nil unless @world_id.is_a?(String)
+
+    frame.page.browser.extensions[@world_id]
+  end
+
   def detached?
     @detached
   end

data/lib/puppeteer/issue.rb

@@ -0,0 +1,16 @@
+# rbs_inline: enabled
+
+class Puppeteer::Issue
+  # @rbs issue: Hash[String, untyped] -- CDP issue payload
+  # @rbs return: void -- No return value
+  def initialize(issue)
+    @code = issue['code']
+    @details = issue['details']
+  end
+
+  # @rbs return: String -- Issue code
+  attr_reader :code
+
+  # @rbs return: Hash[String, untyped] -- Issue details payload
+  attr_reader :details
+end

data/lib/puppeteer/js_coverage.rb

@@ -26,6 +26,7 @@ class Puppeteer::JSCoverage
     @enabled = false
     @script_urls = {}
     @script_sources = {}
+    @script_parsed_tasks = []
   end
 
   def start(
@@ -52,9 +53,10 @@ class Puppeteer::JSCoverage
     @enabled = true
     @script_urls.clear
     @script_sources.clear
+    @script_parsed_tasks.clear
     @event_listeners = []
     @event_listeners << @client.add_event_listener('Debugger.scriptParsed') do |event|
-      Async do
+      @script_parsed_tasks << Async do
         Puppeteer::AsyncUtils.future_with_logging { on_script_parsed(event) }.call
       end
     end
@@ -95,11 +97,22 @@ class Puppeteer::JSCoverage
     response = @client.send_message('Debugger.getScriptSource', scriptId: event['scriptId'])
     @script_urls[event['scriptId']] = url
     @script_sources[event['scriptId']] = response['scriptSource']
+  rescue Puppeteer::Connection::ProtocolError
+    # The page can navigate while we are fetching sources for coverage.
+    # This matches upstream behavior that ignores these transient failures.
+    nil
+  end
+
+  private def drain_script_parsed_tasks
+    pending_tasks = @script_parsed_tasks
+    @script_parsed_tasks = []
+    pending_tasks.each(&:wait)
   end
 
   def stop
     raise 'JSCoverage is not enabled' unless @enabled
     @enabled = false
+    drain_script_parsed_tasks
 
     results = Puppeteer::AsyncUtils.await_promise_all(
       @client.async_send_message('Profiler.takePreciseCoverage'),

data/lib/puppeteer/launcher/browser_options.rb

@@ -32,7 +32,12 @@ module Puppeteer::Launcher
       @default_viewport = options.key?(:default_viewport) ? options[:default_viewport] : Puppeteer::Viewport.new(width: 800, height: 600)
       @slow_mo = options[:slow_mo] || 0
       @network_enabled = options.fetch(:network_enabled, true)
+      @issues_enabled = options.fetch(:issues_enabled, true)
       @protocol_timeout = options[:protocol_timeout]
+      @block_list = options[:block_list]
+      if @block_list && !@block_list.is_a?(Array)
+        raise ArgumentError.new('block_list must be an Array of URL patterns')
+      end
 
       # only for Puppeteer.connect
       @target_filter = options[:target_filter]
@@ -46,7 +51,7 @@ module Puppeteer::Launcher
       end
     end
 
-    attr_reader :default_viewport, :slow_mo, :target_filter, :is_page_target, :network_enabled, :protocol_timeout
+    attr_reader :default_viewport, :slow_mo, :target_filter, :is_page_target, :network_enabled, :issues_enabled, :protocol_timeout, :block_list
 
     def ignore_https_errors?
       @ignore_https_errors

data/lib/puppeteer/launcher/chrome.rb

@@ -87,6 +87,8 @@ module Puppeteer::Launcher
             ignore_https_errors: @browser_options.ignore_https_errors?,
             default_viewport: @browser_options.default_viewport,
             network_enabled: @browser_options.network_enabled,
+            issues_enabled: @browser_options.issues_enabled,
+            block_list: @browser_options.block_list,
             process: runner.proc,
             close_callback: -> { runner.close },
             target_filter_callback: nil,
@@ -129,9 +131,8 @@ module Puppeteer::Launcher
           '--disable-component-update',
           '--disable-default-apps',
           '--disable-dev-shm-usage',
-          '--disable-extensions',
           # AcceptCHFrame disabled because of crbug.com/1348106.
-          '--disable-features=Translate,BackForwardCache,AcceptCHFrame,MediaRouter,OptimizationHints',
+          '--disable-features=Translate,BackForwardCache,AcceptCHFrame,MediaRouter,OptimizationHints,IPH_ReadingModePageActionLabel,ReadAnythingOmniboxChip',
           '--disable-hang-monitor',
           '--disable-ipc-flooding-protection',
           '--disable-popup-blocking',
@@ -177,6 +178,20 @@ module Puppeteer::Launcher
           end
         end
 
+        if chrome_arg_options.enable_extensions
+          chrome_arguments << '--enable-unsafe-extension-debugging'
+          if chrome_arg_options.enable_extensions.is_a?(Array) && !chrome_arg_options.enable_extensions.empty?
+            extension_paths = chrome_arg_options.enable_extensions.map do |path|
+              File.expand_path(path)
+            end
+            joined_paths = extension_paths.join(',')
+            chrome_arguments << "--disable-extensions-except=#{joined_paths}"
+            chrome_arguments << "--load-extension=#{joined_paths}"
+          end
+        else
+          chrome_arguments << '--disable-extensions'
+        end
+
         if chrome_arg_options.args.all? { |arg| arg.start_with?('-') }
           chrome_arguments << 'about:blank'
         end

data/lib/puppeteer/launcher/chrome_arg_options.rb

@@ -31,13 +31,14 @@ module Puppeteer::Launcher
       @user_data_dir = options[:user_data_dir]
       @devtools = options[:devtools] || false
       @headless = options[:headless]
+      @enable_extensions = options[:enable_extensions] || false
       if @headless.nil?
         @headless = !@devtools
       end
       @debugging_port = options[:debugging_port] || 0
     end
 
-    attr_reader :args, :user_data_dir, :debugging_port
+    attr_reader :args, :user_data_dir, :debugging_port, :enable_extensions
 
     def headless?
       @headless

data/lib/puppeteer/locators.rb

@@ -213,15 +213,16 @@ class Puppeteer::Locator
   end
 
   # @rbs value: String -- Value to fill
+  # @rbs typing_threshold: Integer -- Minimum length to switch to direct assignment
   # @rbs return: void -- No return value
-  def fill(value)
+  def fill(value, typing_threshold: 100)
     perform_action('Locator.fill',
       conditions: [
         method(:ensure_element_is_in_viewport_if_needed),
         method(:wait_for_stable_bounding_box_if_needed),
         method(:wait_for_enabled_if_needed),
       ]) do |handle, _options|
-      fill_element(handle, value)
+      fill_element(handle, value, typing_threshold: typing_threshold)
     end
   end
 
@@ -422,7 +423,7 @@ class Puppeteer::Locator
     end
   end
 
-  private def fill_element(handle, value)
+  private def fill_element(handle, value, typing_threshold: 100)
     input_type = handle.evaluate(<<~JAVASCRIPT)
       el => {
         if (el instanceof HTMLSelectElement) {
@@ -461,52 +462,70 @@ class Puppeteer::Locator
     when 'select'
       handle.select(value)
     when 'contenteditable', 'typeable-input'
-      text_to_type = handle.evaluate(<<~JAVASCRIPT, value)
-        (input, newValue) => {
-          const currentValue = input.isContentEditable
-            ? input.innerText
-            : input.value;
+      if value.length < typing_threshold
+        text_to_type = handle.evaluate(<<~JAVASCRIPT, value)
+          (input, newValue) => {
+            const currentValue = input.isContentEditable
+              ? input.innerText
+              : input.value;
+
+            if (
+              newValue.length <= currentValue.length ||
+              !newValue.startsWith(input.value)
+            ) {
+              if (input.isContentEditable) {
+                input.innerText = '';
+              } else {
+                input.value = '';
+              }
+              return newValue;
+            }
+            const originalValue = input.isContentEditable
+              ? input.innerText
+              : input.value;
 
-          if (
-            newValue.length <= currentValue.length ||
-            !newValue.startsWith(input.value)
-          ) {
             if (input.isContentEditable) {
               input.innerText = '';
+              input.innerText = originalValue;
             } else {
               input.value = '';
+              input.value = originalValue;
             }
-            return newValue;
+            return newValue.substring(originalValue.length);
           }
-          const originalValue = input.isContentEditable
-            ? input.innerText
-            : input.value;
-
-          if (input.isContentEditable) {
-            input.innerText = '';
-            input.innerText = originalValue;
-          } else {
-            input.value = '';
-            input.value = originalValue;
-          }
-          return newValue.substring(originalValue.length);
-        }
-      JAVASCRIPT
-      text_to_type = text_to_type.to_s
-      handle.type_text(text_to_type)
+        JAVASCRIPT
+        text_to_type = text_to_type.to_s
+        handle.type_text(text_to_type) unless text_to_type.empty?
+      else
+        fill_directly(handle, value)
+      end
     when 'other-input'
-      handle.focus
-      handle.evaluate(<<~JAVASCRIPT, value)
-        (input, newValue) => {
-          input.value = newValue;
-          input.dispatchEvent(new Event('input', { bubbles: true }));
-          input.dispatchEvent(new Event('change', { bubbles: true }));
-        }
-      JAVASCRIPT
+      fill_directly(handle, value)
     else
       raise Puppeteer::Error.new('Element cannot be filled out.')
     end
   end
+
+  private def fill_directly(handle, value)
+    handle.focus
+    handle.evaluate(<<~JAVASCRIPT, value)
+      (input, newValue) => {
+        const currentValue = input.isContentEditable
+          ? input.innerText
+          : input.value;
+        if (currentValue === newValue) {
+          return;
+        }
+        if (input.isContentEditable) {
+          input.innerText = newValue;
+        } else {
+          input.value = newValue;
+        }
+        input.dispatchEvent(new Event('input', { bubbles: true }));
+        input.dispatchEvent(new Event('change', { bubbles: true }));
+      }
+    JAVASCRIPT
+  end
 end
 
 class Puppeteer::FunctionLocator < Puppeteer::Locator

data/lib/puppeteer/network_manager.rb

@@ -54,6 +54,10 @@ class Puppeteer::NetworkManager
       }
     end
 
+    def active?
+      @offline || @latency != 0 || @download != -1 || @upload != -1
+    end
+
     def refresh
       update_network_conditions
     end
@@ -102,7 +106,6 @@ class Puppeteer::NetworkManager
     @user_cache_disabled = nil
     @internal_network_condition = InternalNetworkCondition.new(method(:send_to_clients))
     @interception_semaphore = Async::Semaphore.new(1)
-
     add_client(@client)
   end
 
@@ -138,10 +141,8 @@ class Puppeteer::NetworkManager
     "#<Puppeteer::HTTPRequest #{values.join(' ')}>"
   end
 
-  private def apply_to_clients
-    @clients.each do |client|
-      yield client
-    end
+  private def apply_to_clients(&block)
+    @clients.each(&block)
   end
 
   private def ignore_client_error?(error)
@@ -192,7 +193,7 @@ class Puppeteer::NetworkManager
     if @protocol_request_interception_enabled
       safe_send_message(client, 'Fetch.enable',
         handleAuthRequests: true,
-        patterns: [{ urlPattern: '*' }],
+        patterns: [{ urlPattern: '*' }]
       )
     else
       safe_send_message(client, 'Fetch.disable')
@@ -208,7 +209,9 @@ class Puppeteer::NetworkManager
     apply_user_agent(client)
     apply_protocol_cache_disabled(client)
     apply_protocol_request_interception(client)
-    safe_send_message(client, 'Network.emulateNetworkConditions', @internal_network_condition.params)
+    if @internal_network_condition.active?
+      safe_send_message(client, 'Network.emulateNetworkConditions', @internal_network_condition.params)
+    end
   end
 
   # @param username [String|NilClass]
@@ -343,11 +346,9 @@ class Puppeteer::NetworkManager
   private def dispatch_intercepted_request(event, fetch_request_id, client:)
     if Async::Task.current?
       Async do
-        begin
-          handle_request(event, fetch_request_id, client: client)
-        rescue => err
-          debug_puts(err)
-        end
+        handle_request(event, fetch_request_id, client: client)
+      rescue => err
+        debug_puts(err)
       end
     else
       handle_request(event, fetch_request_id, client: client)
@@ -365,6 +366,7 @@ class Puppeteer::NetworkManager
     if existing_request &&
        existing_request.url == event_url &&
        existing_request.method == event.dig('request', 'method')
+
       if_present(@network_event_manager.request_extra_info(network_request_id).shift) do |extra_info|
         existing_request.update_headers(extra_info['headers'])
       end
@@ -474,12 +476,10 @@ class Puppeteer::NetworkManager
     end
   end
 
-  private def with_interception_lock
+  private def with_interception_lock(&block)
     return yield unless Async::Task.current?
 
-    @interception_semaphore.acquire do
-      yield
-    end
+    @interception_semaphore.acquire(&block)
   end
 
   private def handle_request_without_network_instrumentation(event, client)