puppeteer-ruby 0.40.5 → 0.41.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f6c1abfbf66f645ac208cf5f3f0ad24c4ae421e576c5b1516803cdc897bad22
-  data.tar.gz: a40b422147eff9261720975d8009ecc3fbd24a1f0d2d828f1fe3b5dce31f5c24
+  metadata.gz: ccfb160c40e2ef032c0da17ab4b1cc9fe610d4caf42f236042ea0757c9e270e7
+  data.tar.gz: 95a3a457aa850116bc3062e9507ef53bd10e1d451ebeb4703cf9a6f5d663b9c5
 SHA512:
-  metadata.gz: f532e2ff58c283c0904d5c76c0e9254d5b91be1f6ff7c576b728d91a727b0bb246f9d0eefe1461b1a7ba28607f33ac7a541d9d9a9ee3da0e5742c330f3c01b6e
-  data.tar.gz: c0dab5bca1e3ae23c6c2f3866cb3127024a9fd9663e01ff20d7edd68ef60060fc2203ffe81d80f322a10428ea4632193e1eef8b502e3375ea0c8413a3172dc61
+  metadata.gz: b65da9f89cc8cfeea05e4e0835f919aab382f47e380d27c1e0a439c70cb0b0b7d9e8519a3361ad545e0c5115216d698740e46f20c96960cf96496ee22ac374ea
+  data.tar.gz: 81c846c795efd81bc10af84b65e02a6dfc3a52d9be2cc5f21cc7bc313e84fcd4ef3fed692bc1bc616389bc43f2a06a6944c2353aa8f8c3a614bb8a4c8ef2097c

data/CHANGELOG.md

@@ -1,7 +1,19 @@
-### main [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.5...main)]
+### main [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.41.0...main)]
 
 - xxx
 
+### 0.41.0 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.7...0.41.0)]
+
+- Port Puppeteer v14.0-v15.2 features, including `ElementHandle#wait_for_xpath`
+
+### 0.40.7 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.6...0.40.7)]
+
+- Port Puppeteer v13.6-v13.7 features.
+
+### 0.40.6 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.5...0.40.6)]
+
+- Port Puppeteer v13.1-v13.5 features mainly for request interception.
+
 ### 0.40.5 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.4...0.40.5)]
 
 Bugfix:

data/README.md

@@ -229,6 +229,13 @@ end
 
 https://yusukeiwaki.github.io/puppeteer-ruby-docs/
 
+## Limitations
+
+### Not compatible with Firefox >= v97.0
+
+:sos: Help and contribution wanted! :sos:
+https://github.com/YusukeIwaki/puppeteer-ruby/issues/220
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/YusukeIwaki/puppeteer-ruby.

data/docs/api_coverage.md

@@ -1,6 +1,6 @@
 # API coverages
-- Puppeteer version: v13.5.1
-- puppeteer-ruby version: 0.40.4
+- Puppeteer version: v15.2.0
+- puppeteer-ruby version: 0.41.0
 
 ## Puppeteer
 
@@ -297,6 +297,7 @@
 * type => `#type_text`
 * uploadFile => `#upload_file`
 * waitForSelector => `#wait_for_selector`
+* waitForXPath => `#wait_for_xpath`
 
 ## HTTPRequest
 
@@ -310,8 +311,8 @@
 * frame
 * headers
 * initiator
-* ~~interceptResolutionState~~
-* ~~isInterceptResolutionHandled~~
+* interceptResolutionState => `#intercept_resolution_state`
+* isInterceptResolutionHandled => `#intercept_resolution_handled?`
 * isNavigationRequest => `#navigation_request?`
 * method
 * postData => `#post_data`

data/lib/puppeteer/browser.rb

@@ -13,7 +13,14 @@ class Puppeteer::Browser
   # @param {?Puppeteer.Viewport} defaultViewport
   # @param process [Puppeteer::BrowserRunner::BrowserProcess|NilClass]
   # @param {function()=} closeCallback
-  def self.create(connection:, context_ids:, ignore_https_errors:, default_viewport:, process:, close_callback:)
+  def self.create(connection:,
+                  context_ids:,
+                  ignore_https_errors:,
+                  default_viewport:,
+                  process:,
+                  close_callback:,
+                  target_filter_callback:,
+                  is_page_target_callback:)
     browser = Puppeteer::Browser.new(
       connection: connection,
       context_ids: context_ids,
@@ -21,6 +28,8 @@ class Puppeteer::Browser
       default_viewport: default_viewport,
       process: process,
       close_callback: close_callback,
+      target_filter_callback: target_filter_callback,
+      is_page_target_callback: is_page_target_callback,
     )
     connection.send_message('Target.setDiscoverTargets', discover: true)
     browser
@@ -32,12 +41,21 @@ class Puppeteer::Browser
   # @param {?Puppeteer.Viewport} defaultViewport
   # @param {?Puppeteer.ChildProcess} process
   # @param {(function():Promise)=} closeCallback
-  def initialize(connection:, context_ids:, ignore_https_errors:, default_viewport:, process:, close_callback:)
+  def initialize(connection:,
+                 context_ids:,
+                 ignore_https_errors:,
+                 default_viewport:,
+                 process:,
+                 close_callback:,
+                 target_filter_callback:,
+                 is_page_target_callback:)
     @ignore_https_errors = ignore_https_errors
     @default_viewport = default_viewport
     @process = process
     @connection = connection
     @close_callback = close_callback
+    @target_filter_callback = target_filter_callback || method(:default_target_filter_callback)
+    @is_page_target_callback = is_page_target_callback || method(:default_is_page_target_callback)
 
     @default_context = Puppeteer::BrowserContext.new(@connection, self, nil)
     @contexts = {}
@@ -54,6 +72,16 @@ class Puppeteer::Browser
     @connection.on_event('Target.targetInfoChanged', &method(:handle_target_info_changed))
   end
 
+  private def default_target_filter_callback(target_info)
+    true
+  end
+
+  private def default_is_page_target_callback(target_info)
+    ['page', 'background_page', 'webview'].include?(target_info.type)
+  end
+
+  attr_reader :is_page_target_callback
+
   # @param event_name [Symbol] either of :disconnected, :targetcreated, :targetchanged, :targetdestroyed
   def on(event_name, &block)
     unless BrowserEmittedEvents.values.include?(event_name.to_s)
@@ -119,12 +147,16 @@ class Puppeteer::Browser
     if @targets[target_info.target_id]
       raise TargetAlreadyExistError.new
     end
+
+    return unless @target_filter_callback.call(target_info)
+
     target = Puppeteer::Target.new(
       target_info: target_info,
       browser_context: context,
       session_factory: -> { @connection.create_session(target_info) },
       ignore_https_errors: @ignore_https_errors,
       default_viewport: @default_viewport,
+      is_page_target_callback: @is_page_target_callback,
     )
     @targets[target_info.target_id] = target
     if_present(@wait_for_creating_targets.delete(target_info.target_id)) do |promise|

data/lib/puppeteer/browser_context.rb

@@ -50,7 +50,9 @@ class Puppeteer::BrowserContext
 
   # @return {!Promise<!Array<!Puppeteer.Page>>}
   def pages
-    targets.select { |target| target.type == 'page' }.map(&:page).reject { |page| !page }
+    targets.select { |target|
+      target.type == 'page' || (target.type == 'other' && @browser.is_page_target_callback&.call(target.target_info))
+    }.map(&:page).reject { |page| !page }
   end
 
   def incognito?

data/lib/puppeteer/browser_runner.rb

@@ -128,7 +128,7 @@ class Puppeteer::BrowserRunner
   def close
     return if @closed
 
-    if @using_temp_user_data_dir && !@for_firefox
+    if @using_temp_user_data_dir
       kill
     elsif @connection
       begin

data/lib/puppeteer/connection.rb

@@ -191,6 +191,7 @@ class Puppeteer::Connection
       'Network.responseReceived',
       'Network.responseReceivedExtraInfo',
       'Page.lifecycleEvent',
+      'Target.receivedMessageFromTarget', # only Firefox
     ]
 
     def handle_message(message)

data/lib/puppeteer/element_handle.rb

@@ -76,6 +76,81 @@ class Puppeteer::ElementHandle < Puppeteer::JSHandle
 
   define_async_method :async_wait_for_selector
 
+  # Wait for the `xpath` within the element. If at the moment of calling the
+  # method the `xpath` already exists, the method will return immediately. If
+  # the `xpath` doesn't appear after the `timeout` milliseconds of waiting, the
+  # function will throw.
+  #
+  # If `xpath` starts with `//` instead of `.//`, the dot will be appended automatically.
+  #
+  # This method works across navigation
+  # ```js
+  # const puppeteer = require('puppeteer');
+  # (async () => {
+  # const browser = await puppeteer.launch();
+  # const page = await browser.newPage();
+  # let currentURL;
+  # page
+  # .waitForXPath('//img')
+  # .then(() => console.log('First URL with image: ' + currentURL));
+  # for (currentURL of [
+  # 'https://example.com',
+  # 'https://google.com',
+  # 'https://bbc.com',
+  # ]) {
+  # await page.goto(currentURL);
+  # }
+  # await browser.close();
+  # })();
+  # ```
+  # @param xpath - A
+  # {@link https://developer.mozilla.org/en-US/docs/Web/XPath | xpath} of an
+  # element to wait for
+  # @param options - Optional waiting parameters
+  # @returns Promise which resolves when element specified by xpath string is
+  # added to DOM. Resolves to `null` if waiting for `hidden: true` and xpath is
+  # not found in DOM.
+  # @remarks
+  # The optional Argument `options` have properties:
+  #
+  # - `visible`: A boolean to wait for element to be present in DOM and to be
+  # visible, i.e. to not have `display: none` or `visibility: hidden` CSS
+  # properties. Defaults to `false`.
+  #
+  # - `hidden`: A boolean wait for element to not be found in the DOM or to be
+  # hidden, i.e. have `display: none` or `visibility: hidden` CSS properties.
+  # Defaults to `false`.
+  #
+  # - `timeout`: A number which is maximum time to wait for in milliseconds.
+  # Defaults to `30000` (30 seconds). Pass `0` to disable timeout. The default
+  # value can be changed by using the {@link Page.setDefaultTimeout} method.
+  def wait_for_xpath(xpath, visible: nil, hidden: nil, timeout: nil)
+    frame = @context.frame
+
+    secondary_world = frame.secondary_world
+    adopted_root = secondary_world.execution_context.adopt_element_handle(self)
+    param_xpath =
+      if xpath.start_with?('//')
+        ".#{xpath}"
+      else
+        xpath
+      end
+    unless param_xpath.start_with?('.//')
+      adopted_root.dispose
+      raise ArgumentError.new("Unsupported xpath expression: #{xpath}")
+    end
+    handle = secondary_world.wait_for_xpath(param_xpath, visible: visible, hidden: hidden, timeout: timeout, root: adopted_root)
+    adopted_root.dispose
+    return nil unless handle
+
+    main_world = frame.main_world
+    result = main_world.execution_context.adopt_element_handle(handle)
+    handle.dispose
+    result
+  end
+
+  define_async_method :async_wait_for_xpath
+
   def as_element
     self
   end
@@ -309,22 +384,35 @@ class Puppeteer::ElementHandle < Puppeteer::JSHandle
     end
 
     fn = <<~JAVASCRIPT
-    (element, values) => {
+    (element, vals) => {
+      const values = new Set(vals);
       if (element.nodeName.toLowerCase() !== 'select') {
         throw new Error('Element is not a <select> element.');
       }
 
-      const options = Array.from(element.options);
-      element.value = undefined;
-      for (const option of options) {
-        option.selected = values.includes(option.value);
-        if (option.selected && !element.multiple) {
-          break;
+      const selectedValues = new Set();
+      if (!element.multiple) {
+        for (const option of element.options) {
+          option.selected = false;
+        }
+        for (const option of element.options) {
+          if (values.has(option.value)) {
+            option.selected = true;
+            selectedValues.add(option.value);
+            break;
+          }
+        }
+      } else {
+        for (const option of element.options) {
+          option.selected = values.has(option.value);
+          if (option.selected) {
+            selectedValues.add(option.value);
+          }
         }
       }
       element.dispatchEvent(new Event('input', { bubbles: true }));
       element.dispatchEvent(new Event('change', { bubbles: true }));
-      return options.filter(option => option.selected).map(option => option.value);
+      return [...selectedValues.values()];
     }
     JAVASCRIPT
     evaluate(fn, values)
@@ -337,10 +425,6 @@ class Puppeteer::ElementHandle < Puppeteer::JSHandle
       raise ArgumentError.new('Multiple file uploads only work with <input type=file multiple>')
     end
 
-    if error_path = file_paths.find { |file_path| !File.exist?(file_path) }
-      raise ArgumentError.new("#{error_path} does not exist or is not readable")
-    end
-
     backend_node_id = @remote_object.node_info(@client)["node"]["backendNodeId"]
 
     # The zero-length array is a special case, it seems that DOM.setFileInputFiles does
@@ -421,7 +505,15 @@ class Puppeteer::ElementHandle < Puppeteer::JSHandle
     end
   end
 
-  def screenshot(type: nil, path: nil, full_page: nil, clip: nil, quality: nil, omit_background: nil, encoding: nil)
+  def screenshot(type: nil,
+                 path: nil,
+                 full_page: nil,
+                 clip: nil,
+                 quality: nil,
+                 omit_background: nil,
+                 encoding: nil,
+                 capture_beyond_viewport: nil,
+                 from_surface: nil)
     needs_viewport_reset = false
 
     box = bounding_box
@@ -465,7 +557,17 @@ class Puppeteer::ElementHandle < Puppeteer::JSHandle
       }
     end
 
-    @page.screenshot(type: type, path: path, full_page: full_page, clip: clip, quality: quality, omit_background: omit_background, encoding: encoding)
+    @page.screenshot(
+      type: type,
+      path: path,
+      full_page:
+      full_page,
+      clip: clip,
+      quality: quality,
+      omit_background: omit_background,
+      encoding: encoding,
+      capture_beyond_viewport: capture_beyond_viewport,
+      from_surface: from_surface)
   ensure
     if needs_viewport_reset
       @page.viewport = viewport

data/lib/puppeteer/execution_context.rb

@@ -2,7 +2,7 @@ class Puppeteer::ExecutionContext
   include Puppeteer::IfPresent
   using Puppeteer::DefineAsyncMethod
 
-  EVALUATION_SCRIPT_URL = '__puppeteer_evaluation_script__'
+  EVALUATION_SCRIPT_URL = 'pprt://__puppeteer_evaluation_script__'
   SOURCE_URL_REGEX = /^[\040\t]*\/\/[@#] sourceURL=\s*(\S*?)\s*$/m
 
   # @param client [Puppeteer::CDPSession]

data/lib/puppeteer/frame.rb

@@ -10,6 +10,7 @@ class Puppeteer::Frame
     @parent_frame = parent_frame
     @id = frame_id
     @detached = false
+    @has_started_loading = false
 
     @loader_id = ''
     @lifecycle_events = Set.new
@@ -46,6 +47,10 @@ class Puppeteer::Frame
 
   attr_accessor :frame_manager, :id, :loader_id, :lifecycle_events, :main_world, :secondary_world
 
+  def has_started_loading?
+    @has_started_loading
+  end
+
   # @param url [String]
   # @param rederer [String]
   # @param timeout [number|nil]
@@ -316,6 +321,10 @@ class Puppeteer::Frame
     @lifecycle_events << name
   end
 
+  def handle_loading_started
+    @has_started_loading = true
+  end
+
   def handle_loading_stopped
     @lifecycle_events << 'DOMContentLoaded'
     @lifecycle_events << 'load'

data/lib/puppeteer/frame_manager.rb

@@ -43,6 +43,9 @@ class Puppeteer::FrameManager
     client.on_event('Page.frameDetached') do |event|
       handle_frame_detached(event['frameId'], event['reason'])
     end
+    client.on_event('Page.frameStartedLoading') do |event|
+      handle_frame_started_loading(event['frameId'])
+    end
     client.on_event('Page.frameStoppedLoading') do |event|
       handle_frame_stopped_loading(event['frameId'])
     end
@@ -71,11 +74,17 @@ class Puppeteer::FrameManager
   private def init(cdp_session = nil)
     client = cdp_session || @client
 
-    results = await_all(
+    promises = [
       client.async_send_message('Page.enable'),
       client.async_send_message('Page.getFrameTree'),
-    )
-    frame_tree = results.last['frameTree']
+      cdp_session&.async_send_message('Target.setAutoAttach', {
+        autoAttach: true,
+        waitForDebuggerOnStart: false,
+        flatten: true,
+      }),
+    ].compact
+    results = await_all(*promises)
+    frame_tree = results[1]['frameTree']
     handle_frame_tree(client, frame_tree)
     await_all(
       client.async_send_message('Page.setLifecycleEventsEnabled', enabled: true),
@@ -112,13 +121,12 @@ class Puppeteer::FrameManager
     option_timeout = timeout || @timeout_settings.navigation_timeout
 
     watcher = Puppeteer::LifecycleWatcher.new(self, frame, option_wait_until, option_timeout)
-    ensure_new_document_navigation = false
 
     begin
       navigate = future do
         result = @client.send_message('Page.navigate', navigate_params)
         loader_id = result['loaderId']
-        ensure_new_document_navigation = !!loader_id
+
         if result['errorText']
           raise NavigationError.new("#{result['errorText']} at #{url}")
         end
@@ -128,14 +136,9 @@ class Puppeteer::FrameManager
         watcher.timeout_or_termination_promise,
       )
 
-      document_navigation_promise =
-        if ensure_new_document_navigation
-          watcher.new_document_navigation_promise
-        else
-          watcher.same_document_navigation_promise
-        end
       await_any(
-        document_navigation_promise,
+        watcher.new_document_navigation_promise,
+        watcher.same_document_navigation_promise,
         watcher.timeout_or_termination_promise,
       )
     rescue Puppeteer::TimeoutError => err
@@ -201,7 +204,14 @@ class Puppeteer::FrameManager
     emit_event(FrameManagerEmittedEvents::LifecycleEvent, frame)
   end
 
-  # @param {string} frameId
+  # @param frame_id [String]
+  def handle_frame_started_loading(frame_id)
+    frame = @frames[frame_id]
+    return if !frame
+    frame.handle_loading_started
+  end
+
+  # @param frame_id [String]
   def handle_frame_stopped_loading(frame_id)
     frame = @frames[frame_id]
     return if !frame

data/lib/puppeteer/http_request.rb

@@ -2,6 +2,8 @@ class Puppeteer::HTTPRequest
   include Puppeteer::DebugPrint
   include Puppeteer::IfPresent
 
+  DEFAULT_INTERCEPT_RESOLUTION_PRIORITY = 0
+
   # defines some methods used only in NetworkManager, Response
   class InternalAccessor
     def initialize(request)
@@ -38,6 +40,43 @@ class Puppeteer::HTTPRequest
     end
   end
 
+  class InterceptResolutionState
+    def self.abort(priority: nil)
+      new(action: 'abort', priority: priority)
+    end
+
+    def self.respond(priority: nil)
+      new(action: 'respond', priority: priority)
+    end
+
+    def self.continue(priority: nil)
+      new(action: 'continue', priority: priority)
+    end
+
+    def self.disabled(priority: nil)
+      new(action: 'disabled', priority: priority)
+    end
+
+    def self.none(priority: nil)
+      new(action: 'none', priority: priority)
+    end
+
+    def self.already_handled(priority: nil)
+      new(action: 'already-handled', priority: priority)
+    end
+
+    private def initialize(action:, priority:)
+      @action = action
+      @priority = priority
+    end
+
+    def priority_unspecified?
+      @priority.nil?
+    end
+
+    attr_reader :action, :priority
+  end
+
   # @param client [Puppeteer::CDPSession]
   # @param frame [Puppeteer::Frame]
   # @param interception_id [string|nil]
@@ -57,9 +96,8 @@ class Puppeteer::HTTPRequest
     @frame = frame
     @redirect_chain = redirect_chain
     @continue_request_overrides = {}
-    @current_strategy = 'none'
-    @current_priority = nil
-    @intercept_actions = []
+    @intercept_resolution_state = InterceptResolutionState.none
+    @intercept_handlers = []
     @initiator = event['initiator']
 
     @headers = {}
@@ -115,19 +153,29 @@ class Puppeteer::HTTPRequest
     @abort_error_reason
   end
 
-  # @returns An array of the current intercept resolution strategy and priority
-  # `[strategy,priority]`. Strategy is one of: `abort`, `respond`, `continue`,
-  #  `disabled`, `none`, or `already-handled`.
-  def intercept_resolution
+  # @returns An InterceptResolutionState object describing the current resolution
+  #  action and priority.
+  #
+  #  InterceptResolutionState contains:
+  #    action: InterceptResolutionAction
+  #    priority?: number
+  #
+  #  InterceptResolutionAction is one of: `abort`, `respond`, `continue`,
+  #  `disabled`, `none`, or `alreay-handled`
+  def intercept_resolution_state
     if !@allow_interception
-      ['disabled']
+      InterceptResolutionState.disabled
     elsif @interception_handled
-      ['already-handled']
+      InterceptResolutionState.already_handled
     else
-      [@current_strategy, @current_priority]
+      @intercept_resolution_state.dup
     end
   end
 
+  def intercept_resolution_handled?
+    @interception_handled
+  end
+
   # Adds an async request handler to the processing queue.
   # Deferred handlers are not guaranteed to execute in any particular order,
   # but they are guarnateed to resolve before the request interception
@@ -135,19 +183,19 @@ class Puppeteer::HTTPRequest
   #
   # @param pending_handler [Proc]
   def enqueue_intercept_action(pending_handler)
-    @intercept_actions << pending_handler
+    @intercept_handlers << pending_handler
   end
 
   # Awaits pending interception handlers and then decides how to fulfill
   # the request interception.
   def finalize_interceptions
-    @intercept_actions.each(&:call)
-    case @intercept_resolution
-    when :abort
+    @intercept_handlers.each(&:call)
+    case intercept_resolution_state.action
+    when 'abort'
       abort_impl(**@abort_error_reason)
-    when :respond
+    when 'respond'
       respond_impl(**@response_for_request)
-    when :continue
+    when 'continue'
       continue_impl(@continue_request_overrides)
     end
   end
@@ -169,8 +217,14 @@ class Puppeteer::HTTPRequest
   private def headers_to_array(headers)
     return nil unless headers
 
-    headers.map do |key, value|
-      { name: key, value: value.to_s }
+    headers.flat_map do |key, value|
+      if value.is_a?(Enumerable)
+        value.map do |v|
+          { name: key, value: v.to_s }
+        end
+      else
+        { name: key, value: value.to_s }
+      end
     end
   end
 
@@ -220,17 +274,16 @@ class Puppeteer::HTTPRequest
     end
 
     @continue_request_overrides = overrides
-    if @current_priority.nil? || priority > @current_priority
-      @current_strategy = :continue
-      @current_priority = priority
+    if @intercept_resolution_state.priority_unspecified? || priority > @intercept_resolution_state.priority
+      @intercept_resolution_state = InterceptResolutionState.continue(priority: priority)
       return
     end
 
-    if priority == @current_priority
-      if @current_strategy == :abort || @current_strategy == :respond
+    if priority == @intercept_resolution_state.priority
+      if @intercept_resolution_state.action == :abort || @intercept_resolution_state.action == :respond
         return
       end
-      @current_strategy = :continue
+      @intercept_resolution_state = InterceptResolutionState.continue(priority: priority)
     end
   end
 
@@ -284,17 +337,16 @@ class Puppeteer::HTTPRequest
       body: body,
     }
 
-    if @current_priority.nil? || priority > @current_priority
-      @current_strategy = :respond
-      @current_priority = priority
+    if @intercept_resolution_state.priority_unspecified? || priority > @intercept_resolution_state.priority
+      @intercept_resolution_state = InterceptResolutionState.respond(priority: priority)
       return
     end
 
-    if priority == @current_priority
-      if @current_strategy == :abort
+    if priority == @intercept_resolution_state.priority
+      if @intercept_resolution_state.action == :abort
         return
       end
-      @current_strategy = :respond
+      @intercept_resolution_state = InterceptResolutionState.respond(priority: priority)
     end
   end
 
@@ -303,7 +355,12 @@ class Puppeteer::HTTPRequest
 
     mock_response_headers = {}
     headers&.each do |key, value|
-      mock_response_headers[key.downcase] = value
+      mock_response_headers[key.downcase] =
+        if value.is_a?(Enumerable)
+          value.map(&:to_s)
+        else
+          value.to_s
+        end
     end
     if content_type
       mock_response_headers['content-type'] = content_type
@@ -360,9 +417,8 @@ class Puppeteer::HTTPRequest
     end
     @abort_error_reason = error_reason
 
-    if @current_priority.nil? || priority > @current_priority
-      @current_strategy = :abort
-      @current_priority = priority
+    if @intercept_resolution_state.priority_unspecified? || priority > @intercept_resolution_state.priority
+      @intercept_resolution_state = InterceptResolutionState.abort(priority: priority)
     end
   end
 

data/lib/puppeteer/launcher/browser_options.rb

@@ -31,9 +31,20 @@ module Puppeteer::Launcher
       # `default_viewport: nil` must be respected here.
       @default_viewport = options.key?(:default_viewport) ? options[:default_viewport] : Puppeteer::Viewport.new(width: 800, height: 600)
       @slow_mo = options[:slow_mo] || 0
+
+      # only for Puppeteer.connect
+      @target_filter = options[:target_filter]
+      if @target_filter && !@target_filter.respond_to?(:call)
+        raise ArgumentError.new('target_filter must be a Proc (target_info => true/false)')
+      end
+
+      @is_page_target = options[:is_page_target]
+      if @is_page_target && !@is_page_target.respond_to?(:call)
+        raise ArgumentError.new('is_page_target must be a Proc (target_info => true/false)')
+      end
     end
 
-    attr_reader :default_viewport, :slow_mo
+    attr_reader :default_viewport, :slow_mo, :target_filter, :is_page_target
 
     def ignore_https_errors?
       @ignore_https_errors

data/lib/puppeteer/launcher/chrome.rb

@@ -87,6 +87,8 @@ module Puppeteer::Launcher
             default_viewport: @browser_options.default_viewport,
             process: runner.proc,
             close_callback: -> { runner.close },
+            target_filter_callback: nil,
+            is_page_target_callback: nil,
           )
         rescue
           runner.kill
@@ -122,7 +124,9 @@ module Puppeteer::Launcher
           '--disable-default-apps',
           '--disable-dev-shm-usage',
           '--disable-extensions',
-          '--disable-features=Translate',
+          # TODO: remove AvoidUnnecessaryBeforeUnloadCheckSync below
+          # once crbug.com/1324138 is fixed and released.
+          '--disable-features=Translate,BackForwardCache,AvoidUnnecessaryBeforeUnloadCheckSync',
           '--disable-hang-monitor',
           '--disable-ipc-flooding-protection',
           '--disable-popup-blocking',
@@ -205,6 +209,8 @@ module Puppeteer::Launcher
         default_viewport: @browser_options.default_viewport,
         process: nil,
         close_callback: -> { connection.send_message('Browser.close') },
+        target_filter_callback: @browser_options.target_filter,
+        is_page_target_callback: @browser_options.is_page_target,
       )
     end
 

data/lib/puppeteer/launcher/firefox.rb

@@ -85,6 +85,8 @@ module Puppeteer::Launcher
             default_viewport: @browser_options.default_viewport,
             process: runner.proc,
             close_callback: -> { runner.close },
+            target_filter_callback: nil,
+            is_page_target_callback: nil,
           )
         rescue
           runner.kill
@@ -132,6 +134,8 @@ module Puppeteer::Launcher
         default_viewport: @browser_options.default_viewport,
         process: nil,
         close_callback: -> { connection.send_message('Browser.close') },
+        target_filter_callback: nil,
+        is_page_target_callback: nil,
       )
     end
 

data/lib/puppeteer/lifecycle_watcher.rb

@@ -43,7 +43,7 @@ class Puppeteer::LifecycleWatcher
       if expected_lifecycle.any? { |event| !frame.lifecycle_events.include?(event) }
         return false
       end
-      if frame.child_frames.any? { |child| !completed?(child) }
+      if frame.child_frames.any? { |child| child.has_started_loading? && !completed?(child) }
         return false
       end
       true
@@ -65,7 +65,6 @@ class Puppeteer::LifecycleWatcher
     @expected_lifecycle = ExpectedLifecycle.new(wait_until)
     @frame_manager = frame_manager
     @frame = frame
-    @initial_loader_id = frame.loader_id
     @timeout = timeout
 
     @listener_ids = {}
@@ -77,6 +76,7 @@ class Puppeteer::LifecycleWatcher
         check_lifecycle_complete
       end,
       @frame_manager.add_event_listener(FrameManagerEmittedEvents::FrameNavigatedWithinDocument, &method(:navigated_within_document)),
+      @frame_manager.add_event_listener(FrameManagerEmittedEvents::FrameNavigated, &method(:navigated)),
       @frame_manager.add_event_listener(FrameManagerEmittedEvents::FrameSwapped, &method(:handle_frame_swapped)),
       @frame_manager.add_event_listener(FrameManagerEmittedEvents::FrameDetached, &method(:handle_frame_detached)),
     ]
@@ -143,6 +143,12 @@ class Puppeteer::LifecycleWatcher
     check_lifecycle_complete
   end
 
+  private def navigated(frame)
+    return if frame != @frame
+    @new_document_navigation = true
+    check_lifecycle_complete
+  end
+
   private def handle_frame_swapped(frame)
     return if frame != @frame
     @swapped = true
@@ -153,17 +159,10 @@ class Puppeteer::LifecycleWatcher
     # We expect navigation to commit.
     return unless @expected_lifecycle.completed?(@frame)
     @lifecycle_promise.fulfill(true) if @lifecycle_promise.pending?
-    if @frame.loader_id == @initial_loader_id && !@has_same_document_navigation
-      if @swapped
-        @swapped = false
-        @new_document_navigation_promise.fulfill(true)
-      end
-      return
-    end
     if @has_same_document_navigation && @same_document_navigation_promise.pending?
       @same_document_navigation_promise.fulfill(true)
     end
-    if @frame.loader_id != @initial_loader_id && @new_document_navigation_promise.pending?
+    if (@swapped || @new_document_navigation) && @new_document_navigation_promise.pending?
       @new_document_navigation_promise.fulfill(true)
     end
   end

data/lib/puppeteer/network_event_manager.rb

@@ -119,4 +119,8 @@ class Puppeteer::NetworkEventManager
   def get_queued_event_group(network_request_id)
     @queued_event_group_map[network_request_id]
   end
+
+  def forget_queued_event_group(network_request_id)
+    @queued_event_group_map.delete(network_request_id)
+  end
 end

data/lib/puppeteer/network_manager.rb

@@ -217,6 +217,7 @@ class Puppeteer::NetworkManager
       # CDP may have sent a Fetch.requestPaused event already. Check for it.
       if_present(@network_event_manager.get_request_paused(network_request_id)) do |request_paused_event|
         fetch_request_id = request_paused_event['requestId']
+        patch_request_event_headers(event, request_paused_event)
         handle_request(event, fetch_request_id)
         @network_event_manager.forget_request_paused(network_request_id)
       end
@@ -277,12 +278,19 @@ class Puppeteer::NetworkManager
     end
 
     if request_will_be_sent_event
+      patch_request_event_headers(request_will_be_sent_event, event)
       handle_request(request_will_be_sent_event, fetch_request_id)
     else
       @network_event_manager.store_request_paused(network_request_id, event)
     end
   end
 
+  private def patch_request_event_headers(request_will_be_sent_event, request_paused_event)
+    request_will_be_sent_event['request']['headers'].merge!(
+      # includes extra headers, like: Accept, Origin
+      request_paused_event['request']['headers'])
+  end
+
   private def handle_request(event, fetch_request_id)
     redirect_chain = []
     if event['redirectResponse']
@@ -387,6 +395,7 @@ class Puppeteer::NetworkManager
     # We may have skipped response and loading events because we didn't have
     # this ExtraInfo event yet. If so, emit those events now.
     if_present(@network_event_manager.get_queued_event_group(event['requestId'])) do |queued_events|
+      @network_event_manager.forget_queued_event_group(event['requestId'])
       emit_response_event(queued_events.response_received_event, event)
       if_present(queued_events.loading_finished_event) do |loading_finished_event|
         emit_loading_finished(loading_finished_event)

data/lib/puppeteer/page/screenshot_options.rb

@@ -2,15 +2,49 @@ require 'mime/types'
 
 class Puppeteer::Page
   # /**
-  #  * @typedef {Object} ScreenshotOptions
-  #  * @property {string=} type
-  #  * @property {string=} path
-  #  * @property {boolean=} fullPage
-  #  * @property {{x: number, y: number, width: number, height: number}=} clip
-  #  * @property {number=} quality
-  #  * @property {boolean=} omitBackground
-  #  * @property {string=} encoding
-  #  */
+  # * @defaultValue 'png'
+  # */
+  # type?: 'png' | 'jpeg' | 'webp';
+  # /**
+  # * The file path to save the image to. The screenshot type will be inferred
+  # * from file extension. If path is a relative path, then it is resolved
+  # * relative to current working directory. If no path is provided, the image
+  # * won't be saved to the disk.
+  # */
+  # path?: string;
+  # /**
+  # * When true, takes a screenshot of the full page.
+  # * @defaultValue false
+  # */
+  # fullPage?: boolean;
+  # /**
+  # * An object which specifies the clipping region of the page.
+  # */
+  # clip?: ScreenshotClip;
+  # /**
+  # * Quality of the image, between 0-100. Not applicable to `png` images.
+  # */
+  # quality?: number;
+  # /**
+  # * Hides default white background and allows capturing screenshots with transparency.
+  # * @defaultValue false
+  # */
+  # omitBackground?: boolean;
+  # /**
+  # * Encoding of the image.
+  # * @defaultValue 'binary'
+  # */
+  # encoding?: 'base64' | 'binary';
+  # /**
+  # * Capture the screenshot beyond the viewport.
+  # * @defaultValue true
+  # */
+  # captureBeyondViewport?: boolean;
+  # /**
+  # * Capture the screenshot from the surface, rather than the view.
+  # * @defaultValue true
+  # */
+  # fromSurface?: boolean;
   class ScreenshotOptions
     # @params options [Hash]
     def initialize(options)
@@ -65,6 +99,18 @@ class Puppeteer::Page
       @clip = options[:clip]
       @omit_background = options[:omit_background]
       @encoding = options[:encoding]
+      @capture_beyond_viewport =
+        if options[:capture_beyond_viewport].nil?
+          true
+        else
+          options[:capture_beyond_viewport]
+        end
+      @from_surface =
+        if options[:from_surface].nil?
+          true
+        else
+          options[:from_surface]
+        end
     end
 
     attr_reader :type, :quality, :path, :clip, :encoding
@@ -76,5 +122,13 @@ class Puppeteer::Page
     def omit_background?
       @omit_background
     end
+
+    def capture_beyond_viewport?
+      @capture_beyond_viewport
+    end
+
+    def from_surface?
+      @from_surface
+    end
   end
 end

data/lib/puppeteer/page.rb

@@ -389,7 +389,18 @@ class Puppeteer::Page
     @client.send_message('Network.getCookies', urls: (urls.empty? ? [url] : urls))['cookies']
   end
 
+  # check if each cookie element has required fields ('name' and 'value')
+  private def assert_cookie_params(cookies, requires:)
+    return if cookies.all? do |cookie|
+      requires.all? { |field_name| cookie[field_name] || cookie[field_name.to_s] }
+    end
+
+    raise ArgumentError.new("Each coookie must have #{requires.join(" and ")} attribute.")
+  end
+
   def delete_cookie(*cookies)
+    assert_cookie_params(cookies, requires: %i(name))
+
     page_url = url
     starts_with_http = page_url.start_with?("http")
     cookies.each do |cookie|
@@ -399,6 +410,8 @@ class Puppeteer::Page
   end
 
   def set_cookie(*cookies)
+    assert_cookie_params(cookies, requires: %i(name value))
+
     page_url = url
     starts_with_http = page_url.start_with?("http")
     items = cookies.map do |cookie|
@@ -801,6 +814,10 @@ class Puppeteer::Page
         predicate
       end
 
+    frames.each do |frame|
+      return frame if frame_predicate.call(frame)
+    end
+
     wait_for_frame_manager_event(
       FrameManagerEmittedEvents::FrameAttached,
       FrameManagerEmittedEvents::FrameNavigated,
@@ -1008,7 +1025,15 @@ class Puppeteer::Page
   # @param quality [Integer]
   # @param omit_background [Boolean]
   # @param encoding [String]
-  def screenshot(type: nil, path: nil, full_page: nil, clip: nil, quality: nil, omit_background: nil, encoding: nil)
+  def screenshot(type: nil,
+                 path: nil,
+                 full_page: nil,
+                 clip: nil,
+                 quality: nil,
+                 omit_background: nil,
+                 encoding: nil,
+                 capture_beyond_viewport: nil,
+                 from_surface: nil)
     options = {
       type: type,
       path: path,
@@ -1017,6 +1042,8 @@ class Puppeteer::Page
       quality:  quality,
       omit_background: omit_background,
       encoding: encoding,
+      capture_beyond_viewport: capture_beyond_viewport,
+      from_surface: from_surface,
     }.compact
     screenshot_options = ScreenshotOptions.new(options)
 
@@ -1045,18 +1072,20 @@ class Puppeteer::Page
       # Overwrite clip for full page at all times.
       clip = { x: 0, y: 0, width: width, height: height, scale: 1 }
 
-      screen_orientation =
-        if @viewport&.landscape?
-          { angle: 90, type: 'landscapePrimary' }
-        else
-          { angle: 0, type: 'portraitPrimary' }
-        end
-      @client.send_message('Emulation.setDeviceMetricsOverride',
-        mobile: @viewport&.mobile? || false,
-        width: width,
-        height: height,
-        deviceScaleFactor: @viewport&.device_scale_factor || 1,
-        screenOrientation: screen_orientation)
+      unless screenshot_options.capture_beyond_viewport?
+        screen_orientation =
+          if @viewport&.landscape?
+            { angle: 90, type: 'landscapePrimary' }
+          else
+            { angle: 0, type: 'portraitPrimary' }
+          end
+        @client.send_message('Emulation.setDeviceMetricsOverride',
+          mobile: @viewport&.mobile? || false,
+          width: width,
+          height: height,
+          deviceScaleFactor: @viewport&.device_scale_factor || 1,
+          screenOrientation: screen_orientation)
+      end
     end
 
     should_set_default_background = screenshot_options.omit_background? && format == 'png'
@@ -1065,6 +1094,8 @@ class Puppeteer::Page
       format: format,
       quality: screenshot_options.quality,
       clip: clip,
+      captureBeyondViewport: screenshot_options.capture_beyond_viewport?,
+      fromSurface: screenshot_options.from_surface?,
     }.compact
     result = @client.send_message('Page.captureScreenshot', screenshot_params)
     reset_default_background_color if should_set_default_background

data/lib/puppeteer/target.rb

@@ -18,14 +18,19 @@ class Puppeteer::Target
   # @param {!function():!Promise<!Puppeteer.CDPSession>} sessionFactory
   # @param {boolean} ignoreHTTPSErrors
   # @param {?Puppeteer.Viewport} defaultViewport
-  def initialize(target_info:, browser_context:, session_factory:, ignore_https_errors:, default_viewport:)
+  def initialize(target_info:,
+                 browser_context:,
+                 session_factory:,
+                 ignore_https_errors:,
+                 default_viewport:,
+                 is_page_target_callback:)
     @target_info = target_info
     @browser_context = browser_context
     @target_id = target_info.target_id
     @session_factory = session_factory
     @ignore_https_errors = ignore_https_errors
     @default_viewport = default_viewport
-
+    @is_page_target_callback = is_page_target_callback
 
     #    /** @type {?Promise<!Puppeteer.Page>} */
     #    this._pagePromise = null;
@@ -37,14 +42,14 @@ class Puppeteer::Target
     end
     @is_closed_promise = resolvable_future
 
-    @is_initialized = @target_info.type != 'page' || !@target_info.url.empty?
+    @is_initialized = !@is_page_target_callback.call(@target_info) || !@target_info.url.empty?
 
     if @is_initialized
       @initialize_callback_promise.fulfill(true)
     end
   end
 
-  attr_reader :target_id, :initialized_promise, :is_closed_promise
+  attr_reader :target_id, :target_info, :initialized_promise, :is_closed_promise
 
   def closed_callback
     @is_closed_promise.fulfill(true)
@@ -83,7 +88,7 @@ class Puppeteer::Target
   end
 
   def page
-    if ['page', 'background_page', 'webview'].include?(@target_info.type) && @page.nil?
+    if @is_page_target_callback.call(@target_info) && @page.nil?
       client = @session_factory.call
       @page = Puppeteer::Page.create(client, self, @ignore_https_errors, @default_viewport)
     end
@@ -145,7 +150,7 @@ class Puppeteer::Target
   def handle_target_info_changed(target_info)
     @target_info = target_info
 
-    if !@is_initialized && (@target_info.type != 'page' || !@target_info.url.empty?)
+    if !@is_initialized && (!@is_page_target_callback.call(@target_info) || !@target_info.url.empty?)
       @is_initialized = true
       @initialize_callback_promise.fulfill(true)
     end

data/lib/puppeteer/version.rb

@@ -1,3 +1,3 @@
 module Puppeteer
-  VERSION = '0.40.5'
+  VERSION = '0.41.0'
 end

data/puppeteer-ruby.gemspec

@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rollbar'
   spec.add_development_dependency 'rspec', '~> 3.11.0'
   spec.add_development_dependency 'rspec_junit_formatter' # for CircleCI.
-  spec.add_development_dependency 'rubocop', '~> 1.26.0'
+  spec.add_development_dependency 'rubocop', '~> 1.31.0'
   spec.add_development_dependency 'rubocop-rspec'
   spec.add_development_dependency 'sinatra'
   spec.add_development_dependency 'webrick'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puppeteer-ruby
 version: !ruby/object:Gem::Version
-  version: 0.40.5
+  version: 0.41.0
 platform: ruby
 authors:
 - YusukeIwaki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-03-26 00:00:00.000000000 Z
+date: 2022-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.26.0
+        version: 1.31.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.26.0
+        version: 1.31.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement