puppeteer-ruby 0.39.0 → 0.40.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e09022ffac5da45a7c40bca6b612c8a67afece662142675625794b15b89a044b
-  data.tar.gz: 9af45888cfcfaff3bc368facbd1b5fb28572924ccf0c8d33b05ba310894638bb
+  metadata.gz: 0b55d70dfff58f262d0385b5cb3c30b14d61dee0b4e62b6159240fd4b108da11
+  data.tar.gz: a61ebc455d7149b1e20422ab86545c7aea4753ce41bfe27c8dead388b9595ba2
 SHA512:
-  metadata.gz: 580410458c1e0f2ac6dd8fb678fce42ed5f12ee6a1ff70b90600b6601b945af4342db53f6b98fe73ba249a9d85e3e8d64efb664534f7b1231265dbfbeb2f8e09
-  data.tar.gz: adecd0293ec7b7ade882d9c331b794b4a14e2e52999c164880e7f4b0fa3d723e7bc6b44a238a5ad04c50ad0bfc9c3d5d33e60b50ea40904dc49af1506ce3e72e
+  metadata.gz: b58e74d4397fa2954df3a5ef90f3e3dc55702e058b2a5a0c84dff1cb236d012019a1f98626972ec3d11afb70708e760846ece034ec8e7dcf41bac1ded218c53c
+  data.tar.gz: 315fdc4a2e7743bf21daf38e260a0c8876c77d8d19ee81797b083ddbb7628a1287e618ee812311995d270e5ee95b7febd4b1ca8e43deea0e33ade601b2b361dd

data/CHANGELOG.md

@@ -1,7 +1,36 @@
-### main [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.39.0...main)]
+### main [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.3...main)]
 
 - xxx
 
+### 0.40.3 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.2...0.40.3)]
+
+Bugfix:
+
+- Prevent `Ctrl+C` from stopping Chrome [#196](https://github.com/YusukeIwaki/puppeteer-ruby/pull/196)
+
+
+### 0.40.2 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.1...0.40.2)]
+
+Bugfix:
+
+- Fix crash on setting response listener [#195](https://github.com/YusukeIwaki/puppeteer-ruby/pull/195)
+
+### 0.40.1 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.0...0.40.1)]
+
+Bugfix:
+
+- Fix error on reload [#190](https://github.com/YusukeIwaki/puppeteer-ruby/pull/190)
+- Fix error message on failing to find Chrome executable.
+- Fix apidoc
+
+### 0.40.0 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.39.0...0.40.0)]
+
+New features:
+
+- Puppeteer 13.0, 13.1 functionalities
+- Support Ruby 3.1
+
+
 ### 0.39.0 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.38.0...0.39.0)]
 
 New features:

data/docs/api_coverage.md

@@ -1,6 +1,6 @@
 # API coverages
-- Puppeteer version: v12.0.0
-- puppeteer-ruby version: 0.39.0
+- Puppeteer version: v13.0.1
+- puppeteer-ruby version: 0.40.3
 
 ## Puppeteer
 
@@ -296,6 +296,7 @@
 * ~~toString~~
 * type => `#type_text`
 * uploadFile => `#upload_file`
+* waitForSelector => `#wait_for_selector`
 
 ## HTTPRequest
 
@@ -309,6 +310,8 @@
 * frame
 * headers
 * initiator
+* ~~interceptResolutionState~~
+* ~~isInterceptResolutionHandled~~
 * isNavigationRequest => `#navigation_request?`
 * method
 * postData => `#post_data`

data/lib/puppeteer/aria_query_handler.rb

@@ -36,10 +36,11 @@ class Puppeteer::AriaQueryHandler
     end
   end
 
-  def wait_for(dom_world, selector, visible: nil, hidden: nil, timeout: nil)
+  def wait_for(dom_world, selector, visible: nil, hidden: nil, timeout: nil, root: nil)
+    # addHandlerToWorld
     binding_function = Puppeteer::DOMWorld::BindingFunction.new(
       name: 'ariaQuerySelector',
-      proc: -> (sel) { query_one(dom_world.send(:document), sel) },
+      proc: -> (sel) { query_one(root || dom_world.send(:document), sel) },
     )
     dom_world.send(:wait_for_selector_in_page,
       '(_, selector) => globalThis.ariaQuerySelector(selector)',
@@ -47,7 +48,9 @@ class Puppeteer::AriaQueryHandler
       visible: visible,
       hidden: hidden,
       timeout: timeout,
-      binding_function: binding_function)
+      binding_function: binding_function,
+      root: root,
+    )
   end
 
   def query_all(element, selector)

data/lib/puppeteer/browser_runner.rb

@@ -31,7 +31,9 @@ class Puppeteer::BrowserRunner
           [executable_path]
         end
 
-      stdin, @stdout, @stderr, @thread = Open3.popen3(env, executable_path, *args)
+      popen3_args = args || []
+      popen3_args << { pgroup: true } unless Puppeteer.env.windows?
+      stdin, @stdout, @stderr, @thread = Open3.popen3(env, executable_path, *popen3_args)
       stdin.close
       @pid = @thread.pid
     rescue Errno::ENOENT => err
@@ -149,7 +151,7 @@ class Puppeteer::BrowserRunner
     # Attempt to remove temporary profile directory to avoid littering.
     begin
       if @using_temp_user_data_dir
-        FileUtils.rm_rf(@temp_directory)
+        FileUtils.rm_rf(@user_data_dir)
       end
     rescue => err
       debug_puts(err)

data/lib/puppeteer/custom_query_handler.rb

@@ -21,12 +21,12 @@ class Puppeteer::CustomQueryHandler
     nil
   end
 
-  def wait_for(dom_world, selector, visible: nil, hidden: nil, timeout: nil)
+  def wait_for(dom_world, selector, visible: nil, hidden: nil, timeout: nil, root: nil)
     unless @query_one
       raise NotImplementedError.new("#{self.class}##{__method__} is not implemented.")
     end
 
-    dom_world.send(:wait_for_selector_in_page, @query_one, selector, visible: visible, hidden: hidden, timeout: timeout)
+    dom_world.send(:wait_for_selector_in_page, @query_one, selector, visible: visible, hidden: hidden, timeout: timeout, root: root)
   end
 
   def query_all(element, selector)

data/lib/puppeteer/dom_world.rb

@@ -420,10 +420,10 @@ class Puppeteer::DOMWorld
   # @param visible [Boolean] Wait for element visible (not 'display: none' nor 'visibility: hidden') on true. default to false.
   # @param hidden [Boolean] Wait for element invisible ('display: none' nor 'visibility: hidden') on true. default to false.
   # @param timeout [Integer]
-  def wait_for_selector(selector, visible: nil, hidden: nil, timeout: nil)
+  def wait_for_selector(selector, visible: nil, hidden: nil, timeout: nil, root: nil)
     # call wait_for_selector_in_page with custom query selector.
     query_selector_manager = Puppeteer::QueryHandlerManager.instance
-    query_selector_manager.detect_query_handler(selector).wait_for(self, visible: visible, hidden: hidden, timeout: timeout)
+    query_selector_manager.detect_query_handler(selector).wait_for(self, visible: visible, hidden: hidden, timeout: timeout, root: root)
   end
 
   private def binding_identifier(name, context)
@@ -497,10 +497,11 @@ class Puppeteer::DOMWorld
   # @param visible [Boolean] Wait for element visible (not 'display: none' nor 'visibility: hidden') on true. default to false.
   # @param hidden [Boolean] Wait for element invisible ('display: none' nor 'visibility: hidden') on true. default to false.
   # @param timeout [Integer]
-  private def wait_for_selector_in_page(query_one, selector, visible: nil, hidden: nil, timeout: nil, binding_function: nil)
+  private def wait_for_selector_in_page(query_one, selector, visible: nil, hidden: nil, timeout: nil, root: nil, binding_function: nil)
     option_wait_for_visible = visible || false
     option_wait_for_hidden = hidden || false
     option_timeout = timeout || @timeout_settings.timeout
+    option_root = root
 
     polling =
       if option_wait_for_visible || option_wait_for_hidden
@@ -511,11 +512,11 @@ class Puppeteer::DOMWorld
     title = "selector #{selector}#{option_wait_for_hidden ? 'to be hidden' : ''}"
 
     selector_predicate = make_predicate_string(
-      predicate_arg_def: '(selector, waitForVisible, waitForHidden)',
+      predicate_arg_def: '(root, selector, waitForVisible, waitForHidden)',
       predicate_query_handler: query_one,
       async: true,
       predicate_body: <<~JAVASCRIPT
-        const node = await predicateQueryHandler(document, selector)
+        const node = await predicateQueryHandler(root, selector)
         return checkWaitForOptions(node, waitForVisible, waitForHidden);
       JAVASCRIPT
     )
@@ -527,6 +528,7 @@ class Puppeteer::DOMWorld
       polling: polling,
       timeout: option_timeout,
       args: [selector, option_wait_for_visible, option_wait_for_hidden],
+      root: option_root,
       binding_function: binding_function,
     )
     handle = wait_task.await_promise
@@ -541,10 +543,11 @@ class Puppeteer::DOMWorld
   # @param visible [Boolean] Wait for element visible (not 'display: none' nor 'visibility: hidden') on true. default to false.
   # @param hidden [Boolean] Wait for element invisible ('display: none' nor 'visibility: hidden') on true. default to false.
   # @param timeout [Integer]
-  def wait_for_xpath(xpath, visible: nil, hidden: nil, timeout: nil)
+  def wait_for_xpath(xpath, visible: nil, hidden: nil, timeout: nil, root: nil)
     option_wait_for_visible = visible || false
     option_wait_for_hidden = hidden || false
     option_timeout = timeout || @timeout_settings.timeout
+    option_root = root
 
     polling =
       if option_wait_for_visible || option_wait_for_hidden
@@ -555,9 +558,9 @@ class Puppeteer::DOMWorld
     title = "XPath #{xpath}#{option_wait_for_hidden ? 'to be hidden' : ''}"
 
     xpath_predicate = make_predicate_string(
-      predicate_arg_def: '(selector, waitForVisible, waitForHidden)',
+      predicate_arg_def: '(root, selector, waitForVisible, waitForHidden)',
       predicate_body: <<~JAVASCRIPT
-        const node = document.evaluate(selector, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+        const node = document.evaluate(selector, root, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
         return checkWaitForOptions(node, waitForVisible, waitForHidden);
       JAVASCRIPT
     )
@@ -569,6 +572,7 @@ class Puppeteer::DOMWorld
       polling: polling,
       timeout: option_timeout,
       args: [xpath, option_wait_for_visible, option_wait_for_hidden],
+      root: option_root,
     )
     handle = wait_task.await_promise
     unless handle.as_element

data/lib/puppeteer/element_handle.rb

@@ -20,6 +20,60 @@ class Puppeteer::ElementHandle < Puppeteer::JSHandle
     @disposed = false
   end
 
+  def inspect
+    values = %i[context remote_object page disposed].map do |sym|
+      value = instance_variable_get(:"@#{sym}")
+      "@#{sym}=#{value}"
+    end
+    "#<Puppeteer::ElementHandle #{values.join(' ')}>"
+  end
+
+  #
+  # Wait for the `selector` to appear within the element. If at the moment of calling the
+  # method the `selector` already exists, the method will return immediately. If
+  # the `selector` doesn't appear after the `timeout` milliseconds of waiting, the
+  # function will throw.
+  #
+  # This method does not work across navigations or if the element is detached from DOM.
+  #
+  # @param selector - A
+  # {@link https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors | selector}
+  # of an element to wait for
+  # @param options - Optional waiting parameters
+  # @returns Promise which resolves when element specified by selector string
+  # is added to DOM. Resolves to `null` if waiting for hidden: `true` and
+  # selector is not found in DOM.
+  # @remarks
+  # The optional parameters in `options` are:
+  #
+  # - `visible`: wait for the selected element to be present in DOM and to be
+  # visible, i.e. to not have `display: none` or `visibility: hidden` CSS
+  # properties. Defaults to `false`.
+  #
+  # - `hidden`: wait for the selected element to not be found in the DOM or to be hidden,
+  # i.e. have `display: none` or `visibility: hidden` CSS properties. Defaults to
+  # `false`.
+  #
+  # - `timeout`: maximum time to wait in milliseconds. Defaults to `30000`
+  # (30 seconds). Pass `0` to disable timeout. The default value can be changed
+  # by using the {@link Page.setDefaultTimeout} method.
+  def wait_for_selector(selector, visible: nil, hidden: nil, timeout: nil)
+    frame = @context.frame
+
+    secondary_world = frame.secondary_world
+    adopted_root = secondary_world.execution_context.adopt_element_handle(self)
+    handle = secondary_world.wait_for_selector(selector, visible: visible, hidden: hidden, timeout: timeout, root: adopted_root)
+    adopted_root.dispose
+    return nil unless handle
+
+    main_world = frame.main_world
+    result = main_world.execution_context.adopt_element_handle(handle)
+    handle.dispose
+    result
+  end
+
+  define_async_method :async_wait_for_selector
+
   def as_element
     self
   end

data/lib/puppeteer/events.rb

@@ -81,6 +81,7 @@ module NetworkManagerEmittedEvents ; end
 
 {
   Request: EventsDefinitionUtils.symbol('NetworkManager.Request'),
+  RequestServedFromCache: EventsDefinitionUtils.symbol('NetworkManager.RequestServedFromCache'),
   Response: EventsDefinitionUtils.symbol('NetworkManager.Response'),
   RequestFailed: EventsDefinitionUtils.symbol('NetworkManager.RequestFailed'),
   RequestFinished: EventsDefinitionUtils.symbol('NetworkManager.RequestFinished'),

data/lib/puppeteer/http_response.rb

@@ -31,7 +31,8 @@ class Puppeteer::HTTPResponse
   # @param client [Puppeteer::CDPSession]
   # @param request [Puppeteer::HTTPRequest]
   # @param response_payload [Hash]
-  def initialize(client, request, response_payload)
+  # @param extra_info [Hash|nil]
+  def initialize(client, request, response_payload, extra_info)
     @client = client
     @request = request
 
@@ -41,14 +42,15 @@ class Puppeteer::HTTPResponse
       port: response_payload['remotePort'],
     )
 
-    @status = response_payload['status']
-    @status_text = response_payload['statusText']
+    @status_text = parse_štatus_text_from_extra_info(extra_info) || response_payload['statusText']
     @url = request.url
     @from_disk_cache = !!response_payload['fromDiskCache']
     @from_service_worker = !!response_payload['fromServiceWorker']
 
+    @status = extra_info ? extra_info['statusCode'] : response_payload['status']
     @headers = {}
-    response_payload['headers'].each do |key, value|
+    headers = extra_info ? extra_info['headers'] : response_payload['headers']
+    headers.each do |key, value|
       @headers[key.downcase] = value
     end
     @security_details = if_present(response_payload['securityDetails']) do |security_payload|
@@ -62,6 +64,25 @@ class Puppeteer::HTTPResponse
 
   attr_reader :remote_address, :url, :status, :status_text, :headers, :security_details, :request
 
+  def inspect
+    values = %i[remote_address url status status_text headers security_details request].map do |sym|
+      value = instance_variable_get(:"@#{sym}")
+      "@#{sym}=#{value}"
+    end
+    "#<Puppeteer::HTTPRequest #{values.join(' ')}>"
+  end
+
+  private def parse_štatus_text_from_extra_info(extra_info)
+    return nil if !extra_info || !extra_info['headersText']
+    first_line = extra_info['headersText'].split("\r").first
+    return nil unless first_line
+    /[^ ]* [^ ]* (.*)/.match(first_line) do |m|
+      return m[1]
+    end
+
+    nil
+  end
+
   # @return [Boolean]
   def ok?
     @status == 0 || (@status >= 200 && @status <= 299)

data/lib/puppeteer/js_handle.rb

@@ -36,6 +36,14 @@ class Puppeteer::JSHandle
 
   attr_reader :context, :remote_object
 
+  def inspect
+    values = %i[context remote_object disposed].map do |sym|
+      value = instance_variable_get(:"@#{sym}")
+      "@#{sym}=#{value}"
+    end
+    "#<Puppeteer::JSHandle #{values.join(' ')}>"
+  end
+
   # @return [Puppeteer::ExecutionContext]
   def execution_context
     @context

data/lib/puppeteer/launcher/chrome.rb

@@ -286,15 +286,15 @@ module Puppeteer::Launcher
         end
 
       chrome_path = chrome_path_map[channel]
-      if chrome_path.is_a?(Proc)
-        chrome_path = chrome_path.call
-      end
-
       unless chrome_path
         raise ArgumentError.new("Invalid channel: '#{channel}'. Allowed channel is #{chrome_path_map.keys}")
       end
 
-      unless File.exist?(chrome_path)
+      if chrome_path.is_a?(Proc)
+        chrome_path = chrome_path.call
+      end
+
+      if !chrome_path || !File.exist?(chrome_path)
         raise "#{channel} is not installed on this system.\nExpected path: #{chrome_path}"
       end
 

data/lib/puppeteer/launcher/firefox.rb

@@ -195,7 +195,7 @@ module Puppeteer::Launcher
         firefox_path = firefox_path.call
       end
 
-      unless File.exist?(firefox_path)
+      if !firefox_path || !File.exist?(firefox_path)
         raise "Nightly version of Firefox is not installed on this system.\nExpected path: #{firefox_path}"
       end
 

data/lib/puppeteer/network_event_manager.rb

@@ -0,0 +1,122 @@
+# Helper class to track network events by request ID
+class Puppeteer::NetworkEventManager
+  def initialize
+    #
+    # There are four possible orders of events:
+    #  A. `_onRequestWillBeSent`
+    #  B. `_onRequestWillBeSent`, `_onRequestPaused`
+    #  C. `_onRequestPaused`, `_onRequestWillBeSent`
+    #  D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`,
+    #     `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`
+    #     (see crbug.com/1196004)
+    #
+    # For `_onRequest` we need the event from `_onRequestWillBeSent` and
+    # optionally the `interceptionId` from `_onRequestPaused`.
+    #
+    # If request interception is disabled, call `_onRequest` once per call to
+    # `_onRequestWillBeSent`.
+    # If request interception is enabled, call `_onRequest` once per call to
+    # `_onRequestPaused` (once per `interceptionId`).
+    #
+    # Events are stored to allow for subsequent events to call `_onRequest`.
+    #
+    # Note that (chains of) redirect requests have the same `requestId` (!) as
+    # the original request. We have to anticipate series of events like these:
+    #  A. `_onRequestWillBeSent`,
+    #     `_onRequestWillBeSent`, ...
+    #  B. `_onRequestWillBeSent`, `_onRequestPaused`,
+    #     `_onRequestWillBeSent`, `_onRequestPaused`, ...
+    #  C. `_onRequestWillBeSent`, `_onRequestPaused`,
+    #     `_onRequestPaused`, `_onRequestWillBeSent`, ...
+    #  D. `_onRequestPaused`, `_onRequestWillBeSent`,
+    #     `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`,
+    #     `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`, ...
+    #     (see crbug.com/1196004)
+    @request_will_be_sent_map = {}
+    @request_paused_map = {}
+    @http_requests_map = {}
+    #
+    # The below maps are used to reconcile Network.responseReceivedExtraInfo
+    # events with their corresponding request. Each response and redirect
+    # response gets an ExtraInfo event, and we don't know which will come first.
+    # This means that we have to store a Response or an ExtraInfo for each
+    # response, and emit the event when we get both of them. In addition, to
+    # handle redirects, we have to make them Arrays to represent the chain of
+    # events.
+    @response_received_extra_info_map = {}
+    @queued_redirect_info_map = {}
+    @queued_event_group_map = {}
+  end
+
+  def forget(network_request_id)
+    @request_will_be_sent_map.delete(network_request_id)
+    @request_paused_map.delete(network_request_id)
+    @queued_event_group_map.delete(network_request_id)
+    @queued_redirect_info_map.delete(network_request_id)
+    @response_received_extra_info_map.delete(network_request_id)
+  end
+
+  def response_extra_info(network_request_id)
+    @response_received_extra_info_map[network_request_id] ||= []
+  end
+
+  private def queued_redirect_info(fetch_request_id)
+    @queued_redirect_info_map[fetch_request_id] ||= []
+  end
+
+  def enqueue_redirect_info(fetch_request_id, redirect_info)
+    queued_redirect_info(fetch_request_id) << redirect_info
+  end
+
+  def take_queued_redirect_info(fetch_request_id)
+    queued_redirect_info(fetch_request_id).shift
+  end
+
+  def num_requests_in_progress
+    @http_requests_map.count { |_, request| !request.response }
+  end
+
+  def store_request_will_be_sent(network_request_id, event)
+    @request_will_be_sent_map[network_request_id] = event
+  end
+
+  def get_request_will_be_sent(network_request_id)
+    @request_will_be_sent_map[network_request_id]
+  end
+
+  def forget_request_will_be_sent(network_request_id)
+    @request_will_be_sent_map.delete(network_request_id)
+  end
+
+  def store_request_paused(network_request_id, event)
+    @request_paused_map[network_request_id] = event
+  end
+
+  def get_request_paused(network_request_id)
+    @request_paused_map[network_request_id]
+  end
+
+  def forget_request_paused(network_request_id)
+    @request_paused_map.delete(network_request_id)
+  end
+
+  def store_request(network_request_id, request)
+    @http_requests_map[network_request_id] = request
+  end
+
+  def get_request(network_request_id)
+    @http_requests_map[network_request_id]
+  end
+
+  def forget_request(network_request_id)
+    @http_requests_map.delete(network_request_id)
+  end
+
+  def enqueue_event_group(network_request_id, queued_event_group)
+    @queued_event_group_map[network_request_id] = queued_event_group
+  end
+
+  def get_queued_event_group(network_request_id)
+    @queued_event_group_map[network_request_id]
+  end
+end

data/lib/puppeteer/network_manager.rb

@@ -53,6 +53,22 @@ class Puppeteer::NetworkManager
     end
   end
 
+  class RedirectInfo
+    def initialize(event:, fetch_request_id:)
+      @event = event
+      @fetch_request_id = fetch_request_id
+    end
+    attr_reader :event, :fetch_request_id
+  end
+
+  class QueuedEventGroup
+    def initialize(response_received_event:)
+      @response_received_event = response_received_event
+    end
+    attr_reader :response_received_event
+    attr_accessor :loading_finished_event, :loading_failed_event
+  end
+
   # @param {!Puppeteer.CDPSession} client
   # @param {boolean} ignoreHTTPSErrors
   # @param {!Puppeteer.FrameManager} frameManager
@@ -60,12 +76,7 @@ class Puppeteer::NetworkManager
     @client = client
     @ignore_https_errors = ignore_https_errors
     @frame_manager = frame_manager
-
-    # @type {!Map<string, !Request>}
-    @request_id_to_request = {}
-
-    # @type {!Map<string, !Protocol.Network.requestWillBeSentPayload>}
-    @request_id_to_request_with_be_sent_event = {}
+    @network_event_manager = Puppeteer::NetworkEventManager.new
 
     @extra_http_headers = {}
 
@@ -73,7 +84,6 @@ class Puppeteer::NetworkManager
     @user_request_interception_enabled = false
     @protocol_request_interception_enabled = false
     @user_cache_disabled = false
-    @request_id_to_interception_id = {}
     @internal_network_condition = InternalNetworkCondition.new(@client)
 
     @client.on_event('Fetch.requestPaused') do |event|
@@ -97,6 +107,9 @@ class Puppeteer::NetworkManager
     @client.on_event('Network.loadingFailed') do |event|
       handle_loading_failed(event)
     end
+    @client.on_event('Network.responseReceivedExtraInfo') do |event|
+      handle_response_received_extra_info(event)
+    end
   end
 
   def init
@@ -106,6 +119,14 @@ class Puppeteer::NetworkManager
     end
   end
 
+  def inspect
+    values = %i[network_event_manager].map do |sym|
+      value = instance_variable_get(:"@#{sym}")
+      "@#{sym}=#{value}"
+    end
+    "#<Puppeteer::HTTPRequest #{values.join(' ')}>"
+  end
+
   # @param username [String|NilClass]
   # @param password [String|NilClass]
   def authenticate(username:, password:)
@@ -131,6 +152,10 @@ class Puppeteer::NetworkManager
     @extra_http_headers.dup
   end
 
+  def num_requests_in_progress
+    @network_event_manager.num_requests_in_progress
+  end
+
   # @param value [TrueClass|FalseClass]
   def offline_mode=(value)
     @internal_network_condition.offline_mode=(value)
@@ -185,14 +210,17 @@ class Puppeteer::NetworkManager
 
   private def handle_request_will_be_sent(event)
     # Request interception doesn't happen for data URLs with Network Service.
-    if @protocol_request_interception_enabled && !event['request']['url'].start_with?('data:')
-      request_id = event['requestId']
-      interception_id = @request_id_to_interception_id.delete(request_id)
-      if interception_id
-        handle_request(event, interception_id)
-      else
-        @request_id_to_request_with_be_sent_event[request_id] = event
+    if @user_request_interception_enabled && !event['request']['url'].start_with?('data:')
+      network_request_id = event['requestId']
+      @network_event_manager.store_request_will_be_sent(network_request_id, event)
+
+      # CDP may have sent a Fetch.requestPaused event already. Check for it.
+      if_present(@network_event_manager.get_request_paused(network_request_id)) do |request_paused_event|
+        fetch_request_id = request_paused_event['requestId']
+        handle_request(event, fetch_request_id)
+        @network_event_manager.forget_request_paused(network_request_id)
       end
+
       return
     end
     handle_request(event, nil)
@@ -233,26 +261,61 @@ class Puppeteer::NetworkManager
       end
     end
 
-    request_id = event['networkId']
-    interception_id = event['requestId']
-    if request_id && (request_will_be_sent_event = @request_id_to_request_with_be_sent_event.delete(request_id))
-      handle_request(request_will_be_sent_event, interception_id)
+    network_request_id = event['networkId']
+    fetch_request_id = event['requestId']
+    return unless network_request_id
+
+    request_will_be_sent_event = @network_event_manager.get_request_will_be_sent(network_request_id)
+
+    # redirect requests have the same `requestId`
+    if request_will_be_sent_event &&
+       (request_will_be_sent_event['request']['url'] != event['request']['url'] ||
+        request_will_be_sent_event['request']['method'] != event['request']['method'])
+
+      @network_event_manager.forget_request_will_be_sent(network_request_id)
+      request_will_be_sent_event = nil
+    end
+
+    if request_will_be_sent_event
+      handle_request(request_will_be_sent_event, fetch_request_id)
     else
-      @request_id_to_interception_id[request_id] = interception_id
+      @network_event_manager.store_request_paused(network_request_id, event)
     end
   end
 
-  private def handle_request(event, interception_id)
+  private def handle_request(event, fetch_request_id)
     redirect_chain = []
     if event['redirectResponse']
-      if_present(@request_id_to_request[event['requestId']]) do |request|
-        handle_request_redirect(request, event['redirectResponse'])
+      # We want to emit a response and requestfinished for the
+      # redirectResponse, but we can't do so unless we have a
+      # responseExtraInfo ready to pair it up with. If we don't have any
+      # responseExtraInfos saved in our queue, they we have to wait until
+      # the next one to emit response and requestfinished, *and* we should
+      # also wait to emit this Request too because it should come after the
+      # response/requestfinished.
+      redirect_response_extra_info = nil
+      if event['redirectHasExtraInfo']
+        redirect_response_extra_info = @network_event_manager.response_extra_info(event['requestId']).shift
+        unless redirect_response_extra_info
+          redirect_info = RedirectInfo.new(
+            event: event,
+            fetch_request_id: fetch_request_id,
+          )
+          @network_event_manager.enqueue_redirect_info(event['requestId'], redirect_info)
+          return
+        end
+      end
+
+      # If we connect late to the target, we could have missed the
+      # requestWillBeSent event.
+      if_present(@network_event_manager.get_request(event['requestId'])) do |request|
+        handle_request_redirect(request, event['redirectResponse'], redirect_response_extra_info)
         redirect_chain = request.internal.redirect_chain
       end
     end
     frame = if_present(event['frameId']) { |frame_id| @frame_manager.frame(frame_id) }
-    request = Puppeteer::HTTPRequest.new(@client, frame, interception_id, @user_request_interception_enabled, event, redirect_chain)
-    @request_id_to_request[event['requestId']] = request
+    request = Puppeteer::HTTPRequest.new(@client, frame, fetch_request_id, @user_request_interception_enabled, event, redirect_chain)
+    @network_event_manager.store_request(event['requestId'], request)
     emit_event(NetworkManagerEmittedEvents::Request, request)
     begin
       request.finalize_interceptions
@@ -262,55 +325,133 @@ class Puppeteer::NetworkManager
   end
 
   private def handle_request_served_from_cache(event)
-    if_present(@request_id_to_request[event['requestId']]) do |request|
+    request = @network_event_manager.get_request(event['requestId'])
+    if request
       request.internal.from_memory_cache = true
     end
+    emit_event(NetworkManagerEmittedEvents::RequestServedFromCache, request)
   end
 
   # @param request [Puppeteer::HTTPRequest]
   # @param response_payload [Hash]
-  private def handle_request_redirect(request, response_payload)
-    response = Puppeteer::HTTPResponse.new(@client, request, response_payload)
+  private def handle_request_redirect(request, response_payload, extra_info)
+    response = Puppeteer::HTTPResponse.new(@client, request, response_payload, extra_info)
     request.internal.response = response
     request.internal.redirect_chain << request
     response.internal.body_loaded_promise.reject(Puppeteer::HTTPResponse::Redirected.new)
-    @request_id_to_request.delete(request.internal.request_id)
-    @attempted_authentications.delete(request.internal.interception_id)
+    forget_request(request, false)
     emit_event(NetworkManagerEmittedEvents::Response, response)
     emit_event(NetworkManagerEmittedEvents::RequestFinished, request)
   end
 
-  # @param event [Hash]
-  private def handle_response_received(event)
-    request = @request_id_to_request[event['requestId']]
+  private def emit_response_event(response_received_event, extra_info)
+    request = @network_event_manager.get_request(response_received_event['requestId'])
     # FileUpload sends a response without a matching request.
     return unless request
 
-    response = Puppeteer::HTTPResponse.new(@client, request, event['response'])
+    unless @network_event_manager.response_extra_info(response_received_event['requestId']).empty?
+      debug_puts("Unexpected extraInfo events for request #{response_received_event['requestId']}")
+    end
+
+    response = Puppeteer::HTTPResponse.new(@client, request, response_received_event['response'], extra_info)
     request.internal.response = response
     emit_event(NetworkManagerEmittedEvents::Response, response)
   end
 
+  # @param event [Hash]
+  private def handle_response_received(event)
+    request = @network_event_manager.get_request(event['requestId'])
+    extra_info = nil
+    if request && !request.internal.from_memory_cache? && event['hasExtraInfo']
+      extra_info = @network_event_manager.response_extra_info(event['requestId']).shift
+
+      unless extra_info
+        # Wait until we get the corresponding ExtraInfo event.
+        @network_event_manager.enqueue_event_group(event['requestId'], QueuedEventGroup.new(response_received_event: event))
+        return
+      end
+    end
+    emit_response_event(event, extra_info)
+  end
+
+  private def handle_response_received_extra_info(event)
+    # We may have skipped a redirect response/request pair due to waiting for
+    # this ExtraInfo event. If so, continue that work now that we have the
+    # request.
+    if_present(@network_event_manager.take_queued_redirect_info(event['requestId'])) do |redirect_info|
+      @network_event_manager.response_extra_info(event['requestId']) << event
+      handle_request(redirect_info.event, redirect_info)
+      return
+    end
+
+    # We may have skipped response and loading events because we didn't have
+    # this ExtraInfo event yet. If so, emit those events now.
+    if_present(@network_event_manager.get_queued_event_group(event['requestId'])) do |queued_events|
+      emit_response_event(queued_events.response_received_event, event)
+      if_present(queued_events.loading_finished_event) do |loading_finished_event|
+        emit_loading_finished(loading_finished_event)
+      end
+      if_present(queued_events.loading_failed_event) do |loading_failed_event|
+        emit_loading_failed(loading_failed_event)
+      end
+      return
+    end
+
+    # Wait until we get another event that can use this ExtraInfo event.
+    @network_event_manager.response_extra_info(event['requestId']) << event
+  end
+
+  private def forget_request(request, forget_events)
+    request_id = request.internal.request_id
+    interception_id = request.internal.interception_id
+
+    @network_event_manager.forget_request(request_id)
+    @attempted_authentications.delete(interception_id)
+    if forget_events
+      @network_event_manager.forget(request_id)
+    end
+  end
+
   private def handle_loading_finished(event)
-    request = @request_id_to_request[event['requestId']]
+    # If the response event for this request is still waiting on a
+    # corresponding ExtraInfo event, then wait to emit this event too.
+    queued_events = @network_event_manager.get_queued_event_group(event['requestId'])
+    if queued_events
+      queued_events.loading_finished_event = event
+    else
+      emit_loading_finished(event)
+    end
+  end
+
+  private def emit_loading_finished(event)
+    request = @network_event_manager.get_request(event['requestId'])
     # For certain requestIds we never receive requestWillBeSent event.
     # @see https://crbug.com/750469
     return unless request
 
-
     # Under certain conditions we never get the Network.responseReceived
     # event from protocol. @see https://crbug.com/883475
     if_present(request.response) do |response|
       response.internal.body_loaded_promise.fulfill(nil)
     end
 
-    @request_id_to_request.delete(request.internal.request_id)
-    @attempted_authentications.delete(request.internal.interception_id)
+    forget_request(request, true)
     emit_event(NetworkManagerEmittedEvents::RequestFinished, request)
   end
 
   private def handle_loading_failed(event)
-    request = @request_id_to_request[event['requestId']]
+    # If the response event for this request is still waiting on a
+    # corresponding ExtraInfo event, then wait to emit this event too.
+    queued_events = @network_event_manager.get_queued_event_group(event['requestId'])
+    if queued_events
+      queued_events.loading_failed_event = event
+    else
+      emit_loading_failed(event)
+    end
+  end
+
+  private def emit_loading_failed(event)
+    request = @network_event_manager.get_request(event['requestId'])
     # For certain requestIds we never receive requestWillBeSent event.
     # @see https://crbug.com/750469
     return unless request
@@ -319,8 +460,7 @@ class Puppeteer::NetworkManager
     if_present(request.response) do |response|
       response.internal.body_loaded_promise.fulfill(nil)
     end
-    @request_id_to_request.delete(request.internal.request_id)
-    @attempted_authentications.delete(request.internal.interception_id)
+    forget_request(request, true)
     emit_event(NetworkManagerEmittedEvents::RequestFailed, request)
   end
 end

data/lib/puppeteer/query_handler_manager.rb

@@ -26,8 +26,8 @@ class Puppeteer::QueryHandlerManager
       @query_handler.query_one(element_handle, @selector)
     end
 
-    def wait_for(dom_world, visible:, hidden:, timeout:)
-      @query_handler.wait_for(dom_world, @selector, visible: visible, hidden: hidden, timeout: timeout)
+    def wait_for(dom_world, visible:, hidden:, timeout:, root:)
+      @query_handler.wait_for(dom_world, @selector, visible: visible, hidden: hidden, timeout: timeout, root: root)
     end
 
     def query_all(element_handle)

data/lib/puppeteer/version.rb

@@ -1,3 +1,3 @@
 module Puppeteer
-  VERSION = '0.39.0'
+  VERSION = '0.40.3'
 end

data/lib/puppeteer/wait_task.rb

@@ -9,7 +9,7 @@ class Puppeteer::WaitTask
     end
   end
 
-  def initialize(dom_world:, predicate_body:, title:, polling:, timeout:, args: [], binding_function: nil)
+  def initialize(dom_world:, predicate_body:, title:, polling:, timeout:, args: [], binding_function: nil, root: nil)
     if polling.is_a?(String)
       if polling != 'raf' && polling != 'mutation'
         raise ArgumentError.new("Unknown polling option: #{polling}")
@@ -25,6 +25,7 @@ class Puppeteer::WaitTask
     @dom_world = dom_world
     @polling = polling
     @timeout = timeout
+    @root = root
     @predicate_body = "return (#{predicate_body})(...args);"
     @args = args
     @binding_function = binding_function
@@ -68,6 +69,7 @@ class Puppeteer::WaitTask
     begin
       success = context.evaluate_handle(
         WAIT_FOR_PREDICATE_PAGE_FUNCTION,
+        @root,
         @predicate_body,
         @polling,
         @timeout,
@@ -121,8 +123,9 @@ class Puppeteer::WaitTask
   private define_async_method :async_rerun
 
   WAIT_FOR_PREDICATE_PAGE_FUNCTION = <<~JAVASCRIPT
-  async function _(predicateBody, polling, timeout, ...args) {
+  async function _(root, predicateBody, polling, timeout, ...args) {
       const predicate = new Function('...args', predicateBody);
+      root = root || document
       let timedOut = false;
       if (timeout)
           setTimeout(() => (timedOut = true), timeout);
@@ -136,7 +139,7 @@ class Puppeteer::WaitTask
        * @return {!Promise<*>}
        */
       async function pollMutation() {
-          const success = await predicate(...args);
+          const success = await predicate(root, ...args);
           if (success) return Promise.resolve(success);
           let fulfill;
           const result = new Promise((x) => (fulfill = x));
@@ -145,13 +148,13 @@ class Puppeteer::WaitTask
                   observer.disconnect();
                   fulfill();
               }
-              const success = await predicate(...args);
+              const success = await predicate(root, ...args);
               if (success) {
                   observer.disconnect();
                   fulfill(success);
               }
           });
-          observer.observe(document, {
+          observer.observe(root, {
               childList: true,
               subtree: true,
               attributes: true,
@@ -168,7 +171,7 @@ class Puppeteer::WaitTask
                   fulfill();
                   return;
               }
-              const success = await predicate(...args);
+              const success = await predicate(root, ...args);
               if (success) fulfill(success);
               else requestAnimationFrame(onRaf);
           }
@@ -183,7 +186,7 @@ class Puppeteer::WaitTask
                   fulfill();
                   return;
               }
-              const success = await predicate(...args);
+              const success = await predicate(root, ...args);
               if (success) fulfill(success);
               else setTimeout(onTimeout, pollInterval);
           }

data/lib/puppeteer.rb

@@ -47,6 +47,7 @@ require 'puppeteer/launcher'
 require 'puppeteer/lifecycle_watcher'
 require 'puppeteer/mouse'
 require 'puppeteer/network_conditions'
+require 'puppeteer/network_event_manager'
 require 'puppeteer/network_manager'
 require 'puppeteer/page'
 require 'puppeteer/protocol_stream_reader'

data/puppeteer-ruby.gemspec

@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'concurrent-ruby', '~> 1.1.0'
   spec.add_dependency 'websocket-driver', '>= 0.6.0'
   spec.add_dependency 'mime-types', '>= 3.0'
-  spec.add_development_dependency 'bundler', '~> 2.2.3'
+  spec.add_development_dependency 'bundler', '~> 2.3.4'
   spec.add_development_dependency 'chunky_png'
   spec.add_development_dependency 'dry-inflector'
   spec.add_development_dependency 'pry-byebug'
@@ -32,7 +32,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rollbar'
   spec.add_development_dependency 'rspec', '~> 3.10.0 '
   spec.add_development_dependency 'rspec_junit_formatter' # for CircleCI.
-  spec.add_development_dependency 'rubocop', '~> 1.23.0'
+  spec.add_development_dependency 'rubocop', '~> 1.24.0'
   spec.add_development_dependency 'rubocop-rspec'
   spec.add_development_dependency 'sinatra'
   spec.add_development_dependency 'webrick'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puppeteer-ruby
 version: !ruby/object:Gem::Version
-  version: 0.39.0
+  version: 0.40.3
 platform: ruby
 authors:
 - YusukeIwaki
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-12-17 00:00:00.000000000 Z
+date: 2022-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -58,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.3
+        version: 2.3.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.2.3
+        version: 2.3.4
 - !ruby/object:Gem::Dependency
   name: chunky_png
   requirement: !ruby/object:Gem::Requirement
@@ -170,14 +170,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.23.0
+        version: 1.24.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.23.0
+        version: 1.24.0
 - !ruby/object:Gem::Dependency
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
@@ -306,6 +306,7 @@ files:
 - lib/puppeteer/mouse.rb
 - lib/puppeteer/network_condition.rb
 - lib/puppeteer/network_conditions.rb
+- lib/puppeteer/network_event_manager.rb
 - lib/puppeteer/network_manager.rb
 - lib/puppeteer/page.rb
 - lib/puppeteer/page/metrics.rb
@@ -346,7 +347,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.3.3
 signing_key: 
 specification_version: 4
 summary: A ruby port of puppeteer