puppeteer-ruby 0.38.0 → 0.40.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d18f0aae5e331fee42f66c8140bdc3cced9c0514a03ab65460c4036a94bc6eb2
-  data.tar.gz: babeb261ac3661a6738eb3d02dddeef22685590791fbd579483edd2da7800606
+  metadata.gz: e88033f07007a1841f99535a96497bef2aa35b58e0599d51886ec760ef8f8201
+  data.tar.gz: 3b4b69f023861cc24a4ecf402e2b6eff0cba0d57fbef37991996964a1b57007b
 SHA512:
-  metadata.gz: 0d70da4b6dc257f39d020286672180d93a4fe2aa321828035eeba80aeff92e7b1f28ff07acd15396d03aad33ab84b4f3474f1c75b587cb833d90f17dde4b024f
-  data.tar.gz: ccb8a58f1ffff28d1498c560c8f17671064ca00ce4b98af9dbc5e844ab1c77d9c28fc503c021ae019d9e799a287976fe01a88bbce3f87979d163f7d384336c0e
+  metadata.gz: 6e63bd3b526e4cc09f309fc206f9395618393ab104438e93eebc190e3efa36a7e12e3037388480b9dd735121f752cbbd0bd5dc1152ae39a57be6f855dc41b41f
+  data.tar.gz: fe6dff5b5b82813c003e67725868a467e2e9b43e16c4bf4bca1eb48c588daa2efd782d4c1c6b351117a3cfbf7daea4feb37b36fde3637abacea33cd2e3244cb4

data/.rubocop.yml

@@ -149,6 +149,8 @@ Style/MethodCallWithArgsParentheses:
     # utils
     - debug_print
     - debug_puts
+    - exit
+    - puts
 
 Style/MethodCallWithoutArgsParentheses:
   Enabled: true

data/CHANGELOG.md

@@ -1,7 +1,35 @@
-### main [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.38.0...main)]
+### main [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.2...main)]
 
 - xxx
 
+### 0.40.2 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.1...0.40.2)]
+
+Bugfix:
+
+- Fix crash on setting response listener [#195](https://github.com/YusukeIwaki/puppeteer-ruby/pull/195)
+
+### 0.40.1 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.40.0...0.40.1)]
+
+Bugfix:
+
+- Fix error on reload [#190](https://github.com/YusukeIwaki/puppeteer-ruby/pull/190)
+- Fix error message on failing to find Chrome executable.
+- Fix apidoc
+
+### 0.40.0 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.39.0...0.40.0)]
+
+New features:
+
+- Puppeteer 13.0, 13.1 functionalities
+- Support Ruby 3.1
+
+
+### 0.39.0 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.38.0...0.39.0)]
+
+New features:
+
+- Puppeteer 12.0 functionalities
+
 ### 0.38.0 [[diff](https://github.com/YusukeIwaki/puppeteer-ruby/compare/0.37.4...0.38.0)]
 
 New features:

data/docs/api_coverage.md

@@ -1,6 +1,6 @@
 # API coverages
-- Puppeteer version: v12.0.0
-- puppeteer-ruby version: 0.38.0
+- Puppeteer version: v13.0.1
+- puppeteer-ruby version: 0.40.2
 
 ## Puppeteer
 
@@ -296,6 +296,7 @@
 * ~~toString~~
 * type => `#type_text`
 * uploadFile => `#upload_file`
+* waitForSelector => `#wait_for_selector`
 
 ## HTTPRequest
 
@@ -309,6 +310,8 @@
 * frame
 * headers
 * initiator
+* ~~interceptResolutionState~~
+* ~~isInterceptResolutionHandled~~
 * isNavigationRequest => `#navigation_request?`
 * method
 * postData => `#post_data`

data/lib/puppeteer/aria_query_handler.rb

@@ -7,10 +7,10 @@ class Puppeteer::AriaQueryHandler
   private def parse_aria_selector(selector)
     known_attributes = %w(name role)
     query_options = {}
-    attribute_regexp = /\[\s*(?<attribute>\w+)\s*=\s*"(?<value>\\.|[^"\\]*)"\s*\]/
+    attribute_regexp = /\[\s*(?<attribute>\w+)\s*=\s*(?<quote>"|')(?<value>\\.|.*?(?=\k<quote>))\k<quote>\s*\]/
     default_name = selector.gsub(attribute_regexp) do
       attribute = $1.strip
-      value = $2
+      value = $3
       unless known_attributes.include?(attribute)
         raise ArgumentError.new("Unkown aria attribute \"#{attribute}\" in selector")
       end
@@ -36,10 +36,11 @@ class Puppeteer::AriaQueryHandler
     end
   end
 
-  def wait_for(dom_world, selector, visible: nil, hidden: nil, timeout: nil)
+  def wait_for(dom_world, selector, visible: nil, hidden: nil, timeout: nil, root: nil)
+    # addHandlerToWorld
     binding_function = Puppeteer::DOMWorld::BindingFunction.new(
       name: 'ariaQuerySelector',
-      proc: -> (selector) { query_one(dom_world.send(:document), selector) },
+      proc: -> (sel) { query_one(root || dom_world.send(:document), sel) },
     )
     dom_world.send(:wait_for_selector_in_page,
       '(_, selector) => globalThis.ariaQuerySelector(selector)',
@@ -47,7 +48,9 @@ class Puppeteer::AriaQueryHandler
       visible: visible,
       hidden: hidden,
       timeout: timeout,
-      binding_function: binding_function)
+      binding_function: binding_function,
+      root: root,
+    )
   end
 
   def query_all(element, selector)

data/lib/puppeteer/browser_runner.rb

@@ -4,13 +4,17 @@ require 'timeout'
 
 # https://github.com/puppeteer/puppeteer/blob/master/lib/Launcher.js
 class Puppeteer::BrowserRunner
+  include Puppeteer::DebugPrint
+
   # @param {string} executablePath
   # @param {!Array<string>} processArguments
   # @param {string=} tempDirectory
-  def initialize(executable_path, process_arguments, temp_directory)
+  def initialize(for_firefox, executable_path, process_arguments, user_data_dir, using_temp_user_data_dir)
+    @for_firefox = for_firefox
     @executable_path = executable_path
     @process_arguments = process_arguments
-    @temp_directory = temp_directory
+    @user_data_dir = user_data_dir
+    @using_temp_user_data_dir = using_temp_user_data_dir
     @proc = nil
     @connection = nil
     @closed = true
@@ -90,8 +94,8 @@ class Puppeteer::BrowserRunner
     @process_closing = -> {
       @proc.dispose
       @closed = true
-      if @temp_directory
-        FileUtils.rm_rf(@temp_directory)
+      if @using_temp_user_data_dir
+        FileUtils.rm_rf(@user_data_dir)
       end
     }
     at_exit do
@@ -122,7 +126,7 @@ class Puppeteer::BrowserRunner
   def close
     return if @closed
 
-    if @temp_directory
+    if @using_temp_user_data_dir && !@for_firefox
       kill
     elsif @connection
       begin
@@ -137,11 +141,18 @@ class Puppeteer::BrowserRunner
 
   # @return {Promise}
   def kill
-    if @temp_directory
-      FileUtils.rm_rf(@temp_directory)
-    end
-    unless @closed
-      @proc.kill
+    # If the process failed to launch (for example if the browser executable path
+    # is invalid), then the process does not get a pid assigned. A call to
+    # `proc.kill` would error, as the `pid` to-be-killed can not be found.
+    @proc&.kill
+
+    # Attempt to remove temporary profile directory to avoid littering.
+    begin
+      if @using_temp_user_data_dir
+        FileUtils.rm_rf(@user_data_dir)
+      end
+    rescue => err
+      debug_puts(err)
     end
   end
 

data/lib/puppeteer/custom_query_handler.rb

@@ -21,12 +21,12 @@ class Puppeteer::CustomQueryHandler
     nil
   end
 
-  def wait_for(dom_world, selector, visible: nil, hidden: nil, timeout: nil)
+  def wait_for(dom_world, selector, visible: nil, hidden: nil, timeout: nil, root: nil)
     unless @query_one
       raise NotImplementedError.new("#{self.class}##{__method__} is not implemented.")
     end
 
-    dom_world.send(:wait_for_selector_in_page, @query_one, selector, visible: visible, hidden: hidden, timeout: timeout)
+    dom_world.send(:wait_for_selector_in_page, @query_one, selector, visible: visible, hidden: hidden, timeout: timeout, root: root)
   end
 
   def query_all(element, selector)

data/lib/puppeteer/dom_world.rb

@@ -420,10 +420,10 @@ class Puppeteer::DOMWorld
   # @param visible [Boolean] Wait for element visible (not 'display: none' nor 'visibility: hidden') on true. default to false.
   # @param hidden [Boolean] Wait for element invisible ('display: none' nor 'visibility: hidden') on true. default to false.
   # @param timeout [Integer]
-  def wait_for_selector(selector, visible: nil, hidden: nil, timeout: nil)
+  def wait_for_selector(selector, visible: nil, hidden: nil, timeout: nil, root: nil)
     # call wait_for_selector_in_page with custom query selector.
     query_selector_manager = Puppeteer::QueryHandlerManager.instance
-    query_selector_manager.detect_query_handler(selector).wait_for(self, visible: visible, hidden: hidden, timeout: timeout)
+    query_selector_manager.detect_query_handler(selector).wait_for(self, visible: visible, hidden: hidden, timeout: timeout, root: root)
   end
 
   private def binding_identifier(name, context)
@@ -497,10 +497,11 @@ class Puppeteer::DOMWorld
   # @param visible [Boolean] Wait for element visible (not 'display: none' nor 'visibility: hidden') on true. default to false.
   # @param hidden [Boolean] Wait for element invisible ('display: none' nor 'visibility: hidden') on true. default to false.
   # @param timeout [Integer]
-  private def wait_for_selector_in_page(query_one, selector, visible: nil, hidden: nil, timeout: nil, binding_function: nil)
+  private def wait_for_selector_in_page(query_one, selector, visible: nil, hidden: nil, timeout: nil, root: nil, binding_function: nil)
     option_wait_for_visible = visible || false
     option_wait_for_hidden = hidden || false
     option_timeout = timeout || @timeout_settings.timeout
+    option_root = root
 
     polling =
       if option_wait_for_visible || option_wait_for_hidden
@@ -511,11 +512,11 @@ class Puppeteer::DOMWorld
     title = "selector #{selector}#{option_wait_for_hidden ? 'to be hidden' : ''}"
 
     selector_predicate = make_predicate_string(
-      predicate_arg_def: '(selector, waitForVisible, waitForHidden)',
+      predicate_arg_def: '(root, selector, waitForVisible, waitForHidden)',
       predicate_query_handler: query_one,
       async: true,
       predicate_body: <<~JAVASCRIPT
-        const node = await predicateQueryHandler(document, selector)
+        const node = await predicateQueryHandler(root, selector)
         return checkWaitForOptions(node, waitForVisible, waitForHidden);
       JAVASCRIPT
     )
@@ -527,6 +528,7 @@ class Puppeteer::DOMWorld
       polling: polling,
       timeout: option_timeout,
       args: [selector, option_wait_for_visible, option_wait_for_hidden],
+      root: option_root,
       binding_function: binding_function,
     )
     handle = wait_task.await_promise
@@ -541,10 +543,11 @@ class Puppeteer::DOMWorld
   # @param visible [Boolean] Wait for element visible (not 'display: none' nor 'visibility: hidden') on true. default to false.
   # @param hidden [Boolean] Wait for element invisible ('display: none' nor 'visibility: hidden') on true. default to false.
   # @param timeout [Integer]
-  def wait_for_xpath(xpath, visible: nil, hidden: nil, timeout: nil)
+  def wait_for_xpath(xpath, visible: nil, hidden: nil, timeout: nil, root: nil)
     option_wait_for_visible = visible || false
     option_wait_for_hidden = hidden || false
     option_timeout = timeout || @timeout_settings.timeout
+    option_root = root
 
     polling =
       if option_wait_for_visible || option_wait_for_hidden
@@ -555,9 +558,9 @@ class Puppeteer::DOMWorld
     title = "XPath #{xpath}#{option_wait_for_hidden ? 'to be hidden' : ''}"
 
     xpath_predicate = make_predicate_string(
-      predicate_arg_def: '(selector, waitForVisible, waitForHidden)',
+      predicate_arg_def: '(root, selector, waitForVisible, waitForHidden)',
       predicate_body: <<~JAVASCRIPT
-        const node = document.evaluate(selector, document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
+        const node = document.evaluate(selector, root, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
         return checkWaitForOptions(node, waitForVisible, waitForHidden);
       JAVASCRIPT
     )
@@ -569,6 +572,7 @@ class Puppeteer::DOMWorld
       polling: polling,
       timeout: option_timeout,
       args: [xpath, option_wait_for_visible, option_wait_for_hidden],
+      root: option_root,
     )
     handle = wait_task.await_promise
     unless handle.as_element

data/lib/puppeteer/element_handle.rb

@@ -20,6 +20,60 @@ class Puppeteer::ElementHandle < Puppeteer::JSHandle
     @disposed = false
   end
 
+  def inspect
+    values = %i[context remote_object page disposed].map do |sym|
+      value = instance_variable_get(:"@#{sym}")
+      "@#{sym}=#{value}"
+    end
+    "#<Puppeteer::ElementHandle #{values.join(' ')}>"
+  end
+
+  #
+  # Wait for the `selector` to appear within the element. If at the moment of calling the
+  # method the `selector` already exists, the method will return immediately. If
+  # the `selector` doesn't appear after the `timeout` milliseconds of waiting, the
+  # function will throw.
+  #
+  # This method does not work across navigations or if the element is detached from DOM.
+  #
+  # @param selector - A
+  # {@link https://developer.mozilla.org/en-US/docs/Web/CSS/CSS_Selectors | selector}
+  # of an element to wait for
+  # @param options - Optional waiting parameters
+  # @returns Promise which resolves when element specified by selector string
+  # is added to DOM. Resolves to `null` if waiting for hidden: `true` and
+  # selector is not found in DOM.
+  # @remarks
+  # The optional parameters in `options` are:
+  #
+  # - `visible`: wait for the selected element to be present in DOM and to be
+  # visible, i.e. to not have `display: none` or `visibility: hidden` CSS
+  # properties. Defaults to `false`.
+  #
+  # - `hidden`: wait for the selected element to not be found in the DOM or to be hidden,
+  # i.e. have `display: none` or `visibility: hidden` CSS properties. Defaults to
+  # `false`.
+  #
+  # - `timeout`: maximum time to wait in milliseconds. Defaults to `30000`
+  # (30 seconds). Pass `0` to disable timeout. The default value can be changed
+  # by using the {@link Page.setDefaultTimeout} method.
+  def wait_for_selector(selector, visible: nil, hidden: nil, timeout: nil)
+    frame = @context.frame
+
+    secondary_world = frame.secondary_world
+    adopted_root = secondary_world.execution_context.adopt_element_handle(self)
+    handle = secondary_world.wait_for_selector(selector, visible: visible, hidden: hidden, timeout: timeout, root: adopted_root)
+    adopted_root.dispose
+    return nil unless handle
+
+    main_world = frame.main_world
+    result = main_world.execution_context.adopt_element_handle(handle)
+    handle.dispose
+    result
+  end
+
+  define_async_method :async_wait_for_selector
+
   def as_element
     self
   end

data/lib/puppeteer/events.rb

@@ -81,6 +81,7 @@ module NetworkManagerEmittedEvents ; end
 
 {
   Request: EventsDefinitionUtils.symbol('NetworkManager.Request'),
+  RequestServedFromCache: EventsDefinitionUtils.symbol('NetworkManager.RequestServedFromCache'),
   Response: EventsDefinitionUtils.symbol('NetworkManager.Response'),
   RequestFailed: EventsDefinitionUtils.symbol('NetworkManager.RequestFailed'),
   RequestFinished: EventsDefinitionUtils.symbol('NetworkManager.RequestFinished'),

data/lib/puppeteer/frame_manager.rb

@@ -382,7 +382,7 @@ class Puppeteer::FrameManager
       @isolated_worlds << context_payload['name']
     end
 
-    context = Puppeteer::ExecutionContext.new(frame._client || @client, context_payload, world)
+    context = Puppeteer::ExecutionContext.new(frame&._client || @client, context_payload, world)
     if world
       world.context = context
     end

data/lib/puppeteer/http_response.rb

@@ -31,7 +31,8 @@ class Puppeteer::HTTPResponse
   # @param client [Puppeteer::CDPSession]
   # @param request [Puppeteer::HTTPRequest]
   # @param response_payload [Hash]
-  def initialize(client, request, response_payload)
+  # @param extra_info [Hash|nil]
+  def initialize(client, request, response_payload, extra_info)
     @client = client
     @request = request
 
@@ -41,14 +42,15 @@ class Puppeteer::HTTPResponse
       port: response_payload['remotePort'],
     )
 
-    @status = response_payload['status']
-    @status_text = response_payload['statusText']
+    @status_text = parse_štatus_text_from_extra_info(extra_info) || response_payload['statusText']
     @url = request.url
     @from_disk_cache = !!response_payload['fromDiskCache']
     @from_service_worker = !!response_payload['fromServiceWorker']
 
+    @status = extra_info ? extra_info['statusCode'] : response_payload['status']
     @headers = {}
-    response_payload['headers'].each do |key, value|
+    headers = extra_info ? extra_info['headers'] : response_payload['headers']
+    headers.each do |key, value|
       @headers[key.downcase] = value
     end
     @security_details = if_present(response_payload['securityDetails']) do |security_payload|
@@ -62,6 +64,25 @@ class Puppeteer::HTTPResponse
 
   attr_reader :remote_address, :url, :status, :status_text, :headers, :security_details, :request
 
+  def inspect
+    values = %i[remote_address url status status_text headers security_details request].map do |sym|
+      value = instance_variable_get(:"@#{sym}")
+      "@#{sym}=#{value}"
+    end
+    "#<Puppeteer::HTTPRequest #{values.join(' ')}>"
+  end
+
+  private def parse_štatus_text_from_extra_info(extra_info)
+    return nil if !extra_info || !extra_info['headersText']
+    first_line = extra_info['headersText'].split("\r").first
+    return nil unless first_line
+    /[^ ]* [^ ]* (.*)/.match(first_line) do |m|
+      return m[1]
+    end
+
+    nil
+  end
+
   # @return [Boolean]
   def ok?
     @status == 0 || (@status >= 200 && @status <= 299)

data/lib/puppeteer/js_handle.rb

@@ -36,6 +36,14 @@ class Puppeteer::JSHandle
 
   attr_reader :context, :remote_object
 
+  def inspect
+    values = %i[context remote_object disposed].map do |sym|
+      value = instance_variable_get(:"@#{sym}")
+      "@#{sym}=#{value}"
+    end
+    "#<Puppeteer::JSHandle #{values.join(' ')}>"
+  end
+
   # @return [Puppeteer::ExecutionContext]
   def execution_context
     @context

data/lib/puppeteer/launcher/chrome.rb

@@ -27,9 +27,6 @@ module Puppeteer::Launcher
           @chrome_arg_options.args.dup
         end
 
-      #
-      # let temporaryUserDataDir = null;
-
       if chrome_arguments.none? { |arg| arg.start_with?('--remote-debugging-') }
         if @launch_options.pipe?
           chrome_arguments << '--remote-debugging-pipe'
@@ -38,10 +35,17 @@ module Puppeteer::Launcher
         end
       end
 
-      temporary_user_data_dir = nil
-      if chrome_arguments.none? { |arg| arg.start_with?('--user-data-dir') }
-        temporary_user_data_dir = Dir.mktmpdir('puppeteer_dev_chrome_profile-', ENV['PUPPETEER_TMP_DIR'])
-        chrome_arguments << "--user-data-dir=#{temporary_user_data_dir}"
+      user_data_dir = chrome_arguments.find { |arg| arg.start_with?('--user-data-dir') }
+      if user_data_dir
+        user_data_dir = user_data_dir.split('=').last
+        unless File.exist?(user_data_dir)
+          raise ArgumentError.new("Chrome user data dir not found at '#{user_data_dir}'")
+        end
+        using_temp_user_data_dir = false
+      else
+        user_data_dir = Dir.mktmpdir('puppeteer_dev_chrome_profile-', ENV['PUPPETEER_TMP_DIR'])
+        chrome_arguments << "--user-data-dir=#{user_data_dir}"
+        using_temp_user_data_dir = true
       end
 
       chrome_executable =
@@ -51,7 +55,13 @@ module Puppeteer::Launcher
           @launch_options.executable_path || fallback_executable_path
         end
       use_pipe = chrome_arguments.include?('--remote-debugging-pipe')
-      runner = Puppeteer::BrowserRunner.new(chrome_executable, chrome_arguments, temporary_user_data_dir)
+      runner = Puppeteer::BrowserRunner.new(
+        false,
+        chrome_executable,
+        chrome_arguments,
+        user_data_dir,
+        using_temp_user_data_dir,
+      )
       runner.start(
         handle_SIGHUP: @launch_options.handle_SIGHUP?,
         handle_SIGTERM: @launch_options.handle_SIGTERM?,
@@ -61,33 +71,39 @@ module Puppeteer::Launcher
         pipe: use_pipe,
       )
 
-      begin
-        connection = runner.setup_connection(
-          use_pipe: use_pipe,
-          timeout: @launch_options.timeout,
-          slow_mo: @browser_options.slow_mo,
-          preferred_revision: @preferred_revision,
-        )
-
-        browser = Puppeteer::Browser.create(
-          connection: connection,
-          context_ids: [],
-          ignore_https_errors: @browser_options.ignore_https_errors?,
-          default_viewport: @browser_options.default_viewport,
-          process: runner.proc,
-          close_callback: -> { runner.close },
-        )
+      browser =
+        begin
+          connection = runner.setup_connection(
+            use_pipe: use_pipe,
+            timeout: @launch_options.timeout,
+            slow_mo: @browser_options.slow_mo,
+            preferred_revision: @preferred_revision,
+          )
+
+          Puppeteer::Browser.create(
+            connection: connection,
+            context_ids: [],
+            ignore_https_errors: @browser_options.ignore_https_errors?,
+            default_viewport: @browser_options.default_viewport,
+            process: runner.proc,
+            close_callback: -> { runner.close },
+          )
+        rescue
+          runner.kill
+          raise
+        end
 
+      begin
         browser.wait_for_target(
           predicate: ->(target) { target.type == 'page' },
           timeout: @launch_options.timeout,
         )
-
-        browser
       rescue
-        runner.kill
+        browser.close
         raise
       end
+
+      browser
     end
 
     class DefaultArgs
@@ -270,15 +286,15 @@ module Puppeteer::Launcher
         end
 
       chrome_path = chrome_path_map[channel]
-      if chrome_path.is_a?(Proc)
-        chrome_path = chrome_path.call
-      end
-
       unless chrome_path
         raise ArgumentError.new("Invalid channel: '#{channel}'. Allowed channel is #{chrome_path_map.keys}")
       end
 
-      unless File.exist?(chrome_path)
+      if chrome_path.is_a?(Proc)
+        chrome_path = chrome_path.call
+      end
+
+      if !chrome_path || !File.exist?(chrome_path)
         raise "#{channel} is not installed on this system.\nExpected path: #{chrome_path}"
       end