puppeteer-ruby 0.45.6 → 0.50.0.alpha5

data/lib/puppeteer/frame_manager.rb

@@ -1,5 +1,3 @@
-require 'timeout'
-
 class Puppeteer::FrameManager
   include Puppeteer::DebugPrint
   include Puppeteer::IfPresent
@@ -12,10 +10,11 @@ class Puppeteer::FrameManager
   # @param {!Puppeteer.Page} page
   # @param {boolean} ignoreHTTPSErrors
   # @param {!Puppeteer.TimeoutSettings} timeoutSettings
-  def initialize(client, page, ignore_https_errors, timeout_settings)
+  # @param {boolean} network_enabled
+  def initialize(client, page, ignore_https_errors, timeout_settings, network_enabled: true)
     @client = client
     @page = page
-    @network_manager = Puppeteer::NetworkManager.new(client, ignore_https_errors, self)
+    @network_manager = Puppeteer::NetworkManager.new(client, ignore_https_errors, self, network_enabled: network_enabled)
     @timeout_settings = timeout_settings
 
     # @type {!Map<string, !Frame>}
@@ -76,17 +75,17 @@ class Puppeteer::FrameManager
   attr_reader :client, :timeout_settings
 
   private def init(target_id, cdp_session = nil)
-    @frames_pending_target_init[target_id] ||= resolvable_future
+    @frames_pending_target_init[target_id] ||= Async::Promise.new
     client = cdp_session || @client
 
     promises = [
       client.async_send_message('Page.enable'),
       client.async_send_message('Page.getFrameTree'),
     ].compact
-    results = await_all(*promises)
+    results = Puppeteer::AsyncUtils.await_promise_all(*promises)
     frame_tree = results[1]['frameTree']
     handle_frame_tree(client, frame_tree)
-    await_all(
+    Puppeteer::AsyncUtils.await_promise_all(
       client.async_send_message('Page.setLifecycleEventsEnabled', enabled: true),
       client.async_send_message('Runtime.enable'),
     )
@@ -98,25 +97,28 @@ class Puppeteer::FrameManager
 
     raise
   ensure
-    @frames_pending_target_init.delete(target_id)&.fulfill(nil)
+    @frames_pending_target_init.delete(target_id)&.resolve(nil)
   end
 
   define_async_method :async_init
 
   attr_reader :network_manager
 
-  class NavigationError < StandardError; end
+  class NavigationError < Puppeteer::Error; end
 
   # @param frame [Puppeteer::Frame]
   # @param url [String]
-  # @param {!{referer?: string, timeout?: number, waitUntil?: string|!Array<string>}=} options
+  # @param {!{referer?: string, referrerPolicy?: string, timeout?: number, waitUntil?: string|!Array<string>}=} options
   # @return [Puppeteer::HTTPResponse]
-  def navigate_frame(frame, url, referer: nil, timeout: nil, wait_until: nil)
+  def navigate_frame(frame, url, referer: nil, referrer_policy: nil, timeout: nil, wait_until: nil)
     assert_no_legacy_navigation_options(wait_until: wait_until)
 
+    referrer_policy ||= @network_manager.extra_http_headers['referer-policy']
+    protocol_referrer_policy = referrer_policy_to_protocol(referrer_policy)
     navigate_params = {
       url: url,
-      referer: referer || @network_manager.extra_http_headers['referer'],
+      referrer: referer || @network_manager.extra_http_headers['referer'],
+      referrerPolicy: protocol_referrer_policy,
       frameId: frame.id,
     }.compact
     option_wait_until = wait_until || ['load']
@@ -126,20 +128,22 @@ class Puppeteer::FrameManager
     ensure_new_document_navigation = false
 
     begin
-      navigate = future do
-        result = @client.send_message('Page.navigate', navigate_params)
-        loader_id = result['loaderId']
-        ensure_new_document_navigation = !!loader_id
-        if result['errorText']
-          raise NavigationError.new("#{result['errorText']} at #{url}")
-        end
+      navigate = Async do
+        Puppeteer::AsyncUtils.future_with_logging do
+          result = @client.send_message('Page.navigate', navigate_params)
+          loader_id = result['loaderId']
+          ensure_new_document_navigation = !!loader_id
+          if result['errorText'] && result['errorText'] != 'net::ERR_HTTP_RESPONSE_CODE_FAILURE'
+            raise NavigationError.new("#{result['errorText']} at #{url}")
+          end
+        end.call
       end
-      await_any(
+      Puppeteer::AsyncUtils.await_promise_race(
         navigate,
         watcher.timeout_or_termination_promise,
       )
 
-      await_any(
+      Puppeteer::AsyncUtils.await_promise_race(
         watcher.timeout_or_termination_promise,
         if ensure_new_document_navigation
           watcher.new_document_navigation_promise
@@ -150,7 +154,7 @@ class Puppeteer::FrameManager
 
       watcher.navigation_response
     rescue Puppeteer::TimeoutError => err
-      raise NavigationError.new(err)
+      raise err
     ensure
       watcher.dispose
     end
@@ -159,22 +163,29 @@ class Puppeteer::FrameManager
   # @param timeout [number|nil]
   # @param wait_until [string|nil] 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2'
   # @return [Puppeteer::HTTPResponse]
-  def wait_for_frame_navigation(frame, timeout: nil, wait_until: nil)
+  def wait_for_frame_navigation(frame, timeout: nil, wait_until: nil, ignore_same_document_navigation: false)
     assert_no_legacy_navigation_options(wait_until: wait_until)
 
     option_wait_until = wait_until || ['load']
     option_timeout = timeout || @timeout_settings.navigation_timeout
     watcher = Puppeteer::LifecycleWatcher.new(self, frame, option_wait_until, option_timeout)
     begin
-      await_any(
-        watcher.timeout_or_termination_promise,
-        watcher.same_document_navigation_promise,
-        watcher.new_document_navigation_promise,
-      )
+      if ignore_same_document_navigation
+        Puppeteer::AsyncUtils.await_promise_race(
+          watcher.timeout_or_termination_promise,
+          watcher.new_document_navigation_promise,
+        )
+      else
+        Puppeteer::AsyncUtils.await_promise_race(
+          watcher.timeout_or_termination_promise,
+          watcher.same_document_navigation_promise,
+          watcher.new_document_navigation_promise,
+        )
+      end
 
       watcher.navigation_response
     rescue Puppeteer::TimeoutError => err
-      raise NavigationError.new(err)
+      raise err
     ensure
       watcher.dispose
     end
@@ -282,10 +293,11 @@ class Puppeteer::FrameManager
     end
 
     if @frames_pending_target_init[parent_frame_id]
-      @frames_pending_attachment[frame_id] ||= resolvable_future
-      @frames_pending_target_init[parent_frame_id].then do |_|
+      @frames_pending_attachment[frame_id] ||= Async::Promise.new
+      Async do
+        @frames_pending_target_init[parent_frame_id].wait
         attach_child_frame(@frames[parent_frame_id], parent_frame_id, frame_id, session)
-        @frames_pending_attachment.delete(frame_id)&.fulfill(nil)
+        @frames_pending_attachment.delete(frame_id)&.resolve(nil)
       end
       return
     end
@@ -293,7 +305,7 @@ class Puppeteer::FrameManager
     raise FrameNotFoundError.new("Parent frame #{parent_frame_id} not found.")
   end
 
-  class FrameNotFoundError < StandardError ; end
+  class FrameNotFoundError < Puppeteer::Error ; end
 
   private def attach_child_frame(parent_frame, parent_frame_id, frame_id, session)
     unless parent_frame
@@ -313,7 +325,8 @@ class Puppeteer::FrameManager
 
 
     if @frames_pending_attachment[frame_id]
-      @frames_pending_attachment[frame_id].then do |_|
+      Async do
+        @frames_pending_attachment[frame_id].wait
         frame = is_main_frame ? @main_frame : @frames[frame_id]
         reattach_frame(frame, frame_id, is_main_frame, frame_payload)
       end
@@ -360,7 +373,6 @@ class Puppeteer::FrameManager
   private def ensure_isolated_world(session, name)
     key = "#{session.id}:#{name}"
     return if @isolated_worlds.include?(key)
-    @isolated_worlds << key
 
     session.send_message('Page.addScriptToEvaluateOnNewDocument',
       source: "//# sourceURL=#{Puppeteer::ExecutionContext::EVALUATION_SCRIPT_URL}",
@@ -369,13 +381,18 @@ class Puppeteer::FrameManager
     create_isolated_worlds_promises = frames.
       select { |frame| frame._client == session }.
       map do |frame|
-        session.async_send_message('Page.createIsolatedWorld',
-          frameId: frame.id,
-          grantUniveralAccess: true,
-          worldName: name,
-        )
+        Async do
+          session.send_message('Page.createIsolatedWorld',
+            frameId: frame.id,
+            grantUniveralAccess: true,
+            worldName: name,
+          )
+        rescue => err
+          debug_puts(err)
+        end
       end
-    await_all(*create_isolated_worlds_promises)
+    Puppeteer::AsyncUtils.await_promise_all(*create_isolated_worlds_promises)
+    @isolated_worlds << key
   end
 
   # @param frame_id [String]
@@ -445,22 +462,20 @@ class Puppeteer::FrameManager
     context = @context_id_to_context[key]
     return unless context
     @context_id_to_context.delete(key)
-    if context.world
-      context.world.delete_context(execution_context_id)
-    end
+    context.world&.delete_context(context)
   end
 
   # @param session [Puppeteer::CDPSession]
   def handle_execution_contexts_cleared(session)
+    session_id = session.id
     @context_id_to_context.select! do |execution_context_id, context|
+      key_session_id, context_id = execution_context_id.split(':', 2)
       # Make sure to only clear execution contexts that belong
       # to the current session.
-      if context.client != session
+      if key_session_id != session_id
         true # keep
       else
-        if context.world
-          context.world.delete_context(execution_context_id)
-        end
+        context.world&.delete_context(context)
         false # remove
       end
     end
@@ -481,6 +496,12 @@ class Puppeteer::FrameManager
     emit_event(FrameManagerEmittedEvents::FrameDetached, frame)
   end
 
+  private def referrer_policy_to_protocol(referrer_policy)
+    return nil if referrer_policy.nil?
+
+    referrer_policy.to_s.gsub(/-([a-z])/) { Regexp.last_match(1).upcase }
+  end
+
   private def assert_no_legacy_navigation_options(wait_until:)
     if wait_until == 'networkidle'
       raise ArgumentError.new('ERROR: "networkidle" option is no longer supported. Use "networkidle2" instead')

data/lib/puppeteer/http_request.rb

@@ -1,3 +1,5 @@
+# rbs_inline: enabled
+
 class Puppeteer::HTTPRequest
   include Puppeteer::DebugPrint
   include Puppeteer::IfPresent
@@ -14,6 +16,10 @@ class Puppeteer::HTTPRequest
       @request.instance_variable_get(:@request_id)
     end
 
+    def client=(client)
+      @request.instance_variable_set(:@client, client)
+    end
+
     def interception_id
       @request.instance_variable_get(:@interception_id)
     end
@@ -90,27 +96,55 @@ class Puppeteer::HTTPRequest
     @interception_id = interception_id
     @allow_interception = allow_interception
     @url = event['request']['url']
-    @resource_type = event['type'].downcase
+    url_fragment = event.dig('request', 'urlFragment')
+    @url += url_fragment if url_fragment
+    resource_type = event['type'] || event['resourceType'] || 'other'
+    @resource_type = resource_type.downcase
     @method = event['request']['method']
     @post_data = event['request']['postData']
+    has_post_data = event.dig('request', 'hasPostData')
+    @has_post_data = has_post_data.nil? ? !@post_data.nil? : has_post_data
     @frame = frame
     @redirect_chain = redirect_chain
     @continue_request_overrides = {}
     @intercept_resolution_state = InterceptResolutionState.none
+    @interception_handled = false
     @intercept_handlers = []
     @initiator = event['initiator']
 
     @headers = {}
-    event['request']['headers'].each do |key, value|
-      @headers[key.downcase] = value
-    end
+    update_headers(event.dig('request', 'headers') || {})
     @from_memory_cache = false
 
     @internal = InternalAccessor.new(self)
   end
 
   attr_reader :internal
-  attr_reader :url, :resource_type, :method, :post_data, :headers, :response, :frame, :initiator
+  attr_reader :client, :url, :resource_type, :method, :post_data, :response, :frame, :initiator
+
+  def update_headers(headers)
+    headers.each do |key, value|
+      @headers[key.downcase] = value
+    end
+  end
+
+  def headers
+    @headers.dup
+  end
+
+  # @rbs return: bool -- Whether request has post data
+  def has_post_data?
+    @has_post_data
+  end
+
+  # @rbs return: String? -- Post data string if available
+  def fetch_post_data
+    response = @client.send_message('Network.getRequestPostData', requestId: @request_id)
+    response['postData']
+  rescue => err
+    debug_puts(err)
+    nil
+  end
 
   def inspect
     values = %i[request_id method url].map do |sym|
@@ -132,6 +166,10 @@ class Puppeteer::HTTPRequest
     end
   end
 
+  private def can_be_intercepted?
+    !@url.start_with?('data:') && !@from_memory_cache
+  end
+
   # @returns the `ContinueRequestOverrides` that will be used
   # if the interception is allowed to continue (ie, `abort()` and
   # `respond()` aren't called).
@@ -189,11 +227,15 @@ class Puppeteer::HTTPRequest
   # Awaits pending interception handlers and then decides how to fulfill
   # the request interception.
   def finalize_interceptions
-    @intercept_handlers.each(&:call)
+    @intercept_handlers.each do |handler|
+      Puppeteer::AsyncUtils.await(handler.call)
+    end
+    @intercept_handlers = []
     case intercept_resolution_state.action
     when 'abort'
-      abort_impl(**@abort_error_reason)
+      abort_impl(@abort_error_reason)
     when 'respond'
+      raise "Response is missing for the interception" if @response_for_request.nil?
       respond_impl(**@response_for_request)
     when 'continue'
       continue_impl(@continue_request_overrides)
@@ -218,23 +260,26 @@ class Puppeteer::HTTPRequest
     return nil unless headers
 
     headers.flat_map do |key, value|
-      if value.is_a?(Enumerable)
-        value.map do |v|
-          { name: key, value: v.to_s }
+      next [] if value.nil?
+
+      name = key.to_s
+      if value.is_a?(Array)
+        value.compact.map do |v|
+          { name: name, value: v.to_s }
         end
       else
-        { name: key, value: value.to_s }
+        { name: name, value: value.to_s }
       end
     end
   end
 
-  class InterceptionNotEnabledError < StandardError
+  class InterceptionNotEnabledError < Puppeteer::Error
     def initialize
       super('Request Interception is not enabled!')
     end
   end
 
-  class AlreadyHandledError < StandardError
+  class AlreadyHandledError < Puppeteer::Error
     def initialize
       super('Request is already handled!')
     end
@@ -255,17 +300,15 @@ class Puppeteer::HTTPRequest
   #
   # @param error_code [String|Symbol]
   def continue(url: nil, method: nil, post_data: nil, headers: nil, priority: nil)
-    # Request interception is not supported for data: urls.
-    return if @url.start_with?('data:')
-
     assert_interception_allowed
     assert_interception_not_handled
+    return unless can_be_intercepted?
 
     overrides = {
       url: url,
       method: method,
-      postData: post_data,
-      headers: headers_to_array(headers),
+      post_data: post_data,
+      headers: headers,
     }.compact
 
     if priority.nil?
@@ -280,7 +323,7 @@ class Puppeteer::HTTPRequest
     end
 
     if priority == @intercept_resolution_state.priority
-      if @intercept_resolution_state.action == :abort || @intercept_resolution_state.action == :respond
+      if @intercept_resolution_state.action == 'abort' || @intercept_resolution_state.action == 'respond'
         return
       end
       @intercept_resolution_state = InterceptResolutionState.continue(priority: priority)
@@ -291,14 +334,24 @@ class Puppeteer::HTTPRequest
     @interception_handled = true
 
     begin
+      raise Puppeteer::Error.new('HTTPRequest is missing interception id needed for Fetch.continueRequest') if @interception_id.nil?
+
+      post_data = overrides[:post_data]
+      overrides = overrides.merge(
+        postData: post_data ? Base64.strict_encode64(post_data.b) : nil,
+        headers: headers_to_array(overrides[:headers]),
+      ).compact
+      overrides.delete(:post_data)
+
       @client.send_message('Fetch.continueRequest',
         requestId: @interception_id,
         **overrides,
       )
     rescue => err
+      @interception_handled = false unless target_closed_error?(err)
       # In certain cases, protocol will return error if the request was already canceled
       # or the page was closed. We should tolerate these errors.
-      debug_puts(err)
+      handle_interception_error(err)
     end
   end
 
@@ -319,11 +372,9 @@ class Puppeteer::HTTPRequest
   # @param content_type [String]
   # @param body [String]
   def respond(status: nil, headers: nil, content_type: nil, body: nil, priority: nil)
-    # Mocking responses for dataURL requests is not currently supported.
-    return if @url.start_with?('data:')
-
     assert_interception_allowed
     assert_interception_not_handled
+    return unless can_be_intercepted?
 
     if priority.nil?
       respond_impl(status: status, headers: headers, content_type: content_type, body: body)
@@ -335,7 +386,7 @@ class Puppeteer::HTTPRequest
       headers: headers,
       content_type: content_type,
       body: body,
-    }
+    }.compact
 
     if @intercept_resolution_state.priority_unspecified? || priority > @intercept_resolution_state.priority
       @intercept_resolution_state = InterceptResolutionState.respond(priority: priority)
@@ -343,7 +394,7 @@ class Puppeteer::HTTPRequest
     end
 
     if priority == @intercept_resolution_state.priority
-      if @intercept_resolution_state.action == :abort
+      if @intercept_resolution_state.action == 'abort'
         return
       end
       @intercept_resolution_state = InterceptResolutionState.respond(priority: priority)
@@ -353,10 +404,18 @@ class Puppeteer::HTTPRequest
   private def respond_impl(status: nil, headers: nil, content_type: nil, body: nil)
     @interception_handled = true
 
+    parsed_body = if body
+      binary = body.to_s.b
+      {
+        content_length: binary.bytesize,
+        base64: Base64.strict_encode64(binary),
+      }
+    end
+
     mock_response_headers = {}
     headers&.each do |key, value|
-      mock_response_headers[key.downcase] =
-        if value.is_a?(Enumerable)
+      mock_response_headers[key.to_s.downcase] =
+        if value.is_a?(Array)
           value.map(&:to_s)
         else
           value.to_s
@@ -365,26 +424,29 @@ class Puppeteer::HTTPRequest
     if content_type
       mock_response_headers['content-type'] = content_type
     end
-    if body
-      mock_response_headers['content-length'] = body.length
+    if parsed_body && parsed_body[:content_length] > 0 && !mock_response_headers.key?('content-length')
+      mock_response_headers['content-length'] = parsed_body[:content_length].to_s
     end
 
     mock_response = {
       responseCode: status || 200,
       responsePhrase: STATUS_TEXTS[(status || 200).to_s],
       responseHeaders: headers_to_array(mock_response_headers),
-      body: if_present(body) { |mock_body| Base64.strict_encode64(mock_body) },
+      body: parsed_body && parsed_body[:base64],
     }.compact
 
     begin
+      raise Puppeteer::Error.new('HTTPRequest is missing interception id needed for Fetch.fulfillRequest') if @interception_id.nil?
+
       @client.send_message('Fetch.fulfillRequest',
         requestId: @interception_id,
         **mock_response,
       )
     rescue => err
+      @interception_handled = false unless target_closed_error?(err)
       # In certain cases, protocol will return error if the request was already canceled
       # or the page was closed. We should tolerate these errors.
-      debug_puts(err)
+      handle_interception_error(err)
     end
   end
 
@@ -402,22 +464,20 @@ class Puppeteer::HTTPRequest
   #
   # @param error_code [String|Symbol]
   def abort(error_code: :failed, priority: nil)
-    # Request interception is not supported for data: urls.
-    return if @url.start_with?('data:')
-
     error_reason = ERROR_REASONS[error_code.to_s]
     unless error_reason
       raise ArgumentError.new("Unknown error code: #{error_code}")
     end
     assert_interception_allowed
     assert_interception_not_handled
+    return unless can_be_intercepted?
 
     if priority.nil?
-      abort_impl(error_reason)
+      return abort_impl(error_reason)
     end
     @abort_error_reason = error_reason
 
-    if @intercept_resolution_state.priority_unspecified? || priority > @intercept_resolution_state.priority
+    if @intercept_resolution_state.priority_unspecified? || priority >= @intercept_resolution_state.priority
       @intercept_resolution_state = InterceptResolutionState.abort(priority: priority)
     end
   end
@@ -426,15 +486,31 @@ class Puppeteer::HTTPRequest
     @interception_handled = true
 
     begin
+      raise Puppeteer::Error.new('HTTPRequest is missing interception id needed for Fetch.failRequest') if @interception_id.nil?
+
       @client.send_message('Fetch.failRequest',
         requestId: @interception_id,
-        errorReason: error_reason,
+        errorReason: error_reason || ERROR_REASONS.fetch('failed'),
       )
     rescue => err
+      @interception_handled = false unless target_closed_error?(err)
       # In certain cases, protocol will return error if the request was already canceled
       # or the page was closed. We should tolerate these errors.
-      debug_puts(err)
+      handle_interception_error(err)
+    end
+  end
+
+  private def target_closed_error?(error)
+    message = error.message.to_s
+    message.match?(/Target closed|Session closed|Connection closed/i)
+  end
+
+  private def handle_interception_error(error)
+    message = error.message.to_s
+    if message.match?(/Invalid header|Unsafe header|Expected "header"|invalid argument/i)
+      raise error
     end
+    debug_puts(error)
   end
 
   ERROR_REASONS = {