puppeteer-ruby 0.0.13 → 0.0.18

data/lib/puppeteer/frame_manager.rb

@@ -4,7 +4,7 @@ class Puppeteer::FrameManager
   include Puppeteer::DebugPrint
   include Puppeteer::IfPresent
   include Puppeteer::EventCallbackable
-  using Puppeteer::AsyncAwaitBehavior
+  using Puppeteer::DefineAsyncMethod
 
   UTILITY_WORLD_NAME = '__puppeteer_utility_world__'
 
@@ -23,7 +23,6 @@ class Puppeteer::FrameManager
 
     # @type {!Map<number, !ExecutionContext>}
     @context_id_to_context = {}
-    @context_id_created = {}
 
     # @type {!Set<string>}
     @isolated_worlds = Set.new
@@ -74,9 +73,7 @@ class Puppeteer::FrameManager
     @network_manager.init
   end
 
-  async def async_init
-    init
-  end
+  define_async_method :async_init
 
   attr_reader :network_manager
 
@@ -281,7 +278,7 @@ class Puppeteer::FrameManager
   # @param url [String]
   def handle_frame_navigated_within_document(frame_id, url)
     frame = @frames[frame_id]
-    return if !frame
+    return unless frame
     frame.navigated_within_document(url)
     emit_event 'Events.FrameManager.FrameNavigatedWithinDocument', frame
     emit_event 'Events.FrameManager.FrameNavigated', frame
@@ -320,44 +317,29 @@ class Puppeteer::FrameManager
       world.context = context
     end
     @context_id_to_context[context_payload['id']] = context
-    @context_id_created[context_payload['id']] = Time.now
   end
 
   # @param {number} executionContextId
   def handle_execution_context_destroyed(execution_context_id)
     context = @context_id_to_context[execution_context_id]
-    return if !context
+    return unless context
     @context_id_to_context.delete(execution_context_id)
-    @context_id_created.delete(execution_context_id)
     if context.world
       context.world.delete_context(execution_context_id)
     end
   end
 
   def handle_execution_contexts_cleared
-    # executionContextsCleared is often notified after executionContextCreated.
-    #   D, [2020-04-06T01:47:03.101227 #13823] DEBUG -- : RECV << {"method"=>"Runtime.executionContextCreated", "params"=>{"context"=>{"id"=>5, "origin"=>"https://github.com", "name"=>"", "auxData"=>{"isDefault"=>true, "type"=>"default", "frameId"=>"71C347B70848B89DDDEFAA8AB5B0BC92"}}}, "sessionId"=>"53F088EED260C28001D26A019F95D9E3"}
-    #   D, [2020-04-06T01:47:03.101439 #13823] DEBUG -- : RECV << {"method"=>"Page.frameNavigated", "params"=>{"frame"=>{"id"=>"71C347B70848B89DDDEFAA8AB5B0BC92", "loaderId"=>"80338225D035AC96BAE8F6D4E81C7D51", "url"=>"https://github.com/search?q=puppeteer", "securityOrigin"=>"https://github.com", "mimeType"=>"text/html"}}, "sessionId"=>"53F088EED260C28001D26A019F95D9E3"}
-    #   D, [2020-04-06T01:47:03.101325 #13823] DEBUG -- : RECV << {"method"=>"Target.targetInfoChanged", "params"=>{"targetInfo"=>{"targetId"=>"71C347B70848B89DDDEFAA8AB5B0BC92", "type"=>"page", "title"=>"https://github.com/search?q=puppeteer", "url"=>"https://github.com/search?q=puppeteer", "attached"=>true, "browserContextId"=>"AF37BC660284CE1552B4ECB147BE9305"}}}
-    #   D, [2020-04-06T01:47:03.101269 #13823] DEBUG -- : RECV << {"method"=>"Runtime.executionContextsCleared", "params"=>{}, "sessionId"=>"53F088EED260C28001D26A019F95D9E3"}
-    # it unexpectedly clears the created execution context.
-    # To avoid the problem, just skip recent created ids.
-    now = Time.now
-    context_ids_to_skip = @context_id_created.select { |k, v| now - v < 1 }.keys
-    @context_id_to_context.reject { |k, v| context_ids_to_skip.include?(k) }.each do |execution_context_id, context|
+    @context_id_to_context.each do |execution_context_id, context|
       if context.world
         context.world.delete_context(execution_context_id)
       end
     end
-    @context_id_to_context.select! { |k, v| context_ids_to_skip.include?(k) }
+    @context_id_to_context.clear
   end
 
   def execution_context_by_id(context_id)
-    context = @context_id_to_context[context_id]
-    if !context
-      raise "INTERNAL ERROR: missing context with id = #{context_id}"
-    end
-    context
+    @context_id_to_context[context_id] or raise "INTERNAL ERROR: missing context with id = #{context_id}"
   end
 
   # @param {!Frame} frame

data/lib/puppeteer/js_handle.rb

@@ -1,5 +1,6 @@
 class Puppeteer::JSHandle
-  using Puppeteer::AsyncAwaitBehavior
+  using Puppeteer::DefineAsyncMethod
+  include Puppeteer::IfPresent
 
   # @param context [Puppeteer::ExecutionContext]
   # @param remote_object [Puppeteer::RemoteObject]
@@ -46,11 +47,7 @@ class Puppeteer::JSHandle
     execution_context.evaluate(page_function, self, *args)
   end
 
-  # @param page_function [String]
-  # @return [Future<Object>]
-  async def async_evaluate(page_function, *args)
-    evaluate(page_function, *args)
-  end
+  define_async_method :async_evaluate
 
   # @param page_function [String]
   # @param args {Array<*>}
@@ -59,28 +56,31 @@ class Puppeteer::JSHandle
     execution_context.evaluate_handle(page_function, self, *args)
   end
 
-  # @param page_function [String]
-  # @param args {Array<*>}
-  # @return [Future<Puppeteer::JSHandle>]
-  async def async_evaluate_handle(page_function, *args)
-    evaluate_handle(page_function, *args)
+  define_async_method :async_evaluate_handle
+
+  # getProperty(propertyName) in JavaScript
+  # @param name [String]
+  # @return [Puppeteer::JSHandle]
+  def property(name)
+    js = <<~JAVASCRIPT
+    (object, propertyName) => {
+      const result = {__proto__: null};
+      result[propertyName] = object[propertyName];
+      return result;
+    }
+    JAVASCRIPT
+    object_handle = evaluate_handle(js, name)
+    properties = object_handle.properties
+    result = properties[name]
+    object_handle.dispose
+    result
   end
 
-  # /**
-  #  * @param {string} propertyName
-  #  * @return {!Promise<?JSHandle>}
-  #  */
-  # async getProperty(propertyName) {
-  #   const objectHandle = await this.evaluateHandle((object, propertyName) => {
-  #     const result = {__proto__: null};
-  #     result[propertyName] = object[propertyName];
-  #     return result;
-  #   }, propertyName);
-  #   const properties = await objectHandle.getProperties();
-  #   const result = properties.get(propertyName) || null;
-  #   await objectHandle.dispose();
-  #   return result;
-  # }
+  # @param name [String]
+  # @return [Puppeteer::JSHandle]
+  def [](name)
+    property(name)
+  end
 
   # getProperties in JavaScript.
   # @return [Hash<String, JSHandle>]
@@ -110,7 +110,7 @@ class Puppeteer::JSHandle
     #
     # However it would be better that RemoteObject is responsible for
     # the logic `if (this._remoteObject.objectId) { ... }`.
-    @remote_object.evaluate_self(@client) || @remote_object.value
+    @remote_object.evaluate_self(@client)&.value || @remote_object.value
   end
 
   def as_element
@@ -128,15 +128,16 @@ class Puppeteer::JSHandle
     @disposed
   end
 
-  # /**
-  #  * @override
-  #  * @return {string}
-  #  */
-  # toString() {
-  #   if (this._remoteObject.objectId) {
-  #     const type =  this._remoteObject.subtype || this._remoteObject.type;
-  #     return 'JSHandle@' + type;
-  #   }
-  #   return 'JSHandle:' + helper.valueFromRemoteObject(this._remoteObject);
-  # }
+  def to_s
+    # original logic was:
+    #   if (this._remoteObject.objectId) {
+    #     const type =  this._remoteObject.subtype || this._remoteObject.type;
+    #     return 'JSHandle@' + type;
+    #   }
+    #   return 'JSHandle:' + helper.valueFromRemoteObject(this._remoteObject);
+    #
+    # However it would be better that RemoteObject is responsible for
+    # the logic `if (this._remoteObject.objectId) { ... }`.
+    if_present(@remote_object.type_str) { |type_str| "JSHandle@#{type_str}" } || "JSHandle:#{@remote_object.value || 'undefined'}"
+  end
 end

data/lib/puppeteer/keyboard.rb

@@ -2,7 +2,7 @@ require_relative './keyboard/key_description'
 require_relative './keyboard/us_keyboard_layout'
 
 class Puppeteer::Keyboard
-  using Puppeteer::AsyncAwaitBehavior
+  using Puppeteer::DefineAsyncMethod
 
   # @param {!Puppeteer.CDPSession} client
   def initialize(client)
@@ -38,12 +38,7 @@ class Puppeteer::Keyboard
     @client.send_message('Input.dispatchKeyEvent', params)
   end
 
-  # @param key [String]
-  # @param text [String]
-  # @return [Future]
-  async def async_down(key, text: nil)
-    down(key, text)
-  end
+  define_async_method :async_down
 
   # @param {string} key
   # @return {number}
@@ -127,22 +122,14 @@ class Puppeteer::Keyboard
     )
   end
 
-  # @param key [String]
-  # @return [Future]
-  async def async_up(key)
-    up(key)
-  end
+  define_async_method :async_up
 
   # @param char [string]
   def send_character(char)
     @client.send_message('Input.insertText', text: char)
   end
 
-  # @param char [string]
-  # @return [Future]
-  async def async_send_character(char)
-    send_character(char)
-  end
+  define_async_method :async_send_character
 
   # @param text [String]
   # @return [Future]
@@ -159,25 +146,18 @@ class Puppeteer::Keyboard
     end
   end
 
-  # @param text [String]
-  # @return [Future]
-  async def async_type_text(text, delay: nil)
-    type_text(text, delay)
-  end
+  define_async_method :async_type_text
 
   # @param key [String]
+  # @param text [String]
   # @return [Future]
-  def press(key, delay: nil)
-    down(key)
+  def press(key, delay: nil, text: nil)
+    down(key, text: text)
     if delay
       sleep(delay.to_i / 1000.0)
     end
     up(key)
   end
 
-  # @param key [String]
-  # @return [Future]
-  async def async_press(key, delay: nil)
-    press(key, delay: delay)
-  end
+  define_async_method :async_press
 end

data/lib/puppeteer/mouse.rb

@@ -1,5 +1,5 @@
 class Puppeteer::Mouse
-  using Puppeteer::AsyncAwaitBehavior
+  using Puppeteer::DefineAsyncMethod
 
   module Button
     NONE = 'none'
@@ -44,13 +44,7 @@ class Puppeteer::Mouse
     end
   end
 
-  # @param x [number]
-  # @param y [number]
-  # @param steps [number]
-  # @return [Future]
-  async def async_move(x, y, steps: nil)
-    move(x, y, steps: steps)
-  end
+  define_async_method :async_move
 
   # @param x [number]
   # @param y [number]
@@ -70,13 +64,7 @@ class Puppeteer::Mouse
     up(button: button, click_count: click_count)
   end
 
-  # @param x [number]
-  # @param y [number]
-  # @param {!{delay?: number, button?: "left"|"right"|"middle", clickCount?: number}=} options
-  # @return [Future]
-  async def async_click(x, y, delay: nil, button: nil, click_count: nil)
-    click(x, y, delay: delay, button: button, click_count: click_count)
-  end
+  define_async_method :async_click
 
   # @param {!{button?: "left"|"right"|"middle", clickCount?: number}=} options
   def down(button: nil, click_count: nil)
@@ -91,11 +79,7 @@ class Puppeteer::Mouse
     )
   end
 
-  # @param {!{button?: "left"|"right"|"middle", clickCount?: number}=} options
-  # @return [Future]
-  async def async_down(button: nil, click_count: nil)
-    down(button: button, click_count: click_count)
-  end
+  define_async_method :async_down
 
   # @param {!{button?: "left"|"right"|"middle", clickCount?: number}=} options
   def up(button: nil, click_count: nil)
@@ -110,9 +94,21 @@ class Puppeteer::Mouse
     )
   end
 
-  # @param {!{button?: "left"|"right"|"middle", clickCount?: number}=} options
-  # @return [Future]
-  async def async_up(button: nil, click_count: nil)
-    up(button: button, click_count: click_count)
+  # Dispatches a `mousewheel` event.
+  #
+  # @param delta_x [Integer]
+  # @param delta_y [Integer]
+  def wheel(delta_x: 0, delta_y: 0)
+    @client.send_message('Input.dispatchMouseEvent',
+      type: 'mouseWheel',
+      x: @x,
+      y: @y,
+      deltaX: delta_x,
+      deltaY: delta_y,
+      modifiers: @keyboard.modifiers,
+      pointerType: 'mouse',
+    )
   end
+
+  define_async_method :async_up
 end

data/lib/puppeteer/network_manager.rb

@@ -1,5 +1,7 @@
 class Puppeteer::NetworkManager
+  include Puppeteer::DebugPrint
   include Puppeteer::EventCallbackable
+  include Puppeteer::IfPresent
 
   class Credentials
     # @param username [String|NilClass]
@@ -23,19 +25,39 @@ class Puppeteer::NetworkManager
     @request_id_to_request = {}
 
     # @type {!Map<string, !Protocol.Network.requestWillBeSentPayload>}
-    @request_id_to_request_with_be_sent_event
+    @request_id_to_request_with_be_sent_event = {}
 
     @extra_http_headers = {}
 
     @offline = false
 
-    # /** @type {!Set<string>} */
-    # this._attemptedAuthentications = new Set();
+    @attempted_authentications = Set.new
     @user_request_interception_enabled = false
     @protocol_request_interception_enabled = false
     @user_cache_disabled = false
-    # /** @type {!Map<string, string>} */
-    # this._requestIdToInterceptionId = new Map();
+    @request_id_to_interception_id = {}
+
+    @client.on_event('Fetch.requestPaused') do |event|
+      handle_request_paused(event)
+    end
+    @client.on_event('Fetch.authRequired') do |event|
+      handle_auth_required(event)
+    end
+    @client.on_event('Network.requestWillBeSent') do |event|
+      handle_request_will_be_sent(event)
+    end
+    @client.on_event('Network.requestServedFromCache') do |event|
+      handle_request_served_from_cache(event)
+    end
+    @client.on_event('Network.responseReceived') do |event|
+      handle_response_received(event)
+    end
+    @client.on_event('Network.loadingFinished') do |event|
+      handle_loading_finished(event)
+    end
+    @client.on_event('Network.loadingFailed') do |event|
+      handle_loading_failed(event)
+    end
   end
 
   def init
@@ -119,4 +141,140 @@ class Puppeteer::NetworkManager
     cache_disabled = @user_cache_disabled || @protocol_request_interception_enabled
     @client.send_message('Network.setCacheDisabled', cacheDisabled: cache_disabled)
   end
+
+  private def handle_request_will_be_sent(event)
+    # Request interception doesn't happen for data URLs with Network Service.
+    if @protocol_request_interception_enabled && !event['request']['url'].start_with?('data:')
+      request_id = event['requestId']
+      interception_id = @request_id_to_interception_id.delete(request_id)
+      if interception_id
+        handle_request(event, interception_id)
+      else
+        @request_id_to_request_with_be_sent_event[request_id] = event
+      end
+      return
+    end
+    handle_request(event, nil)
+  end
+
+  private def handle_auth_required(event)
+    response = 'Default'
+    if @attempted_authentications.include?(event['requestId'])
+      response = 'CancelAuth'
+    elsif @credentials
+      response = 'ProvideCredentials'
+      @attempted_authentications << event['requestId']
+    end
+
+    username = @credentials&.username
+    password = @credentials&.password
+
+    begin
+      @client.send_message('Fetch.continueWithAuth',
+        requestId: event['requestId'],
+        authChallengeResponse: {
+          response: response,
+          username: username,
+          password: password,
+        },
+      )
+    rescue => err
+      debug_puts(err)
+    end
+  end
+
+  private def handle_request_paused(event)
+    if !@user_request_interception_enabled && @protocol_request_interception_enabled
+      begin
+        @client.send_message('Fetch.continueRequest', requestId: event['requestId'])
+      rescue => err
+        debug_puts(err)
+      end
+    end
+
+    request_id = event['networkId']
+    interception_id = event['requestId']
+    if request_id && (request_will_be_sent_event = @request_id_to_request_with_be_sent_event.delete(request_id))
+      handle_request(request_will_be_sent_event, interception_id)
+    else
+      @request_id_to_interception_id[request_id] = interception_id
+    end
+  end
+
+  private def handle_request(event, interception_id)
+    redirect_chain = []
+    if event['redirectResponse']
+      if_present(@request_id_to_request[event['requestId']]) do |request|
+        handle_request_redirect(request, event['redirectResponse'])
+        redirect_chain = request.internal.redirect_chain
+      end
+    end
+    frame = if_present(event['frameId']) { |frame_id| @frame_manager.frame(frame_id) }
+    request = Puppeteer::Request.new(@client, frame, interception_id, @user_request_interception_enabled, event, redirect_chain)
+    @request_id_to_request[event['requestId']] = request
+    emit_event 'Events.NetworkManager.Request', request
+  end
+
+  private def handle_request_served_from_cache(event)
+    if_present(@request_id_to_request[event['requestId']]) do |request|
+      request.internal.from_memory_cache = true
+    end
+  end
+
+  # @param request [Puppeteer::Request]
+  # @param response_payload [Hash]
+  private def handle_request_redirect(request, response_payload)
+    response = Puppeteer::Response.new(@client, request, response_payload)
+    request.internal.response = response
+    request.internal.redirect_chain << request
+    response.internal.body_loaded_promise.reject(Puppeteer::Response::Redirected.new)
+    @request_id_to_request.delete(request.internal.request_id)
+    @attempted_authentications.delete(request.internal.interception_id)
+    emit_event 'Events.NetworkManager.Response', response
+    emit_event 'Events.NetworkManager.RequestFinished', request
+  end
+
+  # @param event [Hash]
+  private def handle_response_received(event)
+    request = @request_id_to_request[event['requestId']]
+    # FileUpload sends a response without a matching request.
+    return unless request
+
+    response = Puppeteer::Response.new(@client, request, event['response'])
+    request.internal.response = response
+    emit_event 'Events.NetworkManager.Response', response
+  end
+
+  private def handle_loading_finished(event)
+    request = @request_id_to_request[event['requestId']]
+    # For certain requestIds we never receive requestWillBeSent event.
+    # @see https://crbug.com/750469
+    return unless request
+
+
+    # Under certain conditions we never get the Network.responseReceived
+    # event from protocol. @see https://crbug.com/883475
+    if_present(request.response) do |response|
+      response.internal.body_loaded_promise.fulfill(nil)
+    end
+
+    @request_id_to_request.delete(request.internal.request_id)
+    @attempted_authentications.delete(request.internal.interception_id)
+    emit_event 'Events.NetworkManager.RequestFinished', request
+  end
+
+  private def handle_loading_failed(event)
+    request = @request_id_to_request[event['requestId']]
+    # For certain requestIds we never receive requestWillBeSent event.
+    # @see https://crbug.com/750469
+    return unless request
+
+    request.internal.failure_text = event['errorText']
+    if_present(request.response) do |response|
+      response.internal.body_loaded_promise.fulfill(nil)
+    end
+    @request_id_to_request.delete(request.internal.request_id)
+    @attempted_authentications.delete(request.internal.interception_id)
+    emit_event 'Events.NetworkManager.RequestFailed', request
+  end
 end