puppeteer-ruby 0.41.0 → 0.43.0

data/lib/puppeteer/browser.rb

@@ -7,13 +7,15 @@ class Puppeteer::Browser
   include Puppeteer::IfPresent
   using Puppeteer::DefineAsyncMethod
 
+  # @param product [String|nil] 'chrome' or 'firefox'
   # @param {!Puppeteer.Connection} connection
   # @param {!Array<string>} contextIds
   # @param {boolean} ignoreHTTPSErrors
   # @param {?Puppeteer.Viewport} defaultViewport
   # @param process [Puppeteer::BrowserRunner::BrowserProcess|NilClass]
   # @param {function()=} closeCallback
-  def self.create(connection:,
+  def self.create(product:,
+                  connection:,
                   context_ids:,
                   ignore_https_errors:,
                   default_viewport:,
@@ -22,6 +24,7 @@ class Puppeteer::Browser
                   target_filter_callback:,
                   is_page_target_callback:)
     browser = Puppeteer::Browser.new(
+      product: product,
       connection: connection,
       context_ids: context_ids,
       ignore_https_errors: ignore_https_errors,
@@ -31,17 +34,19 @@ class Puppeteer::Browser
       target_filter_callback: target_filter_callback,
       is_page_target_callback: is_page_target_callback,
     )
-    connection.send_message('Target.setDiscoverTargets', discover: true)
+    browser.send(:attach)
     browser
   end
 
+  # @param product [String|nil] 'chrome' or 'firefox'
   # @param {!Puppeteer.Connection} connection
   # @param {!Array<string>} contextIds
   # @param {boolean} ignoreHTTPSErrors
   # @param {?Puppeteer.Viewport} defaultViewport
   # @param {?Puppeteer.ChildProcess} process
   # @param {(function():Promise)=} closeCallback
-  def initialize(connection:,
+  def initialize(product:,
+                 connection:,
                  context_ids:,
                  ignore_https_errors:,
                  default_viewport:,
@@ -49,6 +54,7 @@ class Puppeteer::Browser
                  close_callback:,
                  target_filter_callback:,
                  is_page_target_callback:)
+    @product = product || 'chrome'
     @ignore_https_errors = ignore_https_errors
     @default_viewport = default_viewport
     @process = process
@@ -56,20 +62,26 @@ class Puppeteer::Browser
     @close_callback = close_callback
     @target_filter_callback = target_filter_callback || method(:default_target_filter_callback)
     @is_page_target_callback = is_page_target_callback || method(:default_is_page_target_callback)
-
     @default_context = Puppeteer::BrowserContext.new(@connection, self, nil)
     @contexts = {}
+
     context_ids.each do |context_id|
       @contexts[context_id] = Puppeteer::BrowserContext.new(@connection, self, context_id)
     end
-    @targets = {}
-    @wait_for_creating_targets = {}
-    @connection.on_event(ConnectionEmittedEvents::Disconnected) do
-      emit_event(BrowserEmittedEvents::Disconnected)
+
+    if @product == 'firefox'
+      @target_manager = Puppeteer::FirefoxTargetManager.new(
+        connection: connection,
+        target_factory: method(:create_target),
+        target_filter_callback: @target_filter_callback,
+      )
+    else
+      @target_manager = Puppeteer::ChromeTargetManager.new(
+        connection: connection,
+        target_factory: method(:create_target),
+        target_filter_callback: @target_filter_callback,
+      )
     end
-    @connection.on_event('Target.targetCreated', &method(:handle_target_created))
-    @connection.on_event('Target.targetDestroyed', &method(:handle_target_destroyed))
-    @connection.on_event('Target.targetInfoChanged', &method(:handle_target_info_changed))
   end
 
   private def default_target_filter_callback(target_info)
@@ -100,11 +112,45 @@ class Puppeteer::Browser
     super(event_name.to_s, &block)
   end
 
+  private def attach
+    @connection_event_listeners ||= []
+    @connection_event_listeners << @connection.add_event_listener(ConnectionEmittedEvents::Disconnected) do
+      emit_event(BrowserEmittedEvents::Disconnected)
+    end
+    @target_manager_event_listeners ||= []
+    @target_manager.add_event_listener(
+      TargetManagerEmittedEvents::TargetAvailable,
+      &method(:handle_attached_to_target)
+    )
+    @target_manager.add_event_listener(
+      TargetManagerEmittedEvents::TargetGone,
+      &method(:handle_detached_from_target)
+    )
+    @target_manager.add_event_listener(
+      TargetManagerEmittedEvents::TargetChanged,
+      &method(:handle_target_changed)
+    )
+    @target_manager.add_event_listener(
+      TargetManagerEmittedEvents::TargetDiscovered,
+      &method(:handle_target_discovered)
+    )
+    @target_manager.init
+  end
+
+  private def detach
+    @connection.remove_event_listener(*@connection_event_listeners)
+    @target_manager.remove_event_listener(*@target_manager_event_listeners)
+  end
+
   # @return [Puppeteer::BrowserRunner::BrowserProcess]
   def process
     @process
   end
 
+  private def target_manager
+    @target_manager
+  end
+
   # @return [Puppeteer::BrowserContext]
   def create_incognito_browser_context
     result = @connection.send_message('Target.createBrowserContext')
@@ -123,19 +169,16 @@ class Puppeteer::Browser
 
   # @param context_id [String]
   def dispose_context(context_id)
+    return unless context_id
     @connection.send_message('Target.disposeBrowserContext', browserContextId: context_id)
     @contexts.delete(context_id)
   end
 
-  class TargetAlreadyExistError < StandardError
-    def initialize
-      super('Target should not exist before targetCreated')
-    end
-  end
+  class MissingBrowserContextError < StandardError ; end
 
-  # @param {!Protocol.Target.targetCreatedPayload} event
-  def handle_target_created(event)
-    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+  # @param target_info [Puppeteer::Target::TargetInfo]
+  # @param session [CDPSession|nil]
+  def create_target(target_info, session)
     browser_context_id = target_info.browser_context_id
     context =
       if browser_context_id && @contexts.has_key?(browser_context_id)
@@ -144,56 +187,39 @@ class Puppeteer::Browser
         @default_context
       end
 
-    if @targets[target_info.target_id]
-      raise TargetAlreadyExistError.new
+    unless context
+      raise MissingBrowserContextError.new('Missing browser context')
     end
 
-    return unless @target_filter_callback.call(target_info)
-
-    target = Puppeteer::Target.new(
+    Puppeteer::Target.new(
       target_info: target_info,
+      session: session,
       browser_context: context,
+      target_manager: @target_manager,
       session_factory: -> { @connection.create_session(target_info) },
       ignore_https_errors: @ignore_https_errors,
       default_viewport: @default_viewport,
       is_page_target_callback: @is_page_target_callback,
     )
-    @targets[target_info.target_id] = target
-    if_present(@wait_for_creating_targets.delete(target_info.target_id)) do |promise|
-      promise.fulfill(target)
-    end
-    if await target.initialized_promise
+  end
+
+  private def handle_attached_to_target(target)
+    if target.initialized_promise.value!
       emit_event(BrowserEmittedEvents::TargetCreated, target)
-      context.emit_event(BrowserContextEmittedEvents::TargetCreated, target)
+      target.browser_context.emit_event(BrowserContextEmittedEvents::TargetCreated, target)
     end
   end
 
-  # @param {{targetId: string}} event
-  def handle_target_destroyed(event)
-    target_id = event['targetId']
-    target = @targets[target_id]
+  private def handle_detached_from_target(target)
     target.ignore_initialize_callback_promise
-    @targets.delete(target_id)
-    if_present(@wait_for_creating_targets.delete(target_id)) do |promise|
-      promise.reject('target destroyed')
-    end
     target.closed_callback
-    if await target.initialized_promise
+    if target.initialized_promise.value!
       emit_event(BrowserEmittedEvents::TargetDestroyed, target)
       target.browser_context.emit_event(BrowserContextEmittedEvents::TargetDestroyed, target)
     end
   end
 
-  class TargetNotExistError < StandardError
-    def initialize
-      super('target should exist before targetInfoChanged')
-    end
-  end
-
-  # @param {!Protocol.Target.targetInfoChangedPayload} event
-  def handle_target_info_changed(event)
-    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
-    target = @targets[target_info.target_id] or raise TargetNotExistError.new
+  private def handle_target_changed(target, target_info)
     previous_url = target.url
     was_initialized = target.initialized?
     target.handle_target_info_changed(target_info)
@@ -203,6 +229,10 @@ class Puppeteer::Browser
     end
   end
 
+  private def handle_target_discovered(target_info)
+    emit_event('targetdiscovered', target_info)
+  end
+
   # @return [String]
   def ws_endpoint
     @connection.url
@@ -212,44 +242,47 @@ class Puppeteer::Browser
     @default_context.new_page
   end
 
+  class MissingTargetError < StandardError ; end
+  class CreatePageError < StandardError ; end
+
   # @param {?string} contextId
   # @return {!Promise<!Puppeteer.Page>}
   def create_page_in_context(context_id)
-    create_target_params = { url: 'about:blank' }
-    if context_id
-      create_target_params[:browserContextId] = context_id
-    end
+    create_target_params = {
+      url: 'about:blank',
+      browserContextId: context_id,
+    }.compact
     result = @connection.send_message('Target.createTarget', **create_target_params)
     target_id = result['targetId']
-    target = @targets[target_id]
+    target = @target_manager.available_targets[target_id]
     unless target
-      # Target.targetCreated is often notified before the response of Target.createdTarget.
-      # https://github.com/YusukeIwaki/puppeteer-ruby/issues/91
-      # D, [2021-04-07T03:00:10.125241 #187] DEBUG -- : SEND >> {"method":"Target.createTarget","params":{"url":"about:blank","browserContextId":"56A86FC3391B50180CF9A6450A0D8C21"},"id":3}
-      # D, [2021-04-07T03:00:10.142396 #187] DEBUG -- : RECV << {"id"=>3, "result"=>{"targetId"=>"A518447C415A1A3E1A8979454A155632"}}
-      # D, [2021-04-07T03:00:10.145360 #187] DEBUG -- : RECV << {"method"=>"Target.targetCreated", "params"=>{"targetInfo"=>{"targetId"=>"A518447C415A1A3E1A8979454A155632", "type"=>"page", "title"=>"", "url"=>"", "attached"=>false, "canAccessOpener"=>false, "browserContextId"=>"56A86FC3391B50180CF9A6450A0D8C21"}}}
-      # This is just a workaround logic...
-      @wait_for_creating_targets[target_id] = resolvable_future
-      target = await @wait_for_creating_targets[target_id]
+      raise MissingTargetError.new("Missing target for page (id = #{target_id})")
+    end
+    unless target.initialized_promise.value!
+      raise CreatePageError.new("Failed to create target for page (id = #{target_id})")
+    end
+    page = target.page
+    unless page
+      raise CreatePageError.new("Failed to create a page for context (id = #{context_id})")
     end
-    await target.initialized_promise
-    await target.page
+    page
   end
 
-  # @return {!Array<!Target>}
+  # All active targets inside the Browser. In case of multiple browser contexts, returns
+  # an array with all the targets in all browser contexts.
   def targets
-    @targets.values.select { |target| target.initialized? }
+    @target_manager.available_targets.values.select { |target| target.initialized? }
   end
 
 
-  # @return {!Target}
+  # The target associated with the browser.
   def target
-    targets.find { |target| target.type == 'browser' }
+    targets.find { |target| target.type == 'browser' } or raise 'Browser target is not found'
   end
 
   # used only in Target#opener
   private def find_target_by_id(target_id)
-    @targets[target_id]
+    @target_manager.available_targets[target_id]
   end
 
   # @param predicate [Proc(Puppeteer::Target -> Boolean)]
@@ -293,12 +326,12 @@ class Puppeteer::Browser
 
   # @return [String]
   def version
-    get_version.product
+    Version.fetch(@connection).product
   end
 
   # @return [String]
   def user_agent
-    get_version.user_agent
+    Version.fetch(@connection).user_agent
   end
 
   def close
@@ -307,6 +340,7 @@ class Puppeteer::Browser
   end
 
   def disconnect
+    @target_manager.dispose
     @connection.dispose
   end
 
@@ -315,6 +349,10 @@ class Puppeteer::Browser
   end
 
   class Version
+    def self.fetch(connection)
+      new(connection.send_message('Browser.getVersion'))
+    end
+
     def initialize(hash)
       @protocol_version = hash['protocolVersion']
       @product = hash['product']
@@ -325,8 +363,4 @@ class Puppeteer::Browser
 
     attr_reader :protocol_version, :product, :revision, :user_agent, :js_version
   end
-
-  private def get_version
-    Version.new(@connection.send_message('Browser.getVersion'))
-  end
 end

data/lib/puppeteer/browser_connector.rb

@@ -0,0 +1,67 @@
+require_relative './browser'
+require_relative './launcher/browser_options'
+
+class Puppeteer::BrowserConnector
+  def initialize(options)
+    @browser_options = Puppeteer::Launcher::BrowserOptions.new(options)
+    @browser_ws_endpoint = options[:browser_ws_endpoint]
+    @browser_url = options[:browser_url]
+    @transport = options[:transport]
+  end
+
+  # @return [Puppeteer::Browser]
+  def connect_to_browser
+    version = Puppeteer::Browser::Version.fetch(connection)
+    product = version.product.downcase.include?('firefox') ? 'firefox' : 'chrome'
+
+    result = connection.send_message('Target.getBrowserContexts')
+    browser_context_ids = result['browserContextIds']
+
+    Puppeteer::Browser.create(
+      product: product,
+      connection: connection,
+      context_ids: browser_context_ids,
+      ignore_https_errors: @browser_options.ignore_https_errors?,
+      default_viewport: @browser_options.default_viewport,
+      process: nil,
+      close_callback: -> { connection.send_message('Browser.close') },
+      target_filter_callback: @browser_options.target_filter,
+      is_page_target_callback: @browser_options.is_page_target,
+    )
+  end
+
+  private def connection
+    @connection ||=
+      if @browser_ws_endpoint && @browser_url.nil? && @transport.nil?
+        connect_with_browser_ws_endpoint(@browser_ws_endpoint)
+      elsif @browser_ws_endpoint.nil? && @browser_url && @transport.nil?
+        connect_with_browser_url(@browser_url)
+      elsif @browser_ws_endpoint.nil? && @browser_url.nil? && @transport
+        connect_with_transport(@transport)
+      else
+        raise ArgumentError.new("Exactly one of browserWSEndpoint, browserURL or transport must be passed to puppeteer.connect")
+      end
+  end
+
+  # @return [Puppeteer::Connection]
+  private def connect_with_browser_ws_endpoint(browser_ws_endpoint)
+    transport = Puppeteer::WebSocketTransport.create(browser_ws_endpoint)
+    Puppeteer::Connection.new(browser_ws_endpoint, transport, @browser_options.slow_mo)
+  end
+
+  # @return [Puppeteer::Connection]
+  private def connect_with_browser_url(browser_url)
+    require 'net/http'
+    uri = URI(browser_url)
+    uri.path = '/json/version'
+    response_body = Net::HTTP.get(uri)
+    json = JSON.parse(response_body)
+    connection_url = json['webSocketDebuggerUrl']
+    connect_with_browser_ws_endpoint(connection_url)
+  end
+
+  # @return [Puppeteer::Connection]
+  private def connect_with_transport(transport)
+    Puppeteer::Connection.new('', transport, @browser_options.slow_mo)
+  end
+end

data/lib/puppeteer/chrome_target_manager.rb

@@ -0,0 +1,256 @@
+class Puppeteer::ChromeTargetManager
+  include Puppeteer::EventCallbackable
+
+  def initialize(connection:, target_factory:, target_filter_callback:)
+    @discovered_targets_by_target_id = {}
+    @attached_targets_by_target_id = {}
+    @attached_targets_by_session_id = {}
+    @ignored_targets = Set.new
+    @target_ids_for_init = Set.new
+
+    @connection = connection
+    @target_filter_callback = target_filter_callback
+    @target_factory = target_factory
+    @target_interceptors = {}
+    @initialize_promise = resolvable_future
+
+    @connection_event_listeners = []
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetCreated',
+      &method(:handle_target_created)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetDestroyed',
+      &method(:handle_target_destroyed)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'Target.targetInfoChanged',
+      &method(:handle_target_info_changed)
+    )
+    @connection_event_listeners << @connection.add_event_listener(
+      'sessiondetached',
+      &method(:handle_session_detached)
+    )
+
+    setup_attachment_listeners(@connection)
+    @connection.async_send_message('Target.setDiscoverTargets', discover: true)
+  end
+
+  def init
+    @discovered_targets_by_target_id.each do |target_id, target_info|
+      if @target_filter_callback.call(target_info)
+        @target_ids_for_init << target_id
+      end
+    end
+    @connection.send_message('Target.setAutoAttach', {
+      waitForDebuggerOnStart: true,
+      flatten: true,
+      autoAttach: true,
+    })
+    @initialize_promise.value!
+  end
+
+  def dispose
+    @connection.remove_event_listener(*@connection_event_listeners)
+    remove_attachment_listeners(@connection)
+  end
+
+  def available_targets
+    @attached_targets_by_target_id
+  end
+
+  def add_target_interceptor(client, interceptor)
+    interceptors = @target_interceptors[client] || []
+    interceptors << interceptor
+    @target_interceptors[client] = interceptors
+  end
+
+  def remove_target_interceptor(client, interceptor)
+    @target_interceptors[client]&.delete_if { |current| current == interceptor }
+  end
+
+  private def setup_attachment_listeners(session)
+    @attachment_listener_ids ||= {}
+    @attachment_listener_ids[session] ||= []
+
+    @attachment_listener_ids[session] << session.add_event_listener('Target.attachedToTarget') do |event|
+      handle_attached_to_target(session, event)
+    end
+
+    @attachment_listener_ids[session] << session.add_event_listener('Target.detachedFromTarget') do |event|
+      handle_detached_from_target(session, event)
+    end
+  end
+
+  private def remove_attachment_listeners(session)
+    return unless @attachment_listener_ids
+    listener_ids = @attachment_listener_ids.delete(session)
+    return if !listener_ids || listener_ids.empty?
+    session.remove_event_listener(*listener_ids)
+  end
+
+  private def handle_session_detached(session)
+    remove_attachment_listeners(session)
+    @target_interceptors.delete(session)
+  end
+
+  private def handle_target_created(event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    @discovered_targets_by_target_id[target_info.target_id] = target_info
+
+    emit_event(TargetManagerEmittedEvents::TargetDiscovered, target_info)
+
+    # The connection is already attached to the browser target implicitly,
+    # therefore, no new CDPSession is created and we have special handling
+    # here.
+    if target_info.type == 'browser' && target_info.attached
+      return if @attached_targets_by_target_id[target_info.target_id]
+
+      target = @target_factory.call(target_info, nil)
+      @attached_targets_by_target_id[target_info.target_id] = target
+    end
+
+    if target_info.type == 'shared_worker'
+      # Special case (https://crbug.com/1338156): currently, shared_workers
+      # don't get auto-attached. This should be removed once the auto-attach
+      # works.
+      @connection.create_session(target_info)
+    end
+  end
+
+  private def handle_target_destroyed(event)
+    target_id = event['targetId']
+    target_info = @discovered_targets_by_target_id.delete(target_id)
+    finish_initialization_if_ready(target_id)
+
+    if target_info.type == 'service_worker' && @attached_targets_by_target_id.has_key?(target_id)
+      # Special case for service workers: report TargetGone event when
+      # the worker is destroyed.
+      target = @attached_targets_by_target_id.delete(target_id)
+      emit_event(TargetManagerEmittedEvents::TargetGone, target)
+    end
+  end
+
+  private def handle_target_info_changed(event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    @discovered_targets_by_target_id[target_info.target_id] = target_info
+
+    if @ignored_targets.include?(target_info.target_id) || !@attached_targets_by_target_id.has_key?(target_info.target_id) || !target_info.attached
+      return
+    end
+    original_target = @attached_targets_by_target_id[target_info.target_id]
+    emit_event(TargetManagerEmittedEvents::TargetChanged, original_target, target_info)
+  end
+
+  class SessionNotCreatedError < StandardError ; end
+
+  private def handle_attached_to_target(parent_session, event)
+    target_info = Puppeteer::Target::TargetInfo.new(event['targetInfo'])
+    session_id = event['sessionId']
+    session = @connection.session(session_id)
+    unless session
+      raise SessionNotCreatedError.new("Session #{session_id} was not created.")
+    end
+
+    silent_detach = -> {
+      begin
+        session.send_message('Runtime.runIfWaitingForDebugger')
+      rescue => err
+        Logger.new($stderr).warn(err)
+      end
+
+      # We don't use `session.detach()` because that dispatches all commands on
+      # the connection instead of the parent session.
+      begin
+        parent_session.send_message('Target.detachFromTarget', {
+          sessionId: session.id,
+        })
+      rescue => err
+        Logger.new($stderr).warn(err)
+      end
+    }
+
+    # Special case for service workers: being attached to service workers will
+    # prevent them from ever being destroyed. Therefore, we silently detach
+    # from service workers unless the connection was manually created via
+    # `page.worker()`. To determine this, we use
+    # `this.#connection.isAutoAttached(targetInfo.targetId)`. In the future, we
+    # should determine if a target is auto-attached or not with the help of
+    # CDP.
+    if target_info.type == 'service_worker' && @connection.auto_attached?(target_info.target_id)
+      finish_initialization_if_ready(target_info.target_id)
+      silent_detach.call
+      if parent_session.is_a?(Puppeteer::CDPSession)
+        target = @target_factory.call(target_info, parent_session)
+        @attached_targets_by_target_id[target_info.target_id] = target
+        emit_event(TargetManagerEmittedEvents::TargetAvailable, target)
+      end
+
+      return
+    end
+
+    unless @target_filter_callback.call(target_info)
+      @ignored_targets << target_info.target_id
+      finish_initialization_if_ready(target_info.target_id)
+      silent_detach.call
+
+      return
+    end
+
+    target = @attached_targets_by_target_id[target_info.target_id] || @target_factory.call(target_info, session)
+    setup_attachment_listeners(session)
+
+    @attached_targets_by_target_id[target_info.target_id] ||= target
+    @attached_targets_by_session_id[session.id] = target
+
+    @target_interceptors[parent_session]&.each do |interceptor|
+      if parent_session.is_a?(Puppeteer::Connection)
+        interceptor.call(target, nil)
+      else
+        # Sanity check: if parent session is not a connection, it should be
+        # present in #attachedTargetsBySessionId.
+        attached_target = @attached_targets_by_session_id[parent_session.id]
+        unless attached_target
+          raise "No target found for the parent session: #{parent_session.id}"
+        end
+        interceptor.call(target, attached_target)
+      end
+    end
+
+    @target_ids_for_init.delete(target.target_id)
+    future { emit_event(TargetManagerEmittedEvents::TargetAvailable, target) }
+
+    if @target_ids_for_init.empty?
+      @initialize_promise.fulfill(nil) unless @initialize_promise.resolved?
+    end
+
+    future do
+      # TODO: the browser might be shutting down here. What do we do with the error?
+      await_all(
+        session.async_send_message('Target.setAutoAttach', {
+          waitForDebuggerOnStart: true,
+          flatten: true,
+          autoAttach: true,
+        }),
+        session.async_send_message('Runtime.runIfWaitingForDebugger'),
+      )
+    rescue => err
+      Logger.new($stderr).warn(err)
+    end
+  end
+
+  private def finish_initialization_if_ready(target_id)
+    @target_ids_for_init.delete(target_id)
+    if @target_ids_for_init.empty?
+      @initialize_promise.fulfill(nil) unless @initialize_promise.resolved?
+    end
+  end
+
+  private def handle_detached_from_target(parent_session, event)
+    session_id = event['sessionId']
+    target = @attached_targets_by_session_id.delete(session_id)
+    return unless target
+    @attached_targets_by_target_id.delete(target.target_id)
+    emit_event(TargetManagerEmittedEvents::TargetGone, target)
+  end
+end