puppeteer-ruby 0.0.10

data/lib/puppeteer/frame.rb

@@ -0,0 +1,286 @@
+class Puppeteer::Frame
+  # @param {!FrameManager} frameManager
+  # @param {!Puppeteer.CDPSession} client
+  # @param {?Frame} parentFrame
+  # @param {string} frameId
+  def initialize(frame_manager, client, parent_frame, frame_id)
+    @frame_manager = frame_manager
+    @client = client
+    @parent_frame = parent_frame
+    @id = frame_id
+    @detached = false
+
+    @loader_id = ''
+    @lifecycle_events = Set.new
+    @main_world = Puppeteer::DOMWorld.new(frame_manager, self, frame_manager.timeout_settings)
+    @secondary_world = Puppeteer::DOMWorld.new(frame_manager, self, frame_manager.timeout_settings)
+    @child_frames = Set.new
+    if parent_frame
+      parent_frame._child_frames << self
+    end
+  end
+
+  attr_accessor :frame_manager, :id, :loader_id, :lifecycle_events, :main_world, :secondary_world
+
+  # @param url [String]
+  # @param rederer [String]
+  # @param timeout [number|nil]
+  # @param wait_until [string|nil] 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2'
+  # @return [Puppeteer::Response]
+  def goto(url, referer: nil, timeout: nil, wait_until: nil)
+    @frame_manager.navigate_frame(self, url, referer: referer, timeout: timeout, wait_until: wait_until)
+  end
+
+  # @param timeout [number|nil]
+  # @param wait_until [string|nil] 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2'
+  def wait_for_navigation(timeout: nil, wait_until: nil)
+    @frame_manager.wait_for_frame_navigation(self, timeout: timeout, wait_until: wait_until)
+  end
+
+  def execution_context
+    @main_world.execution_context
+  end
+
+  # @param {Function|string} pageFunction
+  # @return {!Promise<!Puppeteer.JSHandle>}
+  def evaluate_handle(page_function, *args)
+    @main_world.evaluate_handle(page_function, *args)
+  end
+
+  # @param {Function|string} pageFunction
+  # @param {!Array<*>} args
+  def evaluate(page_function, *args)
+    @main_world.evaluate(page_function, *args)
+  end
+
+  # `$()` in JavaScript. $ is not allowed to use as a method name in Ruby.
+  # @param {string} selector
+  # @return {!Promise<?Puppeteer.ElementHandle>}
+  def S(selector)
+    @main_world.S(selector)
+  end
+
+
+  # `$x()` in JavaScript. $ is not allowed to use as a method name in Ruby.
+  # @param {string} expression
+  # @return {!Promise<!Array<!Puppeteer.ElementHandle>>}
+  def Sx(expression)
+    @main_world.Sx(expression)
+  end
+
+
+  # `$eval()` in JavaScript. $ is not allowed to use as a method name in Ruby.
+  # @param {string} selector
+  # @param {Function|string} pageFunction
+  # @param {!Array<*>} args
+  # @return {!Promise<(!Object|undefined)>}
+  def Seval(selector, page_function, *args)
+    @main_world.Seval(selector, page_function, *args)
+  end
+
+  # `$$eval()` in JavaScript. $ is not allowed to use as a method name in Ruby.
+  # @param {string} selector
+  # @param {Function|string} pageFunction
+  # @param {!Array<*>} args
+  # @return {!Promise<(!Object|undefined)>}
+  def SSeval(selector, page_function, *args)
+    @main_world.SSeval(selector, page_function, *args)
+  end
+
+  # `$$()` in JavaScript. $ is not allowed to use as a method name in Ruby.
+  # @param {string} selector
+  # @return {!Promise<!Array<!Puppeteer.ElementHandle>>}
+  def SS(selector)
+    @main_world.SS(selector)
+  end
+
+  def content
+    @secondary_world.content
+  end
+
+  # @param {string} html
+  # @param {!{timeout?: number, waitUntil?: string|!Array<string>}=} options
+  def set_content(html, timeout: nil, wait_until: nil)
+    @secondary_world.set_content(html, timeout: timeout, wait_until: wait_until)
+  end
+
+  # @return [String]
+  def name
+    @name || ''
+  end
+
+  # @return [String]
+  def url
+    @url
+  end
+
+  # @return [Frame?]
+  def parent_frame
+    @parent_frame
+  end
+
+  protected def _child_frames
+    @child_frames
+  end
+
+  def child_frames
+    @child_frames.dup
+  end
+
+  def detached?
+    @detached
+  end
+
+  # @param style_tag [Puppeteer::Page::ScriptTag]
+  # @return {!Promise<!ElementHandle>}
+  def add_script_tag(script_tag)
+    @main_world.add_script_tag(script_tag)
+  end
+
+  # @param style_tag [Puppeteer::Page::StyleTag]
+  # @return {!Promise<!ElementHandle>}
+  def add_style_tag(style_tag)
+    @main_world.add_style_tag(style_tag)
+  end
+
+  # @param selector [String]
+  # @param delay [Number]
+  # @param button [String] "left"|"right"|"middle"
+  # @param click_count [Number]
+  def click(selector, delay: nil, button: nil, click_count: nil)
+    @secondary_world.click(selector, delay: delay, button: button, click_count: click_count)
+  end
+
+  # @param {string} selector
+  def focus(selector)
+    @secondary_world.focus(selector)
+  end
+
+  # @param {string} selector
+  def hover(selector)
+    @secondary_world.hover(selector)
+  end
+
+  # @param {string} selector
+  # @param {!Array<string>} values
+  # @return {!Promise<!Array<string>>}
+  def select(selector, *values)
+    @secondary_world.select(selector, *values)
+  end
+
+  # @param {string} selector
+  def tap(selector)
+    @secondary_world.tap(selector)
+  end
+
+  # @param selector [String]
+  # @param text [String]
+  # @param delay [Number]
+  def type_text(selector, text, delay: nil)
+    @main_world.type_text(selector, text, delay: delay)
+  end
+
+  # /**
+  #  * @param {(string|number|Function)} selectorOrFunctionOrTimeout
+  #  * @param {!Object=} options
+  #  * @param {!Array<*>} args
+  #  * @return {!Promise<?Puppeteer.JSHandle>}
+  #  */
+  # waitFor(selectorOrFunctionOrTimeout, options = {}, ...args) {
+  #   const xPathPattern = '//';
+
+  #   if (helper.isString(selectorOrFunctionOrTimeout)) {
+  #     const string = /** @type {string} */ (selectorOrFunctionOrTimeout);
+  #     if (string.startsWith(xPathPattern))
+  #       return this.waitForXPath(string, options);
+  #     return this.waitForSelector(string, options);
+  #   }
+  #   if (helper.isNumber(selectorOrFunctionOrTimeout))
+  #     return new Promise(fulfill => setTimeout(fulfill, /** @type {number} */ (selectorOrFunctionOrTimeout)));
+  #   if (typeof selectorOrFunctionOrTimeout === 'function')
+  #     return this.waitForFunction(selectorOrFunctionOrTimeout, options, ...args);
+  #   return Promise.reject(new Error('Unsupported target type: ' + (typeof selectorOrFunctionOrTimeout)));
+  # }
+
+  # @param selector [String]
+  # @param visible [Boolean] Wait for element visible (not 'display: none' nor 'visibility: hidden') on true. default to false.
+  # @param hidden [Boolean] Wait for element invisible ('display: none' nor 'visibility: hidden') on true. default to false.
+  # @param timeout [Integer]
+  def wait_for_selector(selector, visible: nil, hidden: nil, timeout: nil)
+    handle = @secondary_world.wait_for_selector(selector, visible: visible, hidden: hidden, timeout: timeout)
+    if !handle
+      return nil
+    end
+    main_execution_context = @main_world.execution_context
+    result = main_execution_context.adopt_element_handle(handle)
+    handle.dispose
+    result
+  end
+
+  # @param xpath [String]
+  # @param visible [Boolean] Wait for element visible (not 'display: none' nor 'visibility: hidden') on true. default to false.
+  # @param hidden [Boolean] Wait for element invisible ('display: none' nor 'visibility: hidden') on true. default to false.
+  # @param timeout [Integer]
+  def wait_for_xpath(xpath, visible: nil, hidden: nil, timeout: nil)
+    handle = @secondary_world.wait_for_xpath(xpath, visible: visible, hidden: hidden, timeout: timeout)
+    if !handle
+      return nil
+    end
+    main_execution_context = @main_world.execution_context
+    result = main_execution_context.adopt_element_handle(handle)
+    handle.dispose
+    result
+  end
+
+  # @param {Function|string} pageFunction
+  # @param {!{polling?: string|number, timeout?: number}=} options
+  # @param {!Array<*>} args
+  # @return {!Promise<!Puppeteer.JSHandle>}
+  def wait_for_function(page_function, options = {}, *args)
+    @main_world.wait_for_function(page_function, options, *args)
+  end
+
+  def title
+    @secondary_world.title
+  end
+
+  # @param frame_payload [Hash]
+  def navigated(frame_payload)
+    @name = frame_payload['name']
+    # TODO(lushnikov): remove this once requestInterception has loaderId exposed.
+    @navigation_url = frame_payload['url']
+    @url = frame_payload['url']
+
+    # Ensure loaderId updated.
+    # The order of [Page.lifecycleEvent name="init"] and [Page.frameNavigated] is random... for some reason...
+    @loader_id = frame_payload['loaderId']
+  end
+
+  # @param url [String]
+  def navigated_within_document(url)
+    @url = url
+  end
+
+  def handle_lifecycle_event(loader_id, name)
+    if name == 'init'
+      @loader_id = loader_id
+      @lifecycle_events.clear
+    end
+    @lifecycle_events << name
+  end
+
+  def handle_loading_stopped
+    @lifecycle_events << 'DOMContentLoaded'
+    @lifecycle_events << 'load'
+  end
+
+  def detach
+    @detached = true
+    @main_world.detach
+    @secondary_world.detach
+    if @parent_frame
+      @parent_frame._child_frames.delete(self)
+    end
+    @parent_frame = nil
+  end
+end

data/lib/puppeteer/frame_manager.rb

@@ -0,0 +1,378 @@
+require 'timeout'
+
+class Puppeteer::FrameManager
+  include Puppeteer::DebugPrint
+  include Puppeteer::IfPresent
+  include Puppeteer::EventCallbackable
+  using Puppeteer::AsyncAwaitBehavior
+
+  UTILITY_WORLD_NAME = '__puppeteer_utility_world__'
+
+  # @param {!Puppeteer.CDPSession} client
+  # @param {!Puppeteer.Page} page
+  # @param {boolean} ignoreHTTPSErrors
+  # @param {!Puppeteer.TimeoutSettings} timeoutSettings
+  def initialize(client, page, ignore_https_errors, timeout_settings)
+    @client = client
+    @page = page
+    @network_manager = Puppeteer::NetworkManager.new(client, ignore_https_errors, self)
+    @timeout_settings = timeout_settings
+
+    # @type {!Map<string, !Frame>}
+    @frames = {}
+
+    # @type {!Map<number, !ExecutionContext>}
+    @context_id_to_context = {}
+    @context_id_created = {}
+
+    # @type {!Set<string>}
+    @isolated_worlds = Set.new
+
+    @client.on_event 'Page.frameAttached' do |event|
+      handle_frame_attached(event['frameId'], event['parentFrameId'])
+    end
+    @client.on_event 'Page.frameNavigated' do |event|
+      handle_frame_navigated(event['frame'])
+    end
+    @client.on_event 'Page.navigatedWithinDocument' do |event|
+      handle_frame_navigated_within_document(event['frameId'], event['url'])
+    end
+    @client.on_event 'Page.frameDetached' do |event|
+      handle_frame_detached(event['frameId'])
+    end
+    @client.on_event 'Page.frameStoppedLoading' do |event|
+      handle_frame_stopped_loading(event['frameId'])
+    end
+    @client.on_event 'Runtime.executionContextCreated' do |event|
+      handle_execution_context_created(event['context'])
+    end
+    @client.on_event 'Runtime.executionContextDestroyed' do |event|
+      handle_execution_context_destroyed(event['executionContextId'])
+    end
+    @client.on_event 'Runtime.executionContextsCleared' do |event|
+      handle_execution_contexts_cleared
+    end
+    @client.on_event 'Page.lifecycleEvent' do |event|
+      handle_lifecycle_event(event)
+    end
+  end
+
+  attr_reader :client, :timeout_settings
+
+  private def init
+    results = await_all(
+      @client.async_send_message('Page.enable'),
+      @client.async_send_message('Page.getFrameTree'),
+    )
+    frame_tree = results.last['frameTree']
+    handle_frame_tree(frame_tree)
+    await_all(
+      @client.async_send_message('Page.setLifecycleEventsEnabled', enabled: true),
+      @client.async_send_message('Runtime.enable'),
+    )
+    ensure_isolated_world(UTILITY_WORLD_NAME)
+    @network_manager.init
+  end
+
+  async def async_init
+    init
+  end
+
+  attr_reader :network_manager
+
+  class NavigationError < StandardError; end
+
+  # @param frame [Puppeteer::Frame]
+  # @param url [String]
+  # @param {!{referer?: string, timeout?: number, waitUntil?: string|!Array<string>}=} options
+  # @return [Puppeteer::Response]
+  def navigate_frame(frame, url, referer: nil, timeout: nil, wait_until: nil)
+    assert_no_legacy_navigation_options(wait_until: wait_until)
+
+    navigate_params = {
+      url: url,
+      referer: referer || @network_manager.extra_http_headers['referer'],
+      frameId: frame.id,
+    }.compact
+    option_wait_until = wait_until || ['load']
+    option_timeout = timeout || @timeout_settings.navigation_timeout
+
+    watcher = Puppeteer::LifecycleWatcher.new(self, frame, option_wait_until, option_timeout)
+    ensure_new_document_navigation = false
+
+    begin
+      navigate = future do
+        result = @client.send_message('Page.navigate', navigate_params)
+        loader_id = result['loaderId']
+        ensure_new_document_navigation = !!loader_id
+        if result['errorText']
+          raise NavigationError.new("#{result['errorText']} at #{url}")
+        end
+      end
+      await_any(
+        navigate,
+        watcher.timeout_or_termination_promise,
+      )
+
+      document_navigation_promise =
+        if ensure_new_document_navigation
+          watcher.new_document_navigation_promise
+        else
+          watcher.same_document_navigation_promise
+        end
+      await_any(
+        document_navigation_promise,
+        watcher.timeout_or_termination_promise,
+      )
+    ensure
+      watcher.dispose
+    end
+
+    watcher.navigation_response
+  end
+
+  # @param timeout [number|nil]
+  # @param wait_until [string|nil] 'load' | 'domcontentloaded' | 'networkidle0' | 'networkidle2'
+  # @return [Puppeteer::Response]
+  def wait_for_frame_navigation(frame, timeout: nil, wait_until: nil)
+    assert_no_legacy_navigation_options(wait_until: wait_until)
+
+    option_wait_until = wait_until || ['load']
+    option_timeout = timeout || @timeout_settings.navigation_timeout
+    watcher = Puppeteer::LifecycleWatcher.new(self, frame, option_wait_until, option_timeout)
+    begin
+      await_any(
+        watcher.timeout_or_termination_promise,
+        watcher.same_document_navigation_promise,
+        watcher.new_document_navigation_promise,
+      )
+    ensure
+      watcher.dispose
+    end
+
+    watcher.navigation_response
+  end
+
+  # @param event [Hash]
+  def handle_lifecycle_event(event)
+    frame = @frames[event['frameId']]
+    return if !frame
+    frame.handle_lifecycle_event(event['loaderId'], event['name'])
+    emit_event 'Events.FrameManager.LifecycleEvent', frame
+  end
+
+  # @param {string} frameId
+  def handle_frame_stopped_loading(frame_id)
+    frame = @frames[frame_id]
+    return if !frame
+    frame.handle_loading_stopped
+    emit_event 'Events.FrameManager.LifecycleEvent', frame
+  end
+
+  # @param frame_tree [Hash]
+  def handle_frame_tree(frame_tree)
+    if frame_tree['frame']['parentId']
+      handle_frame_attached(frame_tree['frame']['id'], frame_tree['frame']['parentId'])
+    end
+    handle_frame_navigated(frame_tree['frame'])
+    return if !frame_tree['childFrames']
+
+    frame_tree['childFrames'].each do |child|
+      handle_frame_tree(child)
+    end
+  end
+
+  # @return {!Puppeteer.Page}
+  def page
+    @page
+  end
+
+  # @return {!Frame}
+  def main_frame
+    @main_frame
+  end
+
+  # @return {!Array<!Frame>}
+  def frames
+    @frames.values
+  end
+
+  # @param {!string} frameId
+  # @return {?Frame}
+  def frame(frame_id)
+    @frames[frame_id]
+  end
+
+  # @param {string} frameId
+  # @param {?string} parentFrameId
+  def handle_frame_attached(frame_id, parent_frame_id)
+    return if @frames.has_key?(frame_id)
+    if !parent_frame_id
+      raise ArgymentError.new('parent_frame_id must not be nil')
+    end
+    parent_frame = @frames[parent_frame_id]
+    frame = Puppeteer::Frame.new(self, @client, parent_frame, frame_id)
+    @frames[frame_id] = frame
+
+    emit_event 'Events.FrameManager.FrameAttached', frame
+  end
+
+  # @param frame_payload [Hash]
+  def handle_frame_navigated(frame_payload)
+    is_main_frame = !frame_payload['parentId']
+    frame =
+      if is_main_frame
+        @main_frame
+      else
+        @frames[frame_payload['id']]
+      end
+
+    if !is_main_frame && !frame
+      raise ArgumentError.new('We either navigate top level or have old version of the navigated frame')
+    end
+
+    # Detach all child frames first.
+    if frame
+      frame.child_frames.each do |child|
+        remove_frame_recursively(child)
+      end
+    end
+
+    # Update or create main frame.
+    if is_main_frame
+      if frame
+        # Update frame id to retain frame identity on cross-process navigation.
+        @frames.delete(frame.id)
+        frame.id = frame_payload['id']
+      else
+        # Initial main frame navigation.
+        frame = Puppeteer::Frame.new(self, @client, nil, frame_payload['id'])
+      end
+      @frames[frame_payload['id']] = frame
+      @main_frame = frame
+    end
+
+    # Update frame payload.
+    frame.navigated(frame_payload)
+
+    emit_event 'Events.FrameManager.FrameNavigated', frame
+  end
+
+  # @param name [String]
+  def ensure_isolated_world(name)
+    return if @isolated_worlds.include?(name)
+    @isolated_worlds << name
+
+    @client.send_message('Page.addScriptToEvaluateOnNewDocument',
+      source: "//# sourceURL=#{Puppeteer::ExecutionContext::EVALUATION_SCRIPT_URL}",
+      worldName: name,
+    )
+    create_isolated_worlds_promises = frames.map do |frame|
+      @client.async_send_message('Page.createIsolatedWorld',
+        frameId: frame.id,
+        grantUniveralAccess: true,
+        worldName: name,
+      )
+    end
+    await_all(*create_isolated_worlds_promises)
+  end
+
+  # @param frame_id [String]
+  # @param url [String]
+  def handle_frame_navigated_within_document(frame_id, url)
+    frame = @frames[frame_id]
+    return if !frame
+    frame.navigated_within_document(url)
+    emit_event 'Events.FrameManager.FrameNavigatedWithinDocument', frame
+    emit_event 'Events.FrameManager.FrameNavigated', frame
+  end
+
+  # @param frame_id [String]
+  def handle_frame_detached(frame_id)
+    frame = @frames[frame_id]
+    if frame
+      remove_frame_recursively(frame)
+    end
+  end
+
+  # @param context_payload [Hash]
+  def handle_execution_context_created(context_payload)
+    frame = if_present(context_payload.dig('auxData', 'frameId')) { |frame_id| @frames[frame_id] }
+
+    world = nil
+    if frame
+      if context_payload.dig('auxData', 'isDefault')
+        world = frame.main_world
+      elsif context_payload['name'] == UTILITY_WORLD_NAME && !frame.secondary_world.has_context?
+        # In case of multiple sessions to the same target, there's a race between
+        # connections so we might end up creating multiple isolated worlds.
+        # We can use either.
+        world = frame.secondary_world
+      end
+    end
+
+    if context_payload.dig('auxData', 'type') == 'isolated'
+      @isolated_worlds << context_payload['name']
+    end
+
+    context = Puppeteer::ExecutionContext.new(@client, context_payload, world)
+    if world
+      world.context = context
+    end
+    @context_id_to_context[context_payload['id']] = context
+    @context_id_created[context_payload['id']] = Time.now
+  end
+
+  # @param {number} executionContextId
+  def handle_execution_context_destroyed(execution_context_id)
+    context = @context_id_to_context[execution_context_id]
+    return if !context
+    @context_id_to_context.delete(execution_context_id)
+    @context_id_created.delete(execution_context_id)
+    if context.world
+      context.world.delete_context(execution_context_id)
+    end
+  end
+
+  def handle_execution_contexts_cleared
+    # executionContextsCleared is often notified after executionContextCreated.
+    #   D, [2020-04-06T01:47:03.101227 #13823] DEBUG -- : RECV << {"method"=>"Runtime.executionContextCreated", "params"=>{"context"=>{"id"=>5, "origin"=>"https://github.com", "name"=>"", "auxData"=>{"isDefault"=>true, "type"=>"default", "frameId"=>"71C347B70848B89DDDEFAA8AB5B0BC92"}}}, "sessionId"=>"53F088EED260C28001D26A019F95D9E3"}
+    #   D, [2020-04-06T01:47:03.101439 #13823] DEBUG -- : RECV << {"method"=>"Page.frameNavigated", "params"=>{"frame"=>{"id"=>"71C347B70848B89DDDEFAA8AB5B0BC92", "loaderId"=>"80338225D035AC96BAE8F6D4E81C7D51", "url"=>"https://github.com/search?q=puppeteer", "securityOrigin"=>"https://github.com", "mimeType"=>"text/html"}}, "sessionId"=>"53F088EED260C28001D26A019F95D9E3"}
+    #   D, [2020-04-06T01:47:03.101325 #13823] DEBUG -- : RECV << {"method"=>"Target.targetInfoChanged", "params"=>{"targetInfo"=>{"targetId"=>"71C347B70848B89DDDEFAA8AB5B0BC92", "type"=>"page", "title"=>"https://github.com/search?q=puppeteer", "url"=>"https://github.com/search?q=puppeteer", "attached"=>true, "browserContextId"=>"AF37BC660284CE1552B4ECB147BE9305"}}}
+    #   D, [2020-04-06T01:47:03.101269 #13823] DEBUG -- : RECV << {"method"=>"Runtime.executionContextsCleared", "params"=>{}, "sessionId"=>"53F088EED260C28001D26A019F95D9E3"}
+    # it unexpectedly clears the created execution context.
+    # To avoid the problem, just skip recent created ids.
+    now = Time.now
+    context_ids_to_skip = @context_id_created.select { |k, v| now - v < 1 }.keys
+    @context_id_to_context.reject { |k, v| context_ids_to_skip.include?(k) }.each do |execution_context_id, context|
+      if context.world
+        context.world.delete_context(execution_context_id)
+      end
+    end
+    @context_id_to_context.select! { |k, v| context_ids_to_skip.include?(k) }
+  end
+
+  def execution_context_by_id(context_id)
+    context = @context_id_to_context[context_id]
+    if !context
+      raise "INTERNAL ERROR: missing context with id = #{context_id}"
+    end
+    context
+  end
+
+  # @param {!Frame} frame
+  private def remove_frame_recursively(frame)
+    frame.child_frames.each do |child|
+      remove_frame_recursively(child)
+    end
+    frame.detach
+    @frames.delete(frame.id)
+    emit_event 'Events.FrameManager.FrameDetached', frame
+  end
+
+  private def assert_no_legacy_navigation_options(wait_until:)
+    if wait_until == 'networkidle'
+      raise ArgumentError.new('ERROR: "networkidle" option is no longer supported. Use "networkidle2" instead')
+    end
+  end
+end