puppeteer-ruby 0.0.10

data/lib/puppeteer/browser_context.rb

@@ -0,0 +1,90 @@
+class Puppeteer::BrowserContext
+  include Puppeteer::EventCallbackable
+
+  # @param {!Puppeteer.Connection} connection
+  # @param {!Browser} browser
+  # @param {?string} contextId
+  def initialize(connection, browser, context_id)
+    @connection = connection
+    @browser = browser
+    @id = context_id
+  end
+
+  # @return {!Array<!Target>} target
+  def targets
+    @browser.targets.select { |target| target.browser_context == self }
+  end
+
+  # @param {function(!Target):boolean} predicate
+  # @param {{timeout?: number}=} options
+  # @return {!Promise<!Target>}
+  def wait_for_target(predicate:, timeout: nil)
+    @browser.wait_for_target(
+      predicate: ->(target) { target.browser_context == self && predicate.call(target) },
+      timeout: timeout,
+    )
+  end
+
+  # @return {!Promise<!Array<!Puppeteer.Page>>}
+  def pages
+    targets.select { |target| target.type == 'page' }.map(&:page).reject { |page| !page }
+  end
+
+  def incognito?
+    !@id
+  end
+
+  # /**
+  #  * @param {string} origin
+  #  * @param {!Array<string>} permissions
+  #  */
+  # async overridePermissions(origin, permissions) {
+  #   const webPermissionToProtocol = new Map([
+  #     ['geolocation', 'geolocation'],
+  #     ['midi', 'midi'],
+  #     ['notifications', 'notifications'],
+  #     ['push', 'push'],
+  #     ['camera', 'videoCapture'],
+  #     ['microphone', 'audioCapture'],
+  #     ['background-sync', 'backgroundSync'],
+  #     ['ambient-light-sensor', 'sensors'],
+  #     ['accelerometer', 'sensors'],
+  #     ['gyroscope', 'sensors'],
+  #     ['magnetometer', 'sensors'],
+  #     ['accessibility-events', 'accessibilityEvents'],
+  #     ['clipboard-read', 'clipboardRead'],
+  #     ['clipboard-write', 'clipboardWrite'],
+  #     ['payment-handler', 'paymentHandler'],
+  #     // chrome-specific permissions we have.
+  #     ['midi-sysex', 'midiSysex'],
+  #   ]);
+  #   permissions = permissions.map(permission => {
+  #     const protocolPermission = webPermissionToProtocol.get(permission);
+  #     if (!protocolPermission)
+  #       throw new Error('Unknown permission: ' + permission);
+  #     return protocolPermission;
+  #   });
+  #   await this._connection.send('Browser.grantPermissions', {origin, browserContextId: this._id || undefined, permissions});
+  # }
+
+  # async clearPermissionOverrides() {
+  #   await this._connection.send('Browser.resetPermissions', {browserContextId: this._id || undefined});
+  # }
+
+  # @return [Future<Puppeteer::Page>]
+  def new_page
+    @browser.create_page_in_context(@id)
+  end
+
+  # @return [Browser]
+  def browser
+    @browser
+  end
+
+  def close
+    if !@id
+      raise 'Non-incognito profiles cannot be closed!'
+    end
+    @browser.dispose_context(@id)
+  end
+end

data/lib/puppeteer/browser_fetcher.rb

@@ -0,0 +1,6 @@
+# Download latest chromium.
+class Puppeteer::BrowserFetcher
+  def initialize(project_root, options = {})
+    # 未実装
+  end
+end

data/lib/puppeteer/browser_runner.rb

@@ -0,0 +1,161 @@
+require 'fileutils'
+require 'open3'
+require 'timeout'
+
+# https://github.com/puppeteer/puppeteer/blob/master/lib/Launcher.js
+class Puppeteer::BrowserRunner
+  # @param {string} executablePath
+  # @param {!Array<string>} processArguments
+  # @param {string=} tempDirectory
+  def initialize(executable_path, process_arguments, temp_directory)
+    @executable_path = executable_path
+    @process_arguments = process_arguments
+    @temp_directory = temp_directory
+    @proc = nil
+    @connection = nil
+    @closed = true
+    @listeners = []
+  end
+
+  attr_reader :proc, :connection
+
+  class BrowserProcess
+    def initialize(env, executable_path, args)
+      stdin, @stdout, @stderr, @thread = Open3.popen3(env, executable_path, *args)
+      stdin.close
+      @pid = @thread.pid
+    end
+
+    def kill
+      Process.kill(:KILL, @pid)
+    rescue Errno::ESRCH
+      # already killed
+    end
+
+    def dispose
+      [@stdout, @stderr].each { |io| io.close unless io.closed? }
+      @thread.join
+    end
+
+    attr_reader :stdout, :stderr
+  end
+
+  # @param {!(Launcher.LaunchOptions)=} options
+  def start(
+    executable_path: nil,
+    ignore_default_args: nil,
+    handle_SIGINT: nil,
+    handle_SIGTERM: nil,
+    handle_SIGHUP: nil,
+    timeout: nil,
+    dumpio: nil,
+    env: nil,
+    pipe: nil
+  )
+    @launch_options = Puppeteer::Launcher::LaunchOptions.new({
+      executable_path: executable_path,
+      ignore_default_args: ignore_default_args,
+      handle_SIGINT: handle_SIGINT,
+      handle_SIGTERM: handle_SIGTERM,
+      handle_SIGHUP: handle_SIGHUP,
+      timeout: timeout,
+      dumpio: dumpio,
+      env: env,
+      pipe: pipe,
+    }.compact)
+    @proc = BrowserProcess.new(
+      @launch_options.env,
+      @executable_path,
+      @process_arguments,
+    )
+    # if (dumpio) {
+    #   this.proc.stderr.pipe(process.stderr);
+    #   this.proc.stdout.pipe(process.stdout);
+    # }
+    @closed = false
+    @process_closing = -> {
+      @proc.dispose
+      @closed = true
+      if @temp_directory
+        FileUtils.rm_rf(@temp_directory)
+      end
+    }
+    trap(:EXIT) do
+      kill
+    end
+
+    if @launch_options.handle_SIGINT?
+      trap(:INT) do
+        kill
+        exit 130
+      end
+    end
+
+    if @launch_options.handle_SIGTERM?
+      trap(:TERM) do
+        close
+      end
+    end
+
+    if @launch_options.handle_SIGHUP?
+      trap(:HUP) do
+        close
+      end
+    end
+  end
+
+  # @return {Promise}
+  def close
+    return if @closed
+
+    if @temp_directory
+      kill
+    elsif @connection
+      begin
+        @connection.send_message('Browser.close')
+      rescue
+        kill
+      end
+    end
+
+    @process_closing.call
+  end
+
+  # @return {Promise}
+  def kill
+    unless @closed
+      @proc.kill
+    end
+    if @temp_directory
+      FileUtils.rm_rf(@temp_directory)
+    end
+  end
+
+
+  # @param {!({usePipe?: boolean, timeout: number, slowMo: number, preferredRevision: string})} options
+  # @return {!Promise<!Connection>}
+  def setup_connection(use_pipe:, timeout:, slow_mo:, preferred_revision:)
+    if !use_pipe
+      browser_ws_endpoint = wait_for_ws_endpoint(@proc, timeout, preferred_revision)
+      transport = Puppeteer::WebSocketTransport.create(browser_ws_endpoint)
+      @connection = Puppeteer::Connection.new(browser_ws_endpoint, transport, slow_mo)
+    else
+      raise NotImplementedError.new('PipeTransport is not yet implemented')
+    end
+
+    @connection
+  end
+
+  private def wait_for_ws_endpoint(browser_process, timeout, preferred_revision)
+    Timeout.timeout(timeout / 1000.0) do
+      loop do
+        line = browser_process.stderr.readline
+        /^DevTools listening on (ws:\/\/.*)$/.match(line) do |m|
+          return m[1]
+        end
+      end
+    end
+  rescue Timeout::Error
+    raise Puppeteer::TimeoutError.new("Timed out after #{timeout} ms while trying to connect to the browser! Only Chrome at revision r#{preferredRevision} is guaranteed to work.")
+  end
+end

data/lib/puppeteer/cdp_session.rb

@@ -0,0 +1,100 @@
+class Puppeteer::CDPSession
+  include Puppeteer::DebugPrint
+  include Puppeteer::EventCallbackable
+  using Puppeteer::AsyncAwaitBehavior
+
+  class Error < StandardError; end
+
+  # @param {!Connection} connection
+  # @param {string} targetType
+  # @param {string} sessionId
+  def initialize(connection, target_type, session_id)
+    @callbacks = {}
+    @connection = connection
+    @target_type = target_type
+    @session_id = session_id
+    @pending_messages = {}
+  end
+
+  attr_reader :connection
+
+  # @param method [String]
+  # @param params [Hash]
+  # @returns [Hash]
+  def send_message(method, params = {})
+    await async_send_message(method, params)
+  end
+
+  # @param method [String]
+  # @param params [Hash]
+  # @returns [Future<Hash>]
+  def async_send_message(method, params = {})
+    if !@connection
+      raise Error.new("Protocol error (#{method}): Session closed. Most likely the #{@target_type} has been closed.")
+    end
+    id = @connection.raw_send(message: { sessionId: @session_id, method: method, params: params })
+    promise = resolvable_future
+    callback = Puppeteer::Connection::MessageCallback.new(method: method, promise: promise)
+    if pending_message = @pending_messages.delete(id)
+      debug_puts "Pending message (id: #{id}) is handled"
+      callback_with_message(callback, pending_message)
+    else
+      @callbacks[id] = callback
+    end
+    promise
+  end
+
+  # @param {{id?: number, method: string, params: Object, error: {message: string, data: any}, result?: *}} object
+  def handle_message(message)
+    if message['id']
+      if callback = @callbacks.delete(message['id'])
+        callback_with_message(callback, message)
+      else
+        debug_puts "unknown id: #{id}. Store it into pending message"
+
+        # RECV is often notified before SEND.
+        # Wait about 10 frames before throwing an error.
+        message_id = message['id']
+        @pending_messages[message_id] = message
+        Concurrent::Promises.schedule(0.16, message_id) do |id|
+          if @pending_messages.delete(id)
+            raise Error.new("unknown id: #{id}")
+          end
+        end
+      end
+    else
+      emit_event message['method'], message['params']
+    end
+  end
+
+  private def callback_with_message(callback, message)
+    if message['error']
+      callback.reject(
+        Puppeteer::Connection::ProtocolError.new(
+          method: callback.method,
+          error_message: message['error']['message'],
+          error_data: message['error']['data']))
+    else
+      callback.resolve(message['result'])
+    end
+  end
+
+  def detach
+    if !@connection
+      raise Error.new("Session already detarched. Most likely the #{@target_type} has been closed.")
+    end
+    @connection.send_message('Target.detachFromTarget',  sessionId: @session_id)
+  end
+
+  def handle_closed
+    @callbacks.each_value do |callback|
+      callback.reject(
+        Puppeteer::Connection::ProtocolError.new(
+          method: callback.method,
+          error_message: 'Target Closed.'))
+    end
+    @callbacks.clear
+    @connection = nil
+    emit_event 'Events.CDPSession.Disconnected'
+  end
+end

data/lib/puppeteer/concurrent_ruby_utils.rb

@@ -0,0 +1,37 @@
+# utility methods for Concurrent::Promises.
+module Puppeteer::ConcurrentRubyUtils
+  def await_all(*args)
+    if args.length == 1 && args[0].is_a?(Enumerable)
+      Concurrent::Promises.zip(*(args[0])).value!
+    else
+      Concurrent::Promises.zip(*args).value!
+    end
+  end
+
+  def await_any(*args)
+    if args.length == 1 && args[0].is_a?(Enumerable)
+      Concurrent::Promises.any(*(args[0])).value!
+    else
+      Concurrent::Promises.any(*args).value!
+    end
+  end
+
+  # blocking get value of Future.
+  def await(future_or_value)
+    if future_or_value.is_a?(Concurrent::Promises::Future)
+      future_or_value.value!
+    else
+      future_or_value
+    end
+  end
+
+  def future(&block)
+    Concurrent::Promises.future(&block)
+  end
+
+  def resolvable_future
+    Concurrent::Promises.resolvable_future
+  end
+end
+
+include Puppeteer::ConcurrentRubyUtils

data/lib/puppeteer/connection.rb

@@ -0,0 +1,254 @@
+require 'json'
+
+class Puppeteer::Connection
+  include Puppeteer::DebugPrint
+  include Puppeteer::EventCallbackable
+  using Puppeteer::AsyncAwaitBehavior
+
+  class ProtocolError < StandardError
+    def initialize(method:, error_message:, error_data: nil)
+      msg = "Protocol error (#{method}): #{error_message}"
+      if error_data
+        super("#{msg} #{error_data}")
+      else
+        super(msg)
+      end
+    end
+  end
+
+  # callback object stored in @callbacks.
+  class MessageCallback
+    # @param method [String]
+    # @param promise [Concurrent::Promises::ResolvableFuture]
+    def initialize(method:, promise:)
+      @method = method
+      @promise = promise
+    end
+
+    def resolve(result)
+      @promise.fulfill(result)
+    end
+
+    def reject(error)
+      @promise.reject(error)
+    end
+
+    attr_reader :method
+  end
+
+  def initialize(url, transport, delay = 0)
+    @url = url
+    @last_id = 0
+    @callbacks = {}
+    @delay = delay
+
+    @transport = transport
+    @transport.on_message do |data|
+      async_handle_message(JSON.parse(data))
+    end
+    @transport.on_close do |reason, code|
+      handle_close(reason, code)
+    end
+
+    @sessions = {}
+    @closed = false
+  end
+
+  def self.from_session(session)
+    session.connection
+  end
+
+  # @param {string} sessionId
+  # @return {?CDPSession}
+  def session(session_id)
+    @sessions[session_id]
+  end
+
+  def url
+    @url
+  end
+
+  # @param {string} method
+  # @param {!Object=} params
+  def send_message(method, params = {})
+    await async_send_message(method, params)
+  end
+
+  def async_send_message(method, params = {})
+    id = raw_send(message: { method: method, params: params })
+    promise = resolvable_future
+    @callbacks[id] = MessageCallback.new(method: method, promise: promise)
+    promise
+  end
+
+  private def generate_id
+    @last_id += 1
+  end
+
+  def raw_send(message:)
+    id = generate_id
+    payload = JSON.fast_generate(message.compact.merge(id: id))
+    @transport.send_text(payload)
+    request_debug_printer.handle_payload(payload)
+    id
+  end
+
+  # Just for effective debugging :)
+  class RequestDebugPrinter
+    include Puppeteer::DebugPrint
+
+    def handle_payload(payload)
+      debug_puts "SEND >> #{decorate(payload)}"
+    end
+
+    private def decorate(payload)
+      payload.gsub(/"method":"([^"]+)"/, "\"method\":\"\u001b[32m\\1\u001b[0m\"")
+    end
+  end
+
+  class ResponseDebugPrinter
+    include Puppeteer::DebugPrint
+
+    NON_DEBUG_PRINT_METHODS = [
+      'Network.dataReceived',
+      'Network.loadingFinished',
+      'Network.requestWillBeSent',
+      'Network.requestWillBeSentExtraInfo',
+      'Network.responseReceived',
+      'Network.responseReceivedExtraInfo',
+      'Page.lifecycleEvent',
+    ]
+
+    def handle_message(message)
+      if skip_debug_print?(message['method'])
+        debug_print '.'
+        @prev_log_skipped = true
+      else
+        debug_print "\n" if @prev_log_skipped
+        @prev_log_skipped = nil
+        debug_puts "RECV << #{decorate(message)}"
+      end
+    end
+
+    private def skip_debug_print?(method)
+      method && NON_DEBUG_PRINT_METHODS.include?(method)
+    end
+
+    private def decorate(message)
+      # decorate RED for error.
+      if message['error']
+        return "\u001b[31m#{message}\u001b[0m"
+      end
+
+      # ignore method call response, or with no method.
+      return message if message['id'] || !message['method']
+
+      # decorate cyan for method name.
+      message.to_s.gsub(message['method'], "\u001b[36m#{message['method']}\u001b[0m")
+    end
+  end
+
+  private def request_debug_printer
+    @request_debug_printer ||= RequestDebugPrinter.new
+  end
+
+  private def response_debug_printer
+    @response_debug_printer ||= ResponseDebugPrinter.new
+  end
+
+  private def handle_message(message)
+    if @delay > 0
+      sleep(@delay / 1000.0)
+    end
+
+    response_debug_printer.handle_message(message)
+
+    case message['method']
+    when 'Target.attachedToTarget'
+      session_id = message['params']['sessionId']
+      session = Puppeteer::CDPSession.new(self, message['params']['targetInfo']['type'], session_id)
+      @sessions[session_id] = session
+    when 'Target.detachedFromTarget'
+      session_id = message['params']['sessionId']
+      session = @sessions[session_id]
+      if session
+        session.handle_closed
+        @sessions.delete(session_id)
+      end
+    end
+
+    if message['sessionId']
+      session_id = message['sessionId']
+      @sessions[session_id]&.handle_message(message)
+    elsif message['id']
+      # Callbacks could be all rejected if someone has called `.dispose()`.
+      if callback = @callbacks.delete(message['id'])
+        if message['error']
+          callback.reject(
+            ProtocolError.new(
+              method: callback.method,
+              error_message: response['error']['message'],
+              error_data: response['error']['data']))
+        else
+          callback.resolve(message['result'])
+        end
+      end
+    else
+      emit_event message['method'], message['params']
+    end
+  end
+
+  private async def async_handle_message(message)
+    handle_message(message)
+  end
+
+  private def handle_close
+    return if @closed
+    @closed = true
+    @transport.on_message
+    @transport.on_close
+    @callbacks.each_value do |callback|
+      callback.reject(
+        ProtocolError.new(
+          method: callback.method,
+          error_message: 'Target Closed.'))
+    end
+    @callbacks.clear
+    @sessions.each_value do |session|
+      session.handle_closed
+    end
+    @sessions.clear
+    emit_event 'Events.Connection.Disconnected'
+  end
+
+  def on_close(&block)
+    @on_close = block
+  end
+
+  def on_message(&block)
+    @on_message = block
+  end
+
+  def dispose
+    handle_close
+    @transport.close
+  end
+
+  # @param {Protocol.Target.TargetInfo} targetInfo
+  # @return [CDPSession]
+  def create_session(target_info)
+    result = send_message('Target.attachToTarget', targetId: target_info.target_id, flatten: true)
+    session_id = result['sessionId']
+
+    # Target.attachedToTarget is often notified after the result of Target.attachToTarget.
+    # D, [2020-04-04T23:04:30.736311 #91875] DEBUG -- : RECV << {"id"=>2, "result"=>{"sessionId"=>"DA002F8A95B04710502CB40D8430B95A"}}
+    # D, [2020-04-04T23:04:30.736649 #91875] DEBUG -- : RECV << {"method"=>"Target.attachedToTarget", "params"=>{"sessionId"=>"DA002F8A95B04710502CB40D8430B95A", "targetInfo"=>{"targetId"=>"EBAB949A7DE63F12CB94268AD3A9976B", "type"=>"page", "title"=>"about:blank", "url"=>"about:blank", "attached"=>true, "browserContextId"=>"46D23767E9B79DD9E589101121F6DADD"}, "waitingForDebugger"=>false}}
+    # So we have to wait for "Target.attachedToTarget" a bit.
+    20.times do
+      if @sessions[session_id]
+        return @sessions[session_id]
+      end
+      sleep 0.1
+    end
+  end
+end