ferrum 0.1.0

data/lib/ferrum/network/error.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Ferrum::Network
+  class Error
+    def initialize(data)
+      @data = data
+    end
+
+    def id
+      @data["networkRequestId"]
+    end
+
+    def url
+      @data["url"]
+    end
+
+    def description
+      @data["text"]
+    end
+
+    def time
+      @time ||= Time.strptime(@data["timestamp"].to_s, "%s")
+    end
+  end
+end

data/lib/ferrum/network/request.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Ferrum::Network
+  class Request
+    attr_accessor :response, :error
+
+    def initialize(data)
+      @data = data
+    end
+
+    def id
+      @data["id"]
+    end
+
+    def url
+      @data["url"]
+    end
+
+    def method
+      @data["method"]
+    end
+
+    def headers
+      @data["headers"]
+    end
+
+    def time
+      @time ||= Time.strptime(@data["time"].to_s, "%s")
+    end
+  end
+end

data/lib/ferrum/network/response.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Ferrum::Network
+  class Response
+    attr_accessor :body_size
+
+    def initialize(data)
+      @data = data
+    end
+
+    def id
+      @data["id"]
+    end
+
+    def url
+      @data["url"]
+    end
+
+    def status
+      @data["status"]
+    end
+
+    def status_text
+      @data["statusText"]
+    end
+
+    def headers
+      @data["headers"]
+    end
+
+    def headers_size
+      @data["encodedDataLength"]
+    end
+
+    # FIXME: didn't check if we have it on redirect response
+    def redirect_url
+      @data["redirectURL"]
+    end
+
+    def content_type
+      @content_type ||= @data.dig("headers", "contentType").sub(/;.*\z/, "")
+    end
+  end
+end

data/lib/ferrum/node.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module Ferrum
+  class Node
+    attr_reader :page, :target_id, :node_id, :desc
+
+    def initialize(page, target_id, node_id, desc)
+      @page, @target_id, @node_id, @desc =
+        page, target_id, node_id, desc
+    end
+
+    def node?
+      desc["nodeType"] == 1 # nodeType: 3, nodeName: "#text" e.g.
+    end
+
+    def page_send(name, *args)
+      page.send(name, self, *args)
+    rescue BrowserError => e
+      case e.message
+      when "No node with given id found"
+        raise ObsoleteNode.new(self, e.response)
+      else
+        raise
+      end
+    end
+
+    def at_xpath(selector)
+      page.at_xpath(selector, within: self)
+    end
+
+    def at_css(selector)
+      page.at_css(selector, within: self)
+    end
+
+    def xpath(selector)
+      page.xpath(selector, within: self)
+    end
+
+    def css(selector)
+      page.css(selector, within: self)
+    end
+
+    def text
+      page.evaluate_on(node: self, expression: "this.textContent")
+    end
+
+    def property(name)
+      page_send(:property, name)
+    end
+
+    def [](name)
+      # Although the attribute matters, the property is consistent. Return that in
+      # preference to the attribute for links and images.
+      if ((tag_name == "img") && (name == "src")) || ((tag_name == "a") && (name == "href"))
+        # if attribute exists get the property
+        return page_send(:attribute, name) && page_send(:property, name)
+      end
+
+      value = property(name)
+      value = page_send(:attribute, name) if value.nil? || value.is_a?(Hash)
+
+      value
+    end
+
+    def attributes
+      page_send(:attributes)
+    end
+
+    def value
+      page.evaluate_on(node: self, expression: "this.value")
+    end
+
+    def set(value)
+      if tag_name == "input"
+        case self[:type]
+        when "radio"
+          click
+        when "checkbox"
+          click if value != checked?
+        when "file"
+          files = value.respond_to?(:to_ary) ? value.to_ary.map(&:to_s) : value.to_s
+          page_send(:select_file, files)
+        else
+          page_send(:set, value.to_s)
+        end
+      elsif tag_name == "textarea"
+        page_send(:set, value.to_s)
+      elsif self[:isContentEditable]
+        # FIXME:
+        page_send(:delete_text)
+        send_keys(value.to_s)
+      end
+    end
+
+    def select_option
+      page_send(:select, true)
+    end
+
+    def unselect_option
+      raise NotImplemented
+    end
+
+    def tag_name
+      @tag_name ||= desc["nodeName"].downcase
+    end
+
+    def visible?
+      page_send(:visible?)
+    end
+
+    def checked?
+      self[:checked]
+    end
+
+    def selected?
+      !!self[:selected]
+    end
+
+    def disabled?
+      page_send(:disabled?)
+    end
+
+    def click(keys = [], offset = {})
+      page_send(:click, keys, offset)
+    end
+
+    def right_click(keys = [], offset = {})
+      page_send(:right_click, keys, offset)
+    end
+
+    def double_click(keys = [], offset = {})
+      page_send(:double_click, keys, offset)
+    end
+
+    def hover
+      page_send(:hover)
+    end
+
+    def trigger(event)
+      page_send(:trigger, event)
+    end
+
+    def scroll_to(element, location, position = nil)
+      if element.is_a?(Node)
+        scroll_element_to_location(element, location)
+      elsif location.is_a?(Symbol)
+        scroll_to_location(location)
+      else
+        scroll_to_coords(*position)
+      end
+      self
+    end
+
+    def ==(other)
+      return false unless other.is_a?(Node)
+      # We compare backendNodeId because once nodeId is sent to frontend backend
+      # never returns same nodeId sending 0. In other words frontend is
+      # responsible for keeping track of node ids.
+      target_id == other.target_id && desc["backendNodeId"] == other.desc["backendNodeId"]
+    end
+
+    def send_keys(*keys)
+      page_send(:send_keys, keys)
+    end
+    alias_method :send_key, :send_keys
+
+    def path
+      page_send(:path)
+    end
+
+    def inspect
+      %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} @desc=#{@desc.inspect}>)
+    end
+  end
+end

data/lib/ferrum/page.rb

@@ -0,0 +1,373 @@
+# frozen_string_literal: true
+
+require "ferrum/page/dom"
+require "ferrum/page/input"
+require "ferrum/page/runtime"
+require "ferrum/page/frame"
+require "ferrum/page/net"
+require "ferrum/browser/client"
+require "ferrum/network/error"
+require "ferrum/network/request"
+require "ferrum/network/response"
+
+# RemoteObjectId is from a JavaScript world, and corresponds to any JavaScript
+# object, including JS wrappers for DOM nodes. There is a way to convert between
+# node ids and remote object ids (DOM.requestNode and DOM.resolveNode).
+#
+# NodeId is used for inspection, when backend tracks the node and sends updates to
+# the frontend. If you somehow got NodeId over protocol, backend should have
+# pushed to the frontend all of it's ancestors up to the Document node via
+# DOM.setChildNodes. After that, frontend is always kept up-to-date about anything
+# happening to the node.
+#
+# BackendNodeId is just a unique identifier for a node. Obtaining it does not send
+# any updates, for example, the node may be destroyed without any notification.
+# This is a way to keep a reference to the Node, when you don't necessarily want
+# to keep track of it. One example would be linking to the node from performance
+# data (e.g. relayout root node). BackendNodeId may be either resolved to
+# inspected node (DOM.pushNodesByBackendIdsToFrontend) or described in more
+# details (DOM.describeNode).
+module Ferrum
+  class Page
+    NEW_WINDOW_BUG_SLEEP = 0.3
+
+    include Input, DOM, Runtime, Frame, Net
+
+    attr_accessor :referrer
+    attr_reader :target_id, :status, :response_headers
+
+    def initialize(target_id, browser, new_window = false)
+      @target_id, @browser = target_id, browser
+      @network_traffic = []
+      @event = Concurrent::Event.new.tap(&:set)
+
+      @frames = {}
+      @waiting_frames ||= Set.new
+      @frame_stack = []
+      @accept_modal = []
+      @modal_messages = []
+
+      # Dirty hack because new window doesn't have events at all
+      sleep(NEW_WINDOW_BUG_SLEEP) if new_window
+
+      begin
+        @session_id = @browser.command("Target.attachToTarget", targetId: @target_id)["sessionId"]
+      rescue BrowserError => e
+        if e.message == "No target with given id found"
+          raise NoSuchWindowError
+        else
+          raise
+        end
+      end
+
+      host = @browser.process.host
+      port = @browser.process.port
+      ws_url = "ws://#{host}:#{port}/devtools/page/#{@target_id}"
+      @client = Browser::Client.new(browser, ws_url, 1000)
+
+      subscribe
+      prepare_page
+    end
+
+    def timeout
+      @browser.timeout
+    end
+
+    def goto(url = nil)
+      options = { url: combine_url!(url) }
+      options.merge!(referrer: referrer) if referrer
+      response = command("Page.navigate", timeout: timeout, **options)
+      # https://cs.chromium.org/chromium/src/net/base/net_error_list.h
+      if %w[net::ERR_NAME_NOT_RESOLVED
+            net::ERR_NAME_RESOLUTION_FAILED
+            net::ERR_INTERNET_DISCONNECTED
+            net::ERR_CONNECTION_TIMED_OUT].include?(response["errorText"])
+        raise StatusFailError, "url" => options[:url]
+      end
+      response["frameId"]
+    end
+
+    def close
+      @browser.command("Target.detachFromTarget", sessionId: @session_id)
+      @browser.command("Target.closeTarget", targetId: @target_id)
+      close_connection
+    end
+
+    def close_connection
+      @client.close
+    end
+
+    def resize(width: nil, height: nil, fullscreen: false)
+      result = @browser.command("Browser.getWindowForTarget", targetId: @target_id)
+      @window_id, @bounds = result.values_at("windowId", "bounds")
+
+      if fullscreen
+        @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { windowState: "fullscreen" })
+      else
+        @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { windowState: "normal" })
+        @browser.command("Browser.setWindowBounds", windowId: @window_id, bounds: { width: width, height: height, windowState: "normal" })
+        command("Emulation.setDeviceMetricsOverride", width: width, height: height, deviceScaleFactor: 1, mobile: false)
+      end
+    end
+
+    def refresh
+      command("Page.reload", timeout: timeout)
+    end
+
+    def network_traffic(type = nil)
+      case type.to_s
+      when "all"
+        @network_traffic
+      when "blocked"
+        @network_traffic.select { |r| r.response.nil? } # when request blocked
+      else
+        @network_traffic.select { |r| r.response } # when request isn't blocked
+      end
+    end
+
+    def clear_network_traffic
+      @network_traffic = []
+    end
+
+    def go_back
+      go(-1)
+    end
+
+    def go_forward
+      go(1)
+    end
+
+    def accept_confirm
+      @accept_modal << true
+    end
+
+    def dismiss_confirm
+      @accept_modal << false
+    end
+
+    def accept_prompt(modal_response)
+      @accept_modal << true
+      @modal_response = modal_response
+    end
+
+    def dismiss_prompt
+      @accept_modal << false
+    end
+
+    def find_modal(options)
+      start_time    = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      timeout_sec   = options.fetch(:wait) { session_wait_time }
+      expect_text   = options[:text]
+      expect_regexp = expect_text.is_a?(Regexp) ? expect_text : Regexp.escape(expect_text.to_s)
+      not_found_msg = "Unable to find modal dialog"
+      not_found_msg += " with #{expect_text}" if expect_text
+
+      begin
+        modal_text = @modal_messages.shift
+        raise ModalNotFound if modal_text.nil? || (expect_text && !modal_text.match(expect_regexp))
+      rescue ModalNotFound => e
+        raise e, not_found_msg if (::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - start_time) >= timeout_sec
+        sleep(0.05)
+        retry
+      end
+
+      modal_text
+    end
+
+    def reset_modals
+      @accept_modal = []
+      @modal_response = nil
+      @modal_messages = []
+    end
+
+    def command(method, timeout: 0, **params)
+      result = @client.command(method, params)
+
+      if timeout > 0
+        @event.reset
+        @event.wait(timeout)
+      end
+
+      result
+    end
+
+    private
+
+    def subscribe
+      super
+
+      if @browser.logger
+        @client.on("Runtime.consoleAPICalled") do |params|
+          params["args"].each { |r| @browser.logger.puts(r["value"]) }
+        end
+      end
+
+      if @browser.js_errors
+        @client.on("Runtime.exceptionThrown") do |params|
+          Thread.main.raise JavaScriptError.new(params.dig("exceptionDetails", "exception"))
+        end
+      end
+
+      @client.on("Page.javascriptDialogOpening") do |params|
+        accept_modal = @accept_modal.last
+        if accept_modal == true || accept_modal == false
+          @accept_modal.pop
+          @modal_messages << params["message"]
+          options = { accept: accept_modal }
+          response = @modal_response || params["defaultPrompt"]
+          options.merge!(promptText: response) if response
+          @client.command("Page.handleJavaScriptDialog", **options)
+        else
+          warn "Modal window has been opened, but you didn't wrap your code into (`accept_prompt` | `dismiss_prompt` | `accept_confirm` | `dismiss_confirm` | `accept_alert`), accepting by default"
+          options = { accept: true }
+          response = params["defaultPrompt"]
+          options.merge!(promptText: response) if response
+          @client.command("Page.handleJavaScriptDialog", **options)
+        end
+      end
+
+      @client.on("Page.windowOpen") do
+        @browser.targets.refresh
+      end
+
+      @client.on("Page.navigatedWithinDocument") do
+        @event.set if @waiting_frames.empty?
+      end
+
+      @client.on("Page.domContentEventFired") do |params|
+        # `frameStoppedLoading` doesn't occur if status isn't success
+        if @status != 200
+          @event.set
+          @document_id = get_document_id
+        end
+      end
+
+      @client.on("Network.requestWillBeSent") do |params|
+        if params["frameId"] == @frame_id
+          # Possible types:
+          # Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR,
+          # Fetch, EventSource, WebSocket, Manifest, SignedExchange, Ping,
+          # CSPViolationReport, Other
+          if params["type"] == "Document"
+            @event.reset
+            @request_id = params["requestId"]
+          end
+        end
+
+        id, time = params.values_at("requestId", "wallTime")
+        params = params["request"].merge("id" => id, "time" => time)
+        @network_traffic << Network::Request.new(params)
+      end
+
+      @client.on("Network.responseReceived") do |params|
+        if params["requestId"] == @request_id
+          @response_headers = params.dig("response", "headers")
+          @status = params.dig("response", "status")
+        end
+
+        if request = @network_traffic.find { |r| r.id == params["requestId"] }
+          params = params["response"].merge("id" => params["requestId"])
+          request.response = Network::Response.new(params)
+        end
+      end
+
+      @client.on("Network.loadingFinished") do |params|
+        if request = @network_traffic.find { |r| r.id == params["requestId"] }
+          # Sometimes we never get the Network.responseReceived event.
+          # See https://crbug.com/883475
+          #
+          # Network.loadingFinished's encodedDataLength contains both body and headers
+          # sizes received by wire. See https://crbug.com/764946
+          if response = request.response
+            response.body_size = params["encodedDataLength"] - response.headers_size
+          end
+        end
+      end
+
+      @client.on("Log.entryAdded") do |params|
+        source = params.dig("entry", "source")
+        level = params.dig("entry", "level")
+        if source == "network" && level == "error"
+          id = params.dig("entry", "networkRequestId")
+          if request = @network_traffic.find { |r| r.id == id }
+            request.error = Network::Error.new(params["entry"])
+          end
+        end
+      end
+    end
+
+    def prepare_page
+      command("Page.enable")
+      command("DOM.enable")
+      command("CSS.enable")
+      command("Runtime.enable")
+      command("Log.enable")
+      command("Network.enable")
+
+      if @browser.options[:save_path]
+        command("Page.setDownloadBehavior", behavior: "allow", downloadPath: @browser.options[:save_path])
+      end
+
+      @browser.extensions.each do |extension|
+        @client.command("Page.addScriptToEvaluateOnNewDocument", source: extension)
+      end
+
+      inject_extensions
+
+      width, height = @browser.window_size
+      resize(width: width, height: height)
+
+      url_whitelist = Array(@browser.url_whitelist)
+      url_blacklist = Array(@browser.url_blacklist)
+      intercept_request("*") if !url_whitelist.empty? || !url_blacklist.empty?
+
+      response = command("Page.getNavigationHistory")
+      if response.dig("entries", 0, "transitionType") != "typed"
+        # If we create page by clicking links, submiting forms and so on it
+        # opens a new window for which `frameStoppedLoading` event never
+        # occurs and thus search for nodes cannot be completed. Here we check
+        # the history and if the transitionType for example `link` then
+        # content is already loaded and we can try to get the document.
+        @document_id = get_document_id
+      end
+    end
+
+    def inject_extensions
+      @browser.extensions.each do |extension|
+        # https://github.com/GoogleChrome/puppeteer/issues/1443
+        # https://github.com/ChromeDevTools/devtools-protocol/issues/77
+        # https://github.com/cyrus-and/chrome-remote-interface/issues/319
+        # We also evaluate script just in case because
+        # `Page.addScriptToEvaluateOnNewDocument` doesn't work in popups.
+        @client.command("Runtime.evaluate", expression: extension,
+                                            contextId: execution_context_id,
+                                            returnByValue: true)
+      end
+    end
+
+    def go(delta)
+      history = command("Page.getNavigationHistory")
+      index, entries = history.values_at("currentIndex", "entries")
+
+      if entry = entries[index + delta]
+        # Potential wait because of network event
+        command("Page.navigateToHistoryEntry", timeout: 0.05, entryId: entry["id"])
+      end
+    end
+
+    def combine_url!(url_or_path)
+      url = Addressable::URI.parse(url_or_path)
+      nil_or_relative = url.nil? || url.relative?
+
+      if nil_or_relative && !@browser.base_url
+        raise "Set :base_url browser's option or use absolute url in `goto`, you passed: #{url_or_path}"
+      end
+
+      nil_or_relative ? @browser.base_url.join(url.to_s) : url
+    end
+
+    def get_document_id
+      command("DOM.getDocument", depth: 0).dig("root", "nodeId")
+    end
+  end
+end