RubyGems - ferrum - Versions diffs - 0.3 → 0.4 - Mend

ferrum 0.3 → 0.4

Files changed (16) hide show

checksums.yaml +4 -4
data/README.md +92 -62
data/lib/ferrum/browser.rb +2 -8
data/lib/ferrum/network.rb +141 -0
data/lib/ferrum/network/error.rb +18 -16
data/lib/ferrum/network/exchange.rb +43 -0
data/lib/ferrum/network/intercepted_request.rb +57 -48
data/lib/ferrum/network/request.rb +39 -26
data/lib/ferrum/network/response.rb +46 -39
data/lib/ferrum/page.rb +25 -104
data/lib/ferrum/page/dom.rb +17 -0
data/lib/ferrum/page/frame.rb +20 -8
data/lib/ferrum/page/screenshot.rb +1 -5
data/lib/ferrum/version.rb +1 -1
metadata +4 -3
data/lib/ferrum/page/net.rb +0 -83

checksums.yaml CHANGED

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c762e928c31b548af3ecfc9e2ca3cd90e7315c34c313387a0bde299bc201e31
-  data.tar.gz: c46c348b3631700428723ca110bb4a40e0106cfca9b84ab0f743bf4356f348e9
+  metadata.gz: 6f524fb695add05ab04355a4f694710f0cda6ce317048f234cd6d597a6db5b54
+  data.tar.gz: 4f755ea3ce87efa0a92b2689188ec8ce78979b716e8053e9687fe4f9e97baf2e
 SHA512:
-  metadata.gz: 854dd57cdd66bfd68535d88f087b145e4f700afdb078c3f0776e374d1fd9c919010c32282cef9b5ac4bb1c2cff21ee7307c7d5b3e3e58bc354cf4477206b8cd9
-  data.tar.gz: 207738d98111f3d67322ab926f022e184672b73ae8246acb4a01260d12db07d602f10c29b6612ca1830926091444dae51aeaf0896c5e33d8f202fe9c791dc2a7
+  metadata.gz: f52f33ea6d6292ab9f829f9e4870ba7be5fe0f9c045f4b4fe1e540b5363440027ae8890987e92e95612b91e1d82d28a19ff3d204cabeac0bebde96943d399baa
+  data.tar.gz: 3156ae9cc12403dd2b497119371b895d7f37865318ffa2d424676df91346403d85386615abe2db020dffe77b055de72b5e44ccf69137d9bd1fa01a9ae75ffa0f

data/README.md CHANGED

@@ -159,15 +159,6 @@ browser.goto("https://github.com/")
 browser.refresh
 ```
-#### status : `Integer`
-Contains the status code of the response (e.g., 200 for a success).
-```ruby
-browser.goto("https://github.com/")
-browser.status # => 200
-```
 ## Finders
@@ -261,8 +252,8 @@ browser.body # => '<html itemscope="" itemtype="http://schema.org/WebPage" lang=
 Saves screenshot on a disk or returns it as base64.
 * options `Hash`
-  * :path `String` to save a screenshot on the disk. If passed `:encoding` is
-    set to `:binary` automatically
+  * :path `String` to save a screenshot on the disk. `:encoding` will be set to
+    `:binary` automatically
   * :encoding `Symbol` `:base64` | `:binary` you can set it to return image as
     Base64
   * :format `String` "jpeg" | "png"
@@ -286,8 +277,8 @@ browser.screenshot(full: true, quality: 60) # "iVBORw0KGgoAAAANSUhEUgAABAAAAAMAC
 Saves PDF on a disk or returns it as base64.
 * options `Hash`
-  * :path `String` to save a screenshot on the disk. If passed `:encoding` is
-    set to `:binary` automatically
+  * :path `String` to save a pdf on the disk. `:encoding` will be set to
+    `:binary` automatically
   * :encoding `Symbol` `:base64` | `:binary` you can set it to return pdf as
     Base64
   * :landscape `Boolean` paper orientation. Defaults to false.
@@ -305,17 +296,98 @@ browser.pdf(path: "google.pdf", paper_width: 1.0, paper_height: 1.0) # => 14983
 ## Network
-#### network_traffic : `Array<Network::Request>`
+browser.network
+#### traffic `Array<Network::Exchange>`
+Returns all information about network traffic as `Network::Exchange` instance
+which in general is a wrapper around `request`, `response` and `error`.
+```ruby
+browser.goto("https://github.com/")
+browser.network.traffic # => [#<Ferrum::Network::Exchange, ...]
+```
+#### request : `Network::Request`
+Page request of the main frame.
+```ruby
+browser.goto("https://github.com/")
+browser.network.request # => #<Ferrum::Network::Request...
+```
+#### response : `Network::Response`
+Page response of the main frame.
+```ruby
+browser.goto("https://github.com/")
+browser.network.response # => #<Ferrum::Network::Response...
+```
-Returns all information about network traffic as a request/response array.
+#### status : `Integer`
+Contains the status code of the main page response (e.g., 200 for a
+success). This is just a shortcut for `response.status`.
+```ruby
+browser.goto("https://github.com/")
+browser.network.status # => 200
+```
+#### clear(type)
+Clear browser's cache or collected traffic.
+* type `Symbol` it is either `:traffic` or `:cache`
+```ruby
+traffic = browser.network.traffic # => []
+browser.goto("https://github.com/")
+traffic.size # => 51
+browser.network.clear(:traffic)
+traffic.size # => 0
+```
+#### intercept(\*\*options)
+Set request interception for given options. This method is only sets request
+interception, you should use `on` callback to catch requests and abort or
+continue them.
+* options `Hash`
+  * :pattern `String` \* by default
+  * :resource_type `Symbol` one of the [resource types](https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceType)
+```ruby
+browser = Ferrum::Browser.new
+browser.network.intercept
+browser.on(:request) do |request|
+  if request.match?(/bla-bla/)
+    request.abort
+  else
+    request.continue
+  end
+end
+browser.goto("https://google.com")
+```
-#### clear_network_traffic
+#### authorize(\*\*options)
-Cleans up collected data.
+If site uses authorization you can provide credentials using this method.
-#### response_headers : `Hash`
+* options `Hash`
+  * :type `Symbol` `:server` | `:proxy` site or proxy authorization
+  * :user `String`
+  * :password `String`
-Returns all headers for a given request in `goto` method.
+```ruby
+browser.network.authorize(user: "login", password: "pass")
+browser.goto("http://example.com/authenticated")
+puts browser.network.status # => 200
+puts browser.body # => Welcome, authenticated client
+```
 ### Mouse
@@ -540,23 +612,12 @@ Play around inside given frame
 browser.goto("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
 frame = browser.at_xpath("//iframe")
 browser.within_frame(frame) do
+  puts browser.frame_title # => HTML Demo: <iframe>
   puts browser.frame_url # => https://interactive-examples.mdn.mozilla.net/pages/tabbed/iframe.html
 end
 ```
-## Authorization
-#### authorize(\*\*options)
-If site uses authorization you can provide credentials using this method.
-* options `Hash`
-  * :type `Symbol` `:server` | `:proxy` site or proxy authorization
-  * :user `String`
-  * :password `String`
 ## Dialog
 #### accept(text)
@@ -580,34 +641,3 @@ browser.on(:dialog) do |dialog|
 end
 browser.goto("https://google.com")
 ```
-## Interception
-#### intercept_request(\*\*options)
-Set request interception for given options. This method is only sets request
-interception, you should use `on` callback to catch it.
-* options `Hash`
-  * :pattern `String` \* by default
-  * :resource_type `Symbol` one of the [resource types](https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceType)
-#### on(event)
-Set callback for given event.
-* event `Symbol`
-```ruby
-browser = Ferrum::Browser.new
-browser.intercept_request
-browser.on(:request_intercepted) do |request|
-  if request.match?(/bla-bla/)
-    request.abort
-  else
-    request.continue
-  end
-end
-browser.goto("https://google.com")
-```

data/lib/ferrum/browser.rb CHANGED

@@ -16,15 +16,13 @@ module Ferrum
     extend Forwardable
     delegate %i[window_handle window_handles switch_to_window
                 open_new_window close_window within_window page] => :targets
-    delegate %i[goto back forward refresh status
+    delegate %i[goto back forward refresh
                 at_css at_xpath css xpath current_url title body
-                headers cookies network_traffic clear_network_traffic response_headers
-                intercept_request continue_request abort_request
+                headers cookies network
                 mouse keyboard
                 screenshot pdf
                 evaluate evaluate_on evaluate_async execute
                 frame_url frame_title within_frame
-                authorize
                 on] => :page
     attr_reader :client, :process, :logger, :js_errors, :slowmo, :base_url,
@@ -82,10 +80,6 @@ module Ferrum
       raise
     end
-    def clear_memory_cache
-      page.command("Network.clearBrowserCache")
-    end
     def reset
       @window_size = @original_window_size
       targets.reset

data/lib/ferrum/network.rb ADDED

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+require "ferrum/network/exchange"
+require "ferrum/network/intercepted_request"
+module Ferrum
+  class Network
+    CLEAR_TYPE = %i[traffic cache].freeze
+    AUTHORIZE_TYPE = %i[server proxy].freeze
+    RESOURCE_TYPES = %w[Document Stylesheet Image Media Font Script TextTrack
+                        XHR Fetch EventSource WebSocket Manifest
+                        SignedExchange Ping CSPViolationReport Other].freeze
+    attr_reader :traffic
+    def initialize(page)
+      @page = page
+      @traffic = []
+      @exchange = nil
+    end
+    def request
+      @exchange&.request
+    end
+    def response
+      @exchange&.response
+    end
+    def status
+      response&.status
+    end
+    def clear(type)
+      unless CLEAR_TYPE.include?(type)
+        raise ArgumentError, ":type should be in #{CLEAR_TYPE}"
+      end
+      if type == :traffic
+        @traffic.clear
+      else
+        @page.command("Network.clearBrowserCache")
+      end
+      true
+    end
+    def intercept(pattern: "*", resource_type: nil)
+      pattern = { urlPattern: pattern }
+      if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
+        pattern[:resourceType] = resource_type
+      end
+      @page.command("Network.setRequestInterception", patterns: [pattern])
+    end
+    def authorize(user:, password:, type: :server)
+      unless AUTHORIZE_TYPE.include?(type)
+        raise ArgumentError, ":type should be in #{AUTHORIZE_TYPE}"
+      end
+      @authorized_ids ||= {}
+      @authorized_ids[type] ||= []
+      intercept
+      @page.on(:request) do |request, index, total|
+        if request.auth_challenge?(type)
+          response = authorized_response(@authorized_ids[type],
+                                         request.interception_id,
+                                         user, password)
+          @authorized_ids[type] << request.interception_id
+          request.continue(authChallengeResponse: response)
+        elsif index + 1 < total
+          next # There are other callbacks that can handle this, skip
+        else
+          request.continue
+        end
+      end
+    end
+    def subscribe
+      @page.on("Network.requestWillBeSent") do |params|
+        # On redirects Chrome doesn't change `requestId` and there's no
+        # `Network.responseReceived` event for such request. If there's already
+        # exchange object with this id then we got redirected and params has
+        # `redirectResponse` key which contains the response.
+        if exchange = first_by(params["requestId"])
+          exchange.build_response(params)
+        end
+        exchange = Network::Exchange.new(params)
+        @exchange = exchange if exchange.navigation_request?(@page.frame_id)
+        @traffic << exchange
+      end
+      @page.on("Network.responseReceived") do |params|
+        if exchange = last_by(params["requestId"])
+          exchange.build_response(params)
+        end
+      end
+      @page.on("Network.loadingFinished") do |params|
+        exchange = last_by(params["requestId"])
+        if exchange && exchange.response
+          exchange.response.body_size = params["encodedDataLength"]
+        end
+      end
+      @page.on("Log.entryAdded") do |params|
+        entry = params["entry"] || {}
+        if entry["source"] == "network" &&
+            entry["level"] == "error" &&
+            exchange = last_by(entry["networkRequestId"])
+          exchange.build_error(entry)
+        end
+      end
+    end
+    def authorized_response(ids, interception_id, username, password)
+      if ids.include?(interception_id)
+        { response: "CancelAuth" }
+      elsif username && password
+        { response: "ProvideCredentials",
+          username: username,
+          password: password }
+      else
+        { response: "CancelAuth" }
+      end
+    end
+    def first_by(request_id)
+      @traffic.find { |e| e.request.id == request_id }
+    end
+    def last_by(request_id)
+      @traffic.select { |e| e.request.id == request_id }.last
+    end
+  end
+end

data/lib/ferrum/network/error.rb CHANGED

@@ -1,25 +1,27 @@
 # frozen_string_literal: true
-module Ferrum::Network
-  class Error
-    def initialize(data)
-      @data = data
-    end
+module Ferrum
+  class Network
+    class Error
+      def initialize(data)
+        @data = data
+      end
-    def id
-      @data["networkRequestId"]
-    end
+      def id
+        @data["networkRequestId"]
+      end
-    def url
-      @data["url"]
-    end
+      def url
+        @data["url"]
+      end
-    def description
-      @data["text"]
-    end
+      def description
+        @data["text"]
+      end
-    def time
-      @time ||= Time.strptime(@data["timestamp"].to_s, "%s")
+      def time
+        @time ||= Time.strptime(@data["timestamp"].to_s, "%s")
+      end
     end
   end
 end

data/lib/ferrum/network/exchange.rb ADDED

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+require "ferrum/network/error"
+require "ferrum/network/request"
+require "ferrum/network/response"
+module Ferrum
+  class Network
+    class Exchange
+      attr_reader :request, :response, :error
+      def initialize(params)
+        @response = @error = nil
+        build_request(params)
+      end
+      def build_request(params)
+        @request = Network::Request.new(params)
+      end
+      def build_response(params)
+        @response = Network::Response.new(params)
+      end
+      def build_error(params)
+        @error = Network::Error.new(params)
+      end
+      def navigation_request?(frame_id)
+        request.type?(:document) &&
+          request.frame_id == frame_id
+      end
+      def blocked?
+        response.nil?
+      end
+      def to_a
+        [request, response, error]
+      end
+    end
+  end
+end

data/lib/ferrum/network/intercepted_request.rb CHANGED

@@ -1,53 +1,62 @@
 # frozen_string_literal: true
-module Ferrum::Network
-  class InterceptedRequest
-    attr_accessor :interception_id, :frame_id, :resource_type,
-                  :is_navigation_request
-    def initialize(page, params)
-      @page, @params = page, params
-      @interception_id = params["interceptionId"]
-      @frame_id = params["frameId"]
-      @resource_type = params["resourceType"]
-      @is_navigation_request = params["isNavigationRequest"]
-      @request = params.dig("request")
-    end
-    def auth_challenge?(source)
-      @params.dig("authChallenge", "source")&.downcase&.to_s == source.to_s
-    end
-    def match?(regexp)
-      !!url.match(regexp)
-    end
-    def abort
-      @page.abort_request(interception_id)
-    end
-    def continue(**options)
-      @page.continue_request(interception_id, **options)
-    end
-    def url
-      @request["url"]
-    end
-    def method
-      @request["method"]
-    end
-    def headers
-      @request["headers"]
-    end
-    def initial_priority
-      @request["initialPriority"]
-    end
-    def referrer_policy
-      @request["referrerPolicy"]
+module Ferrum
+  class Network
+    class InterceptedRequest
+      attr_accessor :interception_id, :frame_id, :resource_type
+      def initialize(page, params)
+        @page, @params = page, params
+        @interception_id = params["interceptionId"]
+        @frame_id = params["frameId"]
+        @resource_type = params["resourceType"]
+        @request = params["request"]
+      end
+      def navigation_request?
+        @params["isNavigationRequest"]
+      end
+      def auth_challenge?(source)
+        @params.dig("authChallenge", "source")&.downcase&.to_s == source.to_s
+      end
+      def match?(regexp)
+        !!url.match(regexp)
+      end
+      def continue(**options)
+        options = options.merge(interceptionId: interception_id)
+        @page.command("Network.continueInterceptedRequest", **options)
+      end
+      def abort
+        continue(errorReason: "Aborted")
+      end
+      def url
+        @request["url"]
+      end
+      def method
+        @request["method"]
+      end
+      def headers
+        @request["headers"]
+      end
+      def initial_priority
+        @request["initialPriority"]
+      end
+      def referrer_policy
+        @request["referrerPolicy"]
+      end
+      def inspect
+        %(#<#{self.class} @interception_id=#{@interception_id.inspect} @frame_id=#{@frame_id.inspect} @resource_type=#{@resource_type.inspect} @request=#{@request.inspect}>)
+      end
     end
   end
 end

data/lib/ferrum/network/request.rb CHANGED

@@ -2,32 +2,45 @@
 require "time"
-module Ferrum::Network
-  class Request
-    attr_accessor :response, :error
-    def initialize(data)
-      @data = data
-    end
-    def id
-      @data["id"]
-    end
-    def url
-      @data["url"]
-    end
-    def method
-      @data["method"]
-    end
-    def headers
-      @data["headers"]
-    end
-    def time
-      @time ||= Time.strptime(@data["time"].to_s, "%s")
+module Ferrum
+  class Network
+    class Request
+      def initialize(params)
+        @params = params
+        @request = @params["request"]
+      end
+      def id
+        @params["requestId"]
+      end
+      def type
+        @params["type"]
+      end
+      def type?(value)
+        type.downcase == value.to_s.downcase
+      end
+      def frame_id
+        @params["frameId"]
+      end
+      def url
+        @request["url"]
+      end
+      def method
+        @request["method"]
+      end
+      def headers
+        @request["headers"]
+      end
+      def time
+        @time ||= Time.strptime(@params["wallTime"].to_s, "%s")
+      end
     end
   end
 end

data/lib/ferrum/network/response.rb CHANGED

@@ -1,44 +1,51 @@
 # frozen_string_literal: true
-module Ferrum::Network
-  class Response
-    attr_accessor :body_size
-    def initialize(data)
-      @data = data
-    end
-    def id
-      @data["id"]
-    end
-    def url
-      @data["url"]
-    end
-    def status
-      @data["status"]
-    end
-    def status_text
-      @data["statusText"]
-    end
-    def headers
-      @data["headers"]
-    end
-    def headers_size
-      @data["encodedDataLength"]
-    end
-    # FIXME: didn't check if we have it on redirect response
-    def redirect_url
-      @data["redirectURL"]
-    end
-    def content_type
-      @content_type ||= @data.dig("headers", "contentType").sub(/;.*\z/, "")
+module Ferrum
+  class Network
+    class Response
+      attr_reader :body_size
+      def initialize(params)
+        @params = params
+        @response = params["response"] || @params["redirectResponse"]
+      end
+      def id
+        @params["requestId"]
+      end
+      def url
+        @response["url"]
+      end
+      def status
+        @response["status"]
+      end
+      def status_text
+        @response["statusText"]
+      end
+      def headers
+        @response["headers"]
+      end
+      def headers_size
+        @response["encodedDataLength"]
+      end
+      def content_type
+        @content_type ||= @response.dig("headers", "contentType")&.sub(/;.*\z/, "")
+      end
+      # See https://crbug.com/883475
+      # Sometimes we never get the Network.responseReceived event.
+      # See https://crbug.com/764946
+      # `Network.loadingFinished` encodedDataLength contains both body and
+      # headers sizes received by wire.
+      def body_size=(size)
+        @body_size = size - headers_size
+      end
     end
   end
 end

data/lib/ferrum/page.rb CHANGED

@@ -5,34 +5,13 @@ require "ferrum/keyboard"
 require "ferrum/headers"
 require "ferrum/cookies"
 require "ferrum/dialog"
+require "ferrum/network"
 require "ferrum/page/dom"
 require "ferrum/page/runtime"
 require "ferrum/page/frame"
-require "ferrum/page/net"
 require "ferrum/page/screenshot"
 require "ferrum/browser/client"
-require "ferrum/network/error"
-require "ferrum/network/request"
-require "ferrum/network/response"
-require "ferrum/network/intercepted_request"
-# RemoteObjectId is from a JavaScript world, and corresponds to any JavaScript
-# object, including JS wrappers for DOM nodes. There is a way to convert between
-# node ids and remote object ids (DOM.requestNode and DOM.resolveNode).
-#
-# NodeId is used for inspection, when backend tracks the node and sends updates to
-# the frontend. If you somehow got NodeId over protocol, backend should have
-# pushed to the frontend all of it's ancestors up to the Document node via
-# DOM.setChildNodes. After that, frontend is always kept up-to-date about anything
-# happening to the node.
-#
-# BackendNodeId is just a unique identifier for a node. Obtaining it does not send
-# any updates, for example, the node may be destroyed without any notification.
-# This is a way to keep a reference to the Node, when you don't necessarily want
-# to keep track of it. One example would be linking to the node from performance
-# data (e.g. relayout root node). BackendNodeId may be either resolved to
-# inspected node (DOM.pushNodesByBackendIdsToFrontend) or described in more
-# details (DOM.describeNode).
 module Ferrum
   class Page
     NEW_WINDOW_WAIT = ENV.fetch("FERRUM_NEW_WINDOW_WAIT", 0.3).to_f
@@ -51,17 +30,15 @@ module Ferrum
       end
     end
-    include DOM, Runtime, Frame, Net, Screenshot
+    include DOM, Runtime, Frame, Screenshot
     attr_accessor :referrer
-    attr_reader :target_id, :status,
-                :headers, :cookies, :response_headers,
-                :mouse, :keyboard,
-                :browser
+    attr_reader :target_id, :browser,
+                :headers, :cookies, :network,
+                :mouse, :keyboard
     def initialize(target_id, browser, new_window = false)
       @target_id, @browser = target_id, browser
-      @network_traffic = []
       @event = Event.new.tap(&:set)
       @frames = {}
@@ -80,6 +57,7 @@ module Ferrum
       @mouse, @keyboard = Mouse.new(self), Keyboard.new(self)
       @headers, @cookies = Headers.new(self), Cookies.new(self)
+      @network = Network.new(self)
       subscribe
       prepare_page
@@ -131,21 +109,6 @@ module Ferrum
       command("Page.reload", wait: timeout)
     end
-    def network_traffic(type = nil)
-      case type.to_s
-      when "all"
-        @network_traffic
-      when "blocked"
-        @network_traffic.select { |r| r.response.nil? } # when request blocked
-      else
-        @network_traffic.select { |r| r.response } # when request isn't blocked
-      end
-    end
-    def clear_network_traffic
-      @network_traffic = []
-    end
     def back
       history_navigate(delta: -1)
     end
@@ -171,7 +134,7 @@ module Ferrum
           dialog = Dialog.new(self, params)
           block.call(dialog, index, total)
         end
-      when :request_intercepted
+      when :request
         @client.on("Network.requestIntercepted") do |params, index, total|
           request = Network::InterceptedRequest.new(self, params)
           block.call(request, index, total)
@@ -186,87 +149,45 @@ module Ferrum
     def subscribe
       super
+      network.subscribe
+      on("Network.loadingFailed") do |params|
+        id, canceled = params.values_at("requestId", "canceled")
+        # Set event as we aborted main request we are waiting for
+        if network.request&.id == id && canceled == true
+          @event.set
+          @document_id = get_document_id
+        end
+      end
       if @browser.logger
-        @client.on("Runtime.consoleAPICalled") do |params|
+        on("Runtime.consoleAPICalled") do |params|
           params["args"].each { |r| @browser.logger.puts(r["value"]) }
         end
       end
       if @browser.js_errors
-        @client.on("Runtime.exceptionThrown") do |params|
+        on("Runtime.exceptionThrown") do |params|
           # FIXME https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
           Thread.main.raise JavaScriptError.new(params.dig("exceptionDetails", "exception"))
         end
       end
-      @client.on("Page.windowOpen") do
+      on("Page.windowOpen") do
         @browser.targets.refresh
       end
-      @client.on("Page.navigatedWithinDocument") do
+      on("Page.navigatedWithinDocument") do
         @event.set if @waiting_frames.empty?
       end
-      @client.on("Page.domContentEventFired") do |params|
+      on("Page.domContentEventFired") do |params|
         # `frameStoppedLoading` doesn't occur if status isn't success
-        if @status != 200
+        if network.status != 200
           @event.set
           @document_id = get_document_id
         end
       end
-      @client.on("Network.requestWillBeSent") do |params|
-        if params["frameId"] == @frame_id
-          # Possible types:
-          # Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR,
-          # Fetch, EventSource, WebSocket, Manifest, SignedExchange, Ping,
-          # CSPViolationReport, Other
-          if params["type"] == "Document"
-            @event.reset
-            @request_id = params["requestId"]
-          end
-        end
-        id, time = params.values_at("requestId", "wallTime")
-        params = params["request"].merge("id" => id, "time" => time)
-        @network_traffic << Network::Request.new(params)
-      end
-      @client.on("Network.responseReceived") do |params|
-        if params["requestId"] == @request_id
-          @response_headers = params.dig("response", "headers")
-          @status = params.dig("response", "status")
-        end
-        if request = @network_traffic.find { |r| r.id == params["requestId"] }
-          params = params["response"].merge("id" => params["requestId"])
-          request.response = Network::Response.new(params)
-        end
-      end
-      @client.on("Network.loadingFinished") do |params|
-        if request = @network_traffic.find { |r| r.id == params["requestId"] }
-          # Sometimes we never get the Network.responseReceived event.
-          # See https://crbug.com/883475
-          #
-          # Network.loadingFinished's encodedDataLength contains both body and headers
-          # sizes received by wire. See https://crbug.com/764946
-          if response = request.response
-            response.body_size = params["encodedDataLength"] - response.headers_size
-          end
-        end
-      end
-      @client.on("Log.entryAdded") do |params|
-        source = params.dig("entry", "source")
-        level = params.dig("entry", "level")
-        if source == "network" && level == "error"
-          id = params.dig("entry", "networkRequestId")
-          if request = @network_traffic.find { |r| r.id == id }
-            request.error = Network::Error.new(params["entry"])
-          end
-        end
-      end
     end
     def prepare_page

data/lib/ferrum/page/dom.rb CHANGED

@@ -1,5 +1,22 @@
 # frozen_string_literal: true
+# RemoteObjectId is from a JavaScript world, and corresponds to any JavaScript
+# object, including JS wrappers for DOM nodes. There is a way to convert between
+# node ids and remote object ids (DOM.requestNode and DOM.resolveNode).
+#
+# NodeId is used for inspection, when backend tracks the node and sends updates to
+# the frontend. If you somehow got NodeId over protocol, backend should have
+# pushed to the frontend all of it's ancestors up to the Document node via
+# DOM.setChildNodes. After that, frontend is always kept up-to-date about anything
+# happening to the node.
+#
+# BackendNodeId is just a unique identifier for a node. Obtaining it does not send
+# any updates, for example, the node may be destroyed without any notification.
+# This is a way to keep a reference to the Node, when you don't necessarily want
+# to keep track of it. One example would be linking to the node from performance
+# data (e.g. relayout root node). BackendNodeId may be either resolved to
+# inspected node (DOM.pushNodesByBackendIdsToFrontend) or described in more
+# details (DOM.describeNode).
 module Ferrum
   class Page
     module DOM

data/lib/ferrum/page/frame.rb CHANGED

@@ -3,6 +3,8 @@
 module Ferrum
   class Page
     module Frame
+      attr_reader :frame_id
       def execution_context_id
         context_id = current_execution_context_id
         raise NoExecutionContextError unless context_id
@@ -42,28 +44,28 @@ module Ferrum
       def subscribe
         super if defined?(super)
-        @client.on("Page.frameAttached") do |params|
+        on("Page.frameAttached") do |params|
           @frames[params["frameId"]] = { "parent_id" => params["parentFrameId"] }
         end
-        @client.on("Page.frameStartedLoading") do |params|
+        on("Page.frameStartedLoading") do |params|
           @waiting_frames << params["frameId"]
           @event.reset
         end
-        @client.on("Page.frameNavigated") do |params|
+        on("Page.frameNavigated") do |params|
           id = params["frame"]["id"]
           if frame = @frames[id]
             frame.merge!(params["frame"].select { |k, _| k == "name" || k == "url" })
           end
         end
-        @client.on("Page.frameScheduledNavigation") do |params|
+        on("Page.frameScheduledNavigation") do |params|
           @waiting_frames << params["frameId"]
           @event.reset
         end
-        @client.on("Page.frameStoppedLoading") do |params|
+        on("Page.frameStoppedLoading") do |params|
           # `DOM.performSearch` doesn't work without getting #document node first.
           # It returns node with nodeId 1 and nodeType 9 from which descend the
           # tree and we save it in a variable because if we call that again root
@@ -79,7 +81,17 @@ module Ferrum
           end
         end
-        @client.on("Runtime.executionContextCreated") do |params|
+        on("Network.requestWillBeSent") do |params|
+          if params["frameId"] == @frame_id
+            # Possible types:
+            # Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR,
+            # Fetch, EventSource, WebSocket, Manifest, SignedExchange, Ping,
+            # CSPViolationReport, Other
+            @event.reset if params["type"] == "Document"
+          end
+        end
+        on("Runtime.executionContextCreated") do |params|
           context_id = params.dig("context", "id")
           @execution_context_id ||= context_id
@@ -93,7 +105,7 @@ module Ferrum
           end
         end
-        @client.on("Runtime.executionContextDestroyed") do |params|
+        on("Runtime.executionContextDestroyed") do |params|
           context_id = params["executionContextId"]
           if @execution_context_id == context_id
@@ -104,7 +116,7 @@ module Ferrum
           frame["execution_context_id"] = nil if frame
         end
-        @client.on("Runtime.executionContextsCleared") do
+        on("Runtime.executionContextsCleared") do
           # If we didn't have time to set context id at the beginning we have
           # to set lock and release it when we set something.
           @execution_context_id = nil

data/lib/ferrum/page/screenshot.rb CHANGED

@@ -23,17 +23,13 @@ module Ferrum
       def save_file(path, data)
         bin = Base64.decode64(data)
+        return bin unless path
         File.open(path.to_s, "wb") { |f| f.write(bin) }
       end
       def common_options(encoding: :base64, path: nil, **_)
         encoding = encoding.to_sym
         encoding = :binary if path
-        if encoding == :binary && !path
-          raise "You have to provide `:path` for `:binary` encoding"
-        end
         [path, encoding]
       end

data/lib/ferrum/version.rb CHANGED

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 module Ferrum
-  VERSION = "0.3"
+  VERSION = "0.4"
 end

metadata CHANGED

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ferrum
 version: !ruby/object:Gem::Version
-  version: '0.3'
+  version: '0.4'
 platform: ruby
 authors:
 - Dmitry Vorotilin
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-12 00:00:00.000000000 Z
+date: 2019-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: websocket-driver
@@ -205,7 +205,9 @@ files:
 - lib/ferrum/keyboard.json
 - lib/ferrum/keyboard.rb
 - lib/ferrum/mouse.rb
+- lib/ferrum/network.rb
 - lib/ferrum/network/error.rb
+- lib/ferrum/network/exchange.rb
 - lib/ferrum/network/intercepted_request.rb
 - lib/ferrum/network/request.rb
 - lib/ferrum/network/response.rb
@@ -213,7 +215,6 @@ files:
 - lib/ferrum/page.rb
 - lib/ferrum/page/dom.rb
 - lib/ferrum/page/frame.rb
-- lib/ferrum/page/net.rb
 - lib/ferrum/page/runtime.rb
 - lib/ferrum/page/screenshot.rb
 - lib/ferrum/targets.rb

data/lib/ferrum/page/net.rb DELETED

@@ -1,83 +0,0 @@
-# frozen_string_literal: true
-module Ferrum
-  class Page
-    module Net
-      AUTHORIZE_TYPE = %i[server proxy]
-      RESOURCE_TYPES = %w[Document Stylesheet Image Media Font Script TextTrack
-                          XHR Fetch EventSource WebSocket Manifest
-                          SignedExchange Ping CSPViolationReport Other]
-      def authorize(user:, password:, type: :server)
-        unless AUTHORIZE_TYPE.include?(type)
-          raise ArgumentError, ":type should be in #{AUTHORIZE_TYPE}"
-        end
-        @authorized_ids ||= {}
-        @authorized_ids[type] ||= []
-        intercept_request
-        on(:request_intercepted) do |request, index, total|
-          if request.auth_challenge?(type)
-            response = authorized_response(@authorized_ids[type],
-                                           request.interception_id,
-                                           user, password)
-            @authorized_ids[type] << request.interception_id
-            request.continue(authChallengeResponse: response)
-          elsif index + 1 < total
-            next # There are other callbacks that can handle this, skip
-          else
-            request.continue
-          end
-        end
-      end
-      def intercept_request(pattern: "*", resource_type: nil)
-        pattern = { urlPattern: pattern }
-        if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
-          pattern[:resourceType] = resource_type
-        end
-        command("Network.setRequestInterception", patterns: [pattern])
-      end
-      def continue_request(interception_id, options = nil)
-        options ||= {}
-        options = options.merge(interceptionId: interception_id)
-        command("Network.continueInterceptedRequest", **options)
-      end
-      def abort_request(interception_id)
-        continue_request(interception_id, errorReason: "Aborted")
-      end
-      private
-      def subscribe
-        super if defined?(super)
-        @client.on("Network.loadingFailed") do |params|
-          # Free mutex as we aborted main request we are waiting for
-          if params["requestId"] == @request_id && params["canceled"] == true
-            @event.set
-            @document_id = get_document_id
-          end
-        end
-      end
-      def authorized_response(ids, interception_id, username, password)
-        if ids.include?(interception_id)
-          { response: "CancelAuth" }
-        elsif username && password
-          { response: "ProvideCredentials",
-            username: username,
-            password: password }
-        else
-          { response: "CancelAuth" }
-        end
-      end
-    end
-  end
-end