ferrum 0.12 → 0.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +28 -22
  3. data/lib/ferrum/browser/client.rb +6 -5
  4. data/lib/ferrum/browser/command.rb +9 -6
  5. data/lib/ferrum/browser/options/base.rb +1 -4
  6. data/lib/ferrum/browser/options/chrome.rb +22 -10
  7. data/lib/ferrum/browser/options/firefox.rb +3 -6
  8. data/lib/ferrum/browser/options.rb +84 -0
  9. data/lib/ferrum/browser/process.rb +6 -7
  10. data/lib/ferrum/browser/version_info.rb +71 -0
  11. data/lib/ferrum/browser/web_socket.rb +1 -1
  12. data/lib/ferrum/browser/xvfb.rb +1 -1
  13. data/lib/ferrum/browser.rb +184 -64
  14. data/lib/ferrum/context.rb +3 -2
  15. data/lib/ferrum/contexts.rb +2 -2
  16. data/lib/ferrum/cookies/cookie.rb +183 -0
  17. data/lib/ferrum/cookies.rb +122 -49
  18. data/lib/ferrum/dialog.rb +30 -0
  19. data/lib/ferrum/frame/dom.rb +177 -0
  20. data/lib/ferrum/frame/runtime.rb +41 -61
  21. data/lib/ferrum/frame.rb +91 -3
  22. data/lib/ferrum/headers.rb +28 -0
  23. data/lib/ferrum/keyboard.rb +45 -2
  24. data/lib/ferrum/mouse.rb +84 -0
  25. data/lib/ferrum/network/exchange.rb +104 -5
  26. data/lib/ferrum/network/intercepted_request.rb +3 -12
  27. data/lib/ferrum/network/request.rb +58 -19
  28. data/lib/ferrum/network/request_params.rb +57 -0
  29. data/lib/ferrum/network/response.rb +106 -4
  30. data/lib/ferrum/network.rb +193 -8
  31. data/lib/ferrum/node.rb +21 -1
  32. data/lib/ferrum/page/animation.rb +16 -0
  33. data/lib/ferrum/page/frames.rb +66 -11
  34. data/lib/ferrum/page/screenshot.rb +97 -0
  35. data/lib/ferrum/page/tracing.rb +26 -0
  36. data/lib/ferrum/page.rb +158 -45
  37. data/lib/ferrum/proxy.rb +91 -2
  38. data/lib/ferrum/target.rb +6 -4
  39. data/lib/ferrum/version.rb +1 -1
  40. metadata +7 -101
@@ -0,0 +1,57 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Network
5
+ #
6
+ # Common methods used by both {Request} and {InterceptedRequest}.
7
+ #
8
+ module RequestParams
9
+ #
10
+ # The URL for the request.
11
+ #
12
+ # @return [String]
13
+ #
14
+ def url
15
+ @request["url"]
16
+ end
17
+
18
+ #
19
+ # The URL fragment for the request.
20
+ #
21
+ # @return [String, nil]
22
+ #
23
+ def url_fragment
24
+ @request["urlFragment"]
25
+ end
26
+
27
+ #
28
+ # The request method.
29
+ #
30
+ # @return [String]
31
+ #
32
+ def method
33
+ @request["method"]
34
+ end
35
+
36
+ #
37
+ # The request headers.
38
+ #
39
+ # @return [Hash{String => String}]
40
+ #
41
+ def headers
42
+ @request["headers"]
43
+ end
44
+
45
+ #
46
+ # The optional HTTP `POST` form data.
47
+ #
48
+ # @return [String, nil]
49
+ # The HTTP `POST` form data.
50
+ #
51
+ def post_data
52
+ @request["postData"]
53
+ end
54
+ alias body post_data
55
+ end
56
+ end
57
+ end
@@ -2,43 +2,110 @@
2
2
 
3
3
  module Ferrum
4
4
  class Network
5
+ #
6
+ # Represents a [Network.Response](https://chromedevtools.github.io/devtools-protocol/1-3/Network/#type-Response)
7
+ # object.
8
+ #
5
9
  class Response
6
- attr_reader :body_size, :params
10
+ # The response body size.
11
+ #
12
+ # @return [Integer, nil]
13
+ attr_reader :body_size
7
14
 
15
+ # The parsed JSON attributes for the [Network.Response](https://chromedevtools.github.io/devtools-protocol/1-3/Network/#type-Response)
16
+ # object.
17
+ #
18
+ # @return [Hash{String => Object}]
19
+ attr_reader :params
20
+
21
+ # The response is fully loaded by the browser.
22
+ #
23
+ # @return [Boolean]
24
+ attr_writer :loaded
25
+
26
+ #
27
+ # Initializes the responses object.
28
+ #
29
+ # @param [Page] page
30
+ # The page associated with the network response.
31
+ #
32
+ # @param [Hash{String => Object}] params
33
+ # The parsed JSON attributes for the [Network.Response](https://chromedevtools.github.io/devtools-protocol/1-3/Network/#type-Response)
34
+ #
8
35
  def initialize(page, params)
9
36
  @page = page
10
37
  @params = params
11
38
  @response = params["response"] || params["redirectResponse"]
12
39
  end
13
40
 
41
+ #
42
+ # The request ID associated with the response.
43
+ #
44
+ # @return [String]
45
+ #
14
46
  def id
15
47
  @params["requestId"]
16
48
  end
17
49
 
50
+ #
51
+ # The URL of the response.
52
+ #
53
+ # @return [String]
54
+ #
18
55
  def url
19
56
  @response["url"]
20
57
  end
21
58
 
59
+ #
60
+ # The HTTP status of the response.
61
+ #
62
+ # @return [Integer]
63
+ #
22
64
  def status
23
65
  @response["status"]
24
66
  end
25
67
 
68
+ #
69
+ # The HTTP status text.
70
+ #
71
+ # @return [String]
72
+ #
26
73
  def status_text
27
74
  @response["statusText"]
28
75
  end
29
76
 
77
+ #
78
+ # The headers of the response.
79
+ #
80
+ # @return [Hash{String => String}]
81
+ #
30
82
  def headers
31
83
  @response["headers"]
32
84
  end
33
85
 
86
+ #
87
+ # The total size in bytes of the response.
88
+ #
89
+ # @return [Integer]
90
+ #
34
91
  def headers_size
35
92
  @response["encodedDataLength"]
36
93
  end
37
94
 
95
+ #
96
+ # The resource type of the response.
97
+ #
98
+ # @return [String]
99
+ #
38
100
  def type
39
101
  @params["type"]
40
102
  end
41
103
 
104
+ #
105
+ # The `Content-Type` header value of the response.
106
+ #
107
+ # @return [String, nil]
108
+ #
42
109
  def content_type
43
110
  @content_type ||= headers.find { |k, _| k.downcase == "content-type" }&.last&.sub(/;.*\z/, "")
44
111
  end
@@ -52,26 +119,61 @@ module Ferrum
52
119
  @body_size = size - headers_size
53
120
  end
54
121
 
122
+ #
123
+ # The response body.
124
+ #
125
+ # @return [String]
126
+ #
55
127
  def body
56
128
  @body ||= begin
57
- body, encoded = @page
58
- .command("Network.getResponseBody", requestId: id)
59
- .values_at("body", "base64Encoded")
129
+ body, encoded = @page.command("Network.getResponseBody", requestId: id)
130
+ .values_at("body", "base64Encoded")
60
131
  encoded ? Base64.decode64(body) : body
61
132
  end
62
133
  end
63
134
 
135
+ #
136
+ # @return [Boolean]
137
+ #
64
138
  def main?
65
139
  @page.network.response == self
66
140
  end
67
141
 
142
+ # The response is fully loaded by the browser or not.
143
+ #
144
+ # @return [Boolean]
145
+ def loaded?
146
+ @loaded
147
+ end
148
+
149
+ # Whether the response is a redirect.
150
+ #
151
+ # @return [Boolean]
152
+ def redirect?
153
+ params.key?("redirectResponse")
154
+ end
155
+
156
+ #
157
+ # Compares the response's ID to another response's ID.
158
+ #
159
+ # @return [Boolean]
160
+ # Indicates whether the response has the same ID as the other response
161
+ # object.
162
+ #
68
163
  def ==(other)
69
164
  id == other.id
70
165
  end
71
166
 
167
+ #
168
+ # Inspects the response object.
169
+ #
170
+ # @return [String]
171
+ #
72
172
  def inspect
73
173
  %(#<#{self.class} @params=#{@params.inspect} @response=#{@response.inspect}>)
74
174
  end
175
+
176
+ alias to_h params
75
177
  end
76
178
  end
77
179
  end
@@ -11,14 +11,25 @@ module Ferrum
11
11
  class Network
12
12
  CLEAR_TYPE = %i[traffic cache].freeze
13
13
  AUTHORIZE_TYPE = %i[server proxy].freeze
14
- RESOURCE_TYPES = %w[Document Stylesheet Image Media Font Script TextTrack
15
- XHR Fetch EventSource WebSocket Manifest
16
- SignedExchange Ping CSPViolationReport Other].freeze
14
+ REQUEST_STAGES = %i[Request Response].freeze
15
+ RESOURCE_TYPES = %i[Document Stylesheet Image Media Font Script TextTrack
16
+ XHR Fetch Prefetch EventSource WebSocket Manifest
17
+ SignedExchange Ping CSPViolationReport Preflight Other].freeze
17
18
  AUTHORIZE_BLOCK_MISSING = "Block is missing, call `authorize(...) { |r| r.continue } " \
18
19
  "or subscribe to `on(:request)` events before calling it"
19
20
  AUTHORIZE_TYPE_WRONG = ":type should be in #{AUTHORIZE_TYPE}"
20
21
  ALLOWED_CONNECTION_TYPE = %w[none cellular2g cellular3g cellular4g bluetooth ethernet wifi wimax other].freeze
21
22
 
23
+ # Network traffic.
24
+ #
25
+ # @return [Array<Exchange>]
26
+ # Returns all information about network traffic as {Exchange}
27
+ # instance which in general is a wrapper around `request`, `response` and
28
+ # `error`.
29
+ #
30
+ # @example
31
+ # browser.go_to("https://github.com/")
32
+ # browser.network.traffic # => [#<Ferrum::Network::Exchange, ...]
22
33
  attr_reader :traffic
23
34
 
24
35
  def initialize(page)
@@ -29,6 +40,25 @@ module Ferrum
29
40
  @whitelist = nil
30
41
  end
31
42
 
43
+ #
44
+ # Waits for network idle or raises {Ferrum::TimeoutError} error.
45
+ #
46
+ # @param [Integer] connections
47
+ # how many connections are allowed for network to be idling,
48
+ #
49
+ # @param [Float] duration
50
+ # Sleep for given amount of time and check again.
51
+ #
52
+ # @param [Float] timeout
53
+ # During what time we try to check idle.
54
+ #
55
+ # @raise [Ferrum::TimeoutError]
56
+ #
57
+ # @example
58
+ # browser.go_to("https://example.com/")
59
+ # browser.at_xpath("//a[text() = 'No UI changes button']").click
60
+ # browser.network.wait_for_idle
61
+ #
32
62
  def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.browser.timeout)
33
63
  start = Utils::ElapsedTime.monotonic_time
34
64
 
@@ -55,18 +85,61 @@ module Ferrum
55
85
  total_connections - finished_connections
56
86
  end
57
87
 
88
+ #
89
+ # Page request of the main frame.
90
+ #
91
+ # @return [Request]
92
+ #
93
+ # @example
94
+ # browser.go_to("https://github.com/")
95
+ # browser.network.request # => #<Ferrum::Network::Request...
96
+ #
58
97
  def request
59
98
  @exchange&.request
60
99
  end
61
100
 
101
+ #
102
+ # Page response of the main frame.
103
+ #
104
+ # @return [Response, nil]
105
+ #
106
+ # @example
107
+ # browser.go_to("https://github.com/")
108
+ # browser.network.response # => #<Ferrum::Network::Response...
109
+ #
62
110
  def response
63
111
  @exchange&.response
64
112
  end
65
113
 
114
+ #
115
+ # Contains the status code of the main page response (e.g., 200 for a
116
+ # success). This is just a shortcut for `response.status`.
117
+ #
118
+ # @return [Integer, nil]
119
+ #
120
+ # @example
121
+ # browser.go_to("https://github.com/")
122
+ # browser.network.status # => 200
123
+ #
66
124
  def status
67
125
  response&.status
68
126
  end
69
127
 
128
+ #
129
+ # Clear browser's cache or collected traffic.
130
+ #
131
+ # @param [:traffic, :cache] type
132
+ # The type of traffic to clear.
133
+ #
134
+ # @return [true]
135
+ #
136
+ # @example
137
+ # traffic = browser.network.traffic # => []
138
+ # browser.go_to("https://github.com/")
139
+ # traffic.size # => 51
140
+ # browser.network.clear(:traffic)
141
+ # traffic.size # => 0
142
+ #
70
143
  def clear(type)
71
144
  raise ArgumentError, ":type should be in #{CLEAR_TYPE}" unless CLEAR_TYPE.include?(type)
72
145
 
@@ -91,13 +164,71 @@ module Ferrum
91
164
  end
92
165
  alias allowlist= whitelist=
93
166
 
94
- def intercept(pattern: "*", resource_type: nil)
167
+ #
168
+ # Set request interception for given options. This method is only sets
169
+ # request interception, you should use `on` callback to catch requests and
170
+ # abort or continue them.
171
+ #
172
+ # @param [String] pattern
173
+ #
174
+ # @param [Symbol, nil] resource_type
175
+ # One of the [resource types](https://chromedevtools.github.io/devtools-protocol/tot/Network#type-ResourceType)
176
+ #
177
+ # @example
178
+ # browser = Ferrum::Browser.new
179
+ # browser.network.intercept
180
+ # browser.on(:request) do |request|
181
+ # if request.match?(/bla-bla/)
182
+ # request.abort
183
+ # elsif request.match?(/lorem/)
184
+ # request.respond(body: "Lorem ipsum")
185
+ # else
186
+ # request.continue
187
+ # end
188
+ # end
189
+ # browser.go_to("https://google.com")
190
+ #
191
+ def intercept(pattern: "*", resource_type: nil, request_stage: nil, handle_auth_requests: true)
95
192
  pattern = { urlPattern: pattern }
96
- pattern[:resourceType] = resource_type if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
97
193
 
98
- @page.command("Fetch.enable", handleAuthRequests: true, patterns: [pattern])
194
+ if resource_type && RESOURCE_TYPES.none?(resource_type.to_sym)
195
+ raise ArgumentError, "Unknown resource type '#{resource_type}' must be #{RESOURCE_TYPES.join(' | ')}"
196
+ end
197
+
198
+ if request_stage && REQUEST_STAGES.none?(request_stage.to_sym)
199
+ raise ArgumentError, "Unknown request stage '#{request_stage}' must be #{REQUEST_STAGES.join(' | ')}"
200
+ end
201
+
202
+ pattern[:resourceType] = resource_type if resource_type
203
+ pattern[:requestStage] = request_stage if request_stage
204
+ @page.command("Fetch.enable", patterns: [pattern], handleAuthRequests: handle_auth_requests)
99
205
  end
100
206
 
207
+ #
208
+ # Sets HTTP Basic-Auth credentials.
209
+ #
210
+ # @param [String] user
211
+ # The username to send.
212
+ #
213
+ # @param [String] password
214
+ # The password to send.
215
+ #
216
+ # @param [:server, :proxy] type
217
+ # Specifies whether the credentials are for a website or a proxy.
218
+ #
219
+ # @yield [request]
220
+ # The given block will be passed each authenticated request and can allow
221
+ # or deny the request.
222
+ #
223
+ # @yieldparam [Request] request
224
+ # An HTTP request.
225
+ #
226
+ # @example
227
+ # browser.network.authorize(user: "login", password: "pass") { |req| req.continue }
228
+ # browser.go_to("http://example.com/authenticated")
229
+ # puts browser.network.status # => 200
230
+ # puts browser.body # => Welcome, authenticated client
231
+ #
101
232
  def authorize(user:, password:, type: :server, &block)
102
233
  raise ArgumentError, AUTHORIZE_TYPE_WRONG unless AUTHORIZE_TYPE.include?(type)
103
234
  raise ArgumentError, AUTHORIZE_BLOCK_MISSING if !block_given? && !@page.subscribed?("Fetch.requestPaused")
@@ -151,6 +282,37 @@ module Ferrum
151
282
  Network::Exchange.new(@page, id).tap { |e| @traffic << e }
152
283
  end
153
284
 
285
+ #
286
+ # Activates emulation of network conditions.
287
+ #
288
+ # @param [Boolean] offline
289
+ # Emulate internet disconnection,
290
+ #
291
+ # @param [Integer] latency
292
+ # Minimum latency from request sent to response headers received (ms).
293
+ #
294
+ # @param [Integer] download_throughput
295
+ # Maximal aggregated download throughput (bytes/sec).
296
+ #
297
+ # @param [Integer] upload_throughput
298
+ # Maximal aggregated upload throughput (bytes/sec).
299
+ #
300
+ # @param [String, nil] connection_type
301
+ # Connection type if known:
302
+ # * `"none"`
303
+ # * `"cellular2g"`
304
+ # * `"cellular3g"`
305
+ # * `"cellular4g"`
306
+ # * `"bluetooth"`
307
+ # * `"ethernet"`
308
+ # * `"wifi"`
309
+ # * `"wimax"`
310
+ # * `"other"`
311
+ #
312
+ # @example
313
+ # browser.network.emulate_network_conditions(connection_type: "cellular2g")
314
+ # browser.go_to("https://github.com/")
315
+ #
154
316
  def emulate_network_conditions(offline: false, latency: 0,
155
317
  download_throughput: -1, upload_throughput: -1,
156
318
  connection_type: nil)
@@ -166,10 +328,28 @@ module Ferrum
166
328
  true
167
329
  end
168
330
 
331
+ #
332
+ # Activates offline mode for a page.
333
+ #
334
+ # @example
335
+ # browser.network.offline_mode
336
+ # browser.go_to("https://github.com/")
337
+ # # => Request to https://github.com/ failed (net::ERR_INTERNET_DISCONNECTED) (Ferrum::StatusError)
338
+ #
169
339
  def offline_mode
170
340
  emulate_network_conditions(offline: true, latency: 0, download_throughput: 0, upload_throughput: 0)
171
341
  end
172
342
 
343
+ #
344
+ # Toggles ignoring cache for each request. If true, cache will not be used.
345
+ #
346
+ # @example
347
+ # browser.network.cache(disable: true)
348
+ #
349
+ def cache(disable:)
350
+ @page.command("Network.setCacheDisabled", cacheDisabled: disable)
351
+ end
352
+
173
353
  private
174
354
 
175
355
  def subscribe_request_will_be_sent
@@ -192,6 +372,7 @@ module Ferrum
192
372
  if params["redirectResponse"]
193
373
  previous_exchange = select(request.id)[-2]
194
374
  response = Network::Response.new(@page, params)
375
+ response.loaded = true
195
376
  previous_exchange.response = response
196
377
  end
197
378
 
@@ -214,8 +395,12 @@ module Ferrum
214
395
 
215
396
  def subscribe_loading_finished
216
397
  @page.on("Network.loadingFinished") do |params|
217
- exchange = select(params["requestId"]).last
218
- exchange.response.body_size = params["encodedDataLength"] if exchange&.response
398
+ response = select(params["requestId"]).last&.response
399
+
400
+ if response
401
+ response.loaded = true
402
+ response.body_size = params["encodedDataLength"]
403
+ end
219
404
  end
220
405
  end
221
406
 
data/lib/ferrum/node.rb CHANGED
@@ -88,6 +88,26 @@ module Ferrum
88
88
  raise NotImplementedError
89
89
  end
90
90
 
91
+ def scroll_into_view
92
+ tap { page.command("DOM.scrollIntoViewIfNeeded", nodeId: node_id) }
93
+ end
94
+
95
+ def in_viewport?(of: nil)
96
+ function = <<~JS
97
+ function(element, scope) {
98
+ const rect = element.getBoundingClientRect();
99
+ const [height, width] = scope
100
+ ? [scope.offsetHeight, scope.offsetWidth]
101
+ : [window.innerHeight, window.innerWidth];
102
+ return rect.top >= 0 &&
103
+ rect.left >= 0 &&
104
+ rect.bottom <= height &&
105
+ rect.right <= width;
106
+ }
107
+ JS
108
+ page.evaluate_func(function, self, of)
109
+ end
110
+
91
111
  def select_file(value)
92
112
  page.command("DOM.setFileInputFiles", slowmoable: true, nodeId: node_id, files: Array(value))
93
113
  end
@@ -208,7 +228,7 @@ module Ferrum
208
228
 
209
229
  def content_quads
210
230
  quads = page.command("DOM.getContentQuads", nodeId: node_id)["quads"]
211
- raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.size.zero?
231
+ raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.empty?
212
232
 
213
233
  quads
214
234
  end
@@ -3,10 +3,26 @@
3
3
  module Ferrum
4
4
  class Page
5
5
  module Animation
6
+ #
7
+ # Returns playback rate for CSS animations, defaults to `1`.
8
+ #
9
+ # @return [Integer]
10
+ #
6
11
  def playback_rate
7
12
  command("Animation.getPlaybackRate")["playbackRate"]
8
13
  end
9
14
 
15
+ #
16
+ # Sets playback rate of CSS animations.
17
+ #
18
+ # @param [Integer] value
19
+ #
20
+ # @example
21
+ # browser = Ferrum::Browser.new
22
+ # browser.playback_rate = 2000
23
+ # browser.go_to("https://google.com")
24
+ # browser.playback_rate # => 2000
25
+ #
10
26
  def playback_rate=(value)
11
27
  command("Animation.setPlaybackRate", playbackRate: value)
12
28
  end