ferrum 0.11 → 0.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/README.md +178 -29
  4. data/lib/ferrum/browser/binary.rb +46 -0
  5. data/lib/ferrum/browser/client.rb +13 -12
  6. data/lib/ferrum/browser/command.rb +7 -8
  7. data/lib/ferrum/browser/options/base.rb +1 -7
  8. data/lib/ferrum/browser/options/chrome.rb +17 -11
  9. data/lib/ferrum/browser/options/firefox.rb +11 -4
  10. data/lib/ferrum/browser/process.rb +41 -35
  11. data/lib/ferrum/browser/subscriber.rb +1 -3
  12. data/lib/ferrum/browser/web_socket.rb +9 -12
  13. data/lib/ferrum/browser/xvfb.rb +4 -8
  14. data/lib/ferrum/browser.rb +44 -12
  15. data/lib/ferrum/context.rb +6 -2
  16. data/lib/ferrum/contexts.rb +10 -8
  17. data/lib/ferrum/cookies.rb +10 -9
  18. data/lib/ferrum/errors.rb +115 -0
  19. data/lib/ferrum/frame/runtime.rb +20 -17
  20. data/lib/ferrum/frame.rb +32 -24
  21. data/lib/ferrum/headers.rb +2 -2
  22. data/lib/ferrum/keyboard.rb +11 -11
  23. data/lib/ferrum/mouse.rb +8 -7
  24. data/lib/ferrum/network/auth_request.rb +7 -2
  25. data/lib/ferrum/network/exchange.rb +14 -10
  26. data/lib/ferrum/network/intercepted_request.rb +10 -8
  27. data/lib/ferrum/network/request.rb +5 -0
  28. data/lib/ferrum/network/response.rb +4 -4
  29. data/lib/ferrum/network.rb +124 -35
  30. data/lib/ferrum/node.rb +69 -23
  31. data/lib/ferrum/page/animation.rb +0 -1
  32. data/lib/ferrum/page/frames.rb +46 -20
  33. data/lib/ferrum/page/screenshot.rb +51 -65
  34. data/lib/ferrum/page/stream.rb +38 -0
  35. data/lib/ferrum/page/tracing.rb +71 -0
  36. data/lib/ferrum/page.rb +81 -36
  37. data/lib/ferrum/proxy.rb +58 -0
  38. data/lib/ferrum/{rbga.rb → rgba.rb} +4 -2
  39. data/lib/ferrum/target.rb +1 -0
  40. data/lib/ferrum/utils/attempt.rb +20 -0
  41. data/lib/ferrum/utils/elapsed_time.rb +27 -0
  42. data/lib/ferrum/utils/platform.rb +28 -0
  43. data/lib/ferrum/version.rb +1 -1
  44. data/lib/ferrum.rb +4 -146
  45. metadata +60 -51
@@ -14,6 +14,10 @@ module Ferrum
14
14
  RESOURCE_TYPES = %w[Document Stylesheet Image Media Font Script TextTrack
15
15
  XHR Fetch EventSource WebSocket Manifest
16
16
  SignedExchange Ping CSPViolationReport Other].freeze
17
+ AUTHORIZE_BLOCK_MISSING = "Block is missing, call `authorize(...) { |r| r.continue } " \
18
+ "or subscribe to `on(:request)` events before calling it"
19
+ AUTHORIZE_TYPE_WRONG = ":type should be in #{AUTHORIZE_TYPE}"
20
+ ALLOWED_CONNECTION_TYPE = %w[none cellular2g cellular3g cellular4g bluetooth ethernet wifi wimax other].freeze
17
21
 
18
22
  attr_reader :traffic
19
23
 
@@ -21,13 +25,16 @@ module Ferrum
21
25
  @page = page
22
26
  @traffic = []
23
27
  @exchange = nil
28
+ @blacklist = nil
29
+ @whitelist = nil
24
30
  end
25
31
 
26
32
  def wait_for_idle(connections: 0, duration: 0.05, timeout: @page.browser.timeout)
27
- start = Ferrum.monotonic_time
33
+ start = Utils::ElapsedTime.monotonic_time
28
34
 
29
35
  until idle?(connections)
30
- raise TimeoutError if Ferrum.timeout?(start, timeout)
36
+ raise TimeoutError if Utils::ElapsedTime.timeout?(start, timeout)
37
+
31
38
  sleep(duration)
32
39
  end
33
40
  end
@@ -61,9 +68,7 @@ module Ferrum
61
68
  end
62
69
 
63
70
  def clear(type)
64
- unless CLEAR_TYPE.include?(type)
65
- raise ArgumentError, ":type should be in #{CLEAR_TYPE}"
66
- end
71
+ raise ArgumentError, ":type should be in #{CLEAR_TYPE}" unless CLEAR_TYPE.include?(type)
67
72
 
68
73
  if type == :traffic
69
74
  @traffic.clear
@@ -74,23 +79,28 @@ module Ferrum
74
79
  true
75
80
  end
76
81
 
82
+ def blacklist=(patterns)
83
+ @blacklist = Array(patterns)
84
+ blacklist_subscribe
85
+ end
86
+ alias blocklist= blacklist=
87
+
88
+ def whitelist=(patterns)
89
+ @whitelist = Array(patterns)
90
+ whitelist_subscribe
91
+ end
92
+ alias allowlist= whitelist=
93
+
77
94
  def intercept(pattern: "*", resource_type: nil)
78
95
  pattern = { urlPattern: pattern }
79
- if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
80
- pattern[:resourceType] = resource_type
81
- end
96
+ pattern[:resourceType] = resource_type if resource_type && RESOURCE_TYPES.include?(resource_type.to_s)
82
97
 
83
98
  @page.command("Fetch.enable", handleAuthRequests: true, patterns: [pattern])
84
99
  end
85
100
 
86
101
  def authorize(user:, password:, type: :server, &block)
87
- unless AUTHORIZE_TYPE.include?(type)
88
- raise ArgumentError, ":type should be in #{AUTHORIZE_TYPE}"
89
- end
90
-
91
- if !block_given? && !@page.subscribed?("Fetch.requestPaused")
92
- raise ArgumentError, "Block is missing, call `authorize(...) { |r| r.continue } or subscribe to `on(:request)` events before calling it"
93
- end
102
+ raise ArgumentError, AUTHORIZE_TYPE_WRONG unless AUTHORIZE_TYPE.include?(type)
103
+ raise ArgumentError, AUTHORIZE_BLOCK_MISSING if !block_given? && !@page.subscribed?("Fetch.requestPaused")
94
104
 
95
105
  @authorized_ids ||= {}
96
106
  @authorized_ids[type] ||= []
@@ -116,6 +126,53 @@ module Ferrum
116
126
  end
117
127
 
118
128
  def subscribe
129
+ subscribe_request_will_be_sent
130
+ subscribe_response_received
131
+ subscribe_loading_finished
132
+ subscribe_loading_failed
133
+ subscribe_log_entry_added
134
+ end
135
+
136
+ def authorized_response(ids, request_id, username, password)
137
+ if ids.include?(request_id)
138
+ { response: "CancelAuth" }
139
+ elsif username && password
140
+ { response: "ProvideCredentials",
141
+ username: username,
142
+ password: password }
143
+ end
144
+ end
145
+
146
+ def select(request_id)
147
+ @traffic.select { |e| e.id == request_id }
148
+ end
149
+
150
+ def build_exchange(id)
151
+ Network::Exchange.new(@page, id).tap { |e| @traffic << e }
152
+ end
153
+
154
+ def emulate_network_conditions(offline: false, latency: 0,
155
+ download_throughput: -1, upload_throughput: -1,
156
+ connection_type: nil)
157
+ params = {
158
+ offline: offline, latency: latency,
159
+ downloadThroughput: download_throughput,
160
+ uploadThroughput: upload_throughput
161
+ }
162
+
163
+ params[:connectionType] = connection_type if connection_type && ALLOWED_CONNECTION_TYPE.include?(connection_type)
164
+
165
+ @page.command("Network.emulateNetworkConditions", **params)
166
+ true
167
+ end
168
+
169
+ def offline_mode
170
+ emulate_network_conditions(offline: true, latency: 0, download_throughput: 0, upload_throughput: 0)
171
+ end
172
+
173
+ private
174
+
175
+ def subscribe_request_will_be_sent
119
176
  @page.on("Network.requestWillBeSent") do |params|
120
177
  request = Network::Request.new(params)
121
178
 
@@ -140,25 +197,29 @@ module Ferrum
140
197
 
141
198
  exchange.request = request
142
199
 
143
- if exchange.navigation_request?(@page.main_frame.id)
144
- @exchange = exchange
145
- end
200
+ @exchange = exchange if exchange.navigation_request?(@page.main_frame.id)
146
201
  end
202
+ end
147
203
 
204
+ def subscribe_response_received
148
205
  @page.on("Network.responseReceived") do |params|
149
- if exchange = select(params["requestId"]).last
206
+ exchange = select(params["requestId"]).last
207
+
208
+ if exchange
150
209
  response = Network::Response.new(@page, params)
151
210
  exchange.response = response
152
211
  end
153
212
  end
213
+ end
154
214
 
215
+ def subscribe_loading_finished
155
216
  @page.on("Network.loadingFinished") do |params|
156
217
  exchange = select(params["requestId"]).last
157
- if exchange && exchange.response
158
- exchange.response.body_size = params["encodedDataLength"]
159
- end
218
+ exchange.response.body_size = params["encodedDataLength"] if exchange&.response
160
219
  end
220
+ end
161
221
 
222
+ def subscribe_loading_failed
162
223
  @page.on("Network.loadingFailed") do |params|
163
224
  exchange = select(params["requestId"]).last
164
225
  exchange.error ||= Network::Error.new
@@ -169,7 +230,9 @@ module Ferrum
169
230
  exchange.error.monotonic_time = params["timestamp"]
170
231
  exchange.error.canceled = params["canceled"]
171
232
  end
233
+ end
172
234
 
235
+ def subscribe_log_entry_added
173
236
  @page.on("Log.entryAdded") do |params|
174
237
  entry = params["entry"] || {}
175
238
  if entry["source"] == "network" && entry["level"] == "error"
@@ -184,24 +247,50 @@ module Ferrum
184
247
  end
185
248
  end
186
249
 
187
- def authorized_response(ids, request_id, username, password)
188
- if ids.include?(request_id)
189
- { response: "CancelAuth" }
190
- elsif username && password
191
- { response: "ProvideCredentials",
192
- username: username,
193
- password: password }
194
- else
195
- { response: "CancelAuth" }
250
+ def blacklist_subscribe
251
+ return unless blacklist?
252
+ raise ArgumentError, "You can't use blacklist along with whitelist" if whitelist?
253
+
254
+ @blacklist_subscribe ||= begin
255
+ intercept
256
+
257
+ @page.on(:request) do |request|
258
+ if @blacklist.any? { |p| request.match?(p) }
259
+ request.abort
260
+ else
261
+ request.continue
262
+ end
263
+ end
264
+
265
+ true
196
266
  end
197
267
  end
198
268
 
199
- def select(request_id)
200
- @traffic.select { |e| e.id == request_id }
269
+ def whitelist_subscribe
270
+ return unless whitelist?
271
+ raise ArgumentError, "You can't use whitelist along with blacklist" if blacklist?
272
+
273
+ @whitelist_subscribe ||= begin
274
+ intercept
275
+
276
+ @page.on(:request) do |request|
277
+ if @whitelist.any? { |p| request.match?(p) }
278
+ request.continue
279
+ else
280
+ request.abort
281
+ end
282
+ end
283
+
284
+ true
285
+ end
201
286
  end
202
287
 
203
- def build_exchange(id)
204
- Network::Exchange.new(@page, id).tap { |e| @traffic << e }
288
+ def blacklist?
289
+ Array(@blacklist).any?
290
+ end
291
+
292
+ def whitelist?
293
+ Array(@whitelist).any?
205
294
  end
206
295
  end
207
296
  end
data/lib/ferrum/node.rb CHANGED
@@ -10,7 +10,8 @@ module Ferrum
10
10
  def initialize(frame, target_id, node_id, description)
11
11
  @page = frame.page
12
12
  @target_id = target_id
13
- @node_id, @description = node_id, description
13
+ @node_id = node_id
14
+ @description = description
14
15
  @tag_name = description["nodeName"].downcase
15
16
  end
16
17
 
@@ -38,15 +39,16 @@ module Ferrum
38
39
  end
39
40
 
40
41
  def wait_for_stop_moving(delay: MOVING_WAIT_DELAY, attempts: MOVING_WAIT_ATTEMPTS)
41
- Ferrum.with_attempts(errors: NodeMovingError, max: attempts, wait: 0) do
42
- previous, current = get_content_quads_with(delay: delay)
42
+ Utils::Attempt.with_retry(errors: NodeMovingError, max: attempts, wait: 0) do
43
+ previous, current = content_quads_with(delay: delay)
43
44
  raise NodeMovingError.new(self, previous, current) if previous != current
45
+
44
46
  current
45
47
  end
46
48
  end
47
49
 
48
50
  def moving?(delay: MOVING_WAIT_DELAY)
49
- previous, current = get_content_quads_with(delay: delay)
51
+ previous, current = content_quads_with(delay: delay)
50
52
  previous == current
51
53
  end
52
54
 
@@ -122,17 +124,52 @@ module Ferrum
122
124
  def property(name)
123
125
  evaluate("this['#{name}']")
124
126
  end
127
+ alias [] property
125
128
 
126
129
  def attribute(name)
127
130
  evaluate("this.getAttribute('#{name}')")
128
131
  end
129
132
 
133
+ def selected
134
+ function = <<~JS
135
+ function(element) {
136
+ if (element.nodeName.toLowerCase() !== 'select') {
137
+ throw new Error('Element is not a <select> element.');
138
+ }
139
+ return Array.from(element).filter(option => option.selected);
140
+ }
141
+ JS
142
+ page.evaluate_func(function, self, on: self)
143
+ end
144
+
145
+ def select(*values, by: :value)
146
+ tap do
147
+ function = <<~JS
148
+ function(element, values, by) {
149
+ if (element.nodeName.toLowerCase() !== 'select') {
150
+ throw new Error('Element is not a <select> element.');
151
+ }
152
+ const options = Array.from(element.options);
153
+ element.value = undefined;
154
+ for (const option of options) {
155
+ option.selected = values.some((value) => option[by] === value);
156
+ if (option.selected && !element.multiple) break;
157
+ }
158
+ element.dispatchEvent(new Event('input', { bubbles: true }));
159
+ element.dispatchEvent(new Event('change', { bubbles: true }));
160
+ }
161
+ JS
162
+ page.evaluate_func(function, self, values.flatten, by, on: self)
163
+ end
164
+ end
165
+
130
166
  def evaluate(expression)
131
167
  page.evaluate_on(node: self, expression: expression)
132
168
  end
133
169
 
134
170
  def ==(other)
135
171
  return false unless other.is_a?(Node)
172
+
136
173
  # We compare backendNodeId because once nodeId is sent to frontend backend
137
174
  # never returns same nodeId sending 0. In other words frontend is
138
175
  # responsible for keeping track of node ids.
@@ -147,30 +184,39 @@ module Ferrum
147
184
  points = wait_for_stop_moving.map { |q| to_points(q) }.first
148
185
  get_position(points, x, y, position)
149
186
  rescue CoordinatesNotFoundError
150
- x, y = get_bounding_rect_coordinates
151
- raise if x == 0 && y == 0
187
+ x, y = bounding_rect_coordinates
188
+ raise if x.zero? && y.zero?
189
+
152
190
  [x, y]
153
191
  end
154
192
 
193
+ # Returns a hash of the computed styles for the node
194
+ def computed_style
195
+ page
196
+ .command("CSS.getComputedStyleForNode", nodeId: node_id)["computedStyle"]
197
+ .each_with_object({}) { |style, memo| memo.merge!(style["name"] => style["value"]) }
198
+ end
199
+
155
200
  private
156
201
 
157
- def get_bounding_rect_coordinates
202
+ def bounding_rect_coordinates
158
203
  evaluate <<~JS
159
204
  [this.getBoundingClientRect().left + window.pageXOffset + (this.offsetWidth / 2),
160
205
  this.getBoundingClientRect().top + window.pageYOffset + (this.offsetHeight / 2)]
161
206
  JS
162
207
  end
163
208
 
164
- def get_content_quads
209
+ def content_quads
165
210
  quads = page.command("DOM.getContentQuads", nodeId: node_id)["quads"]
166
- raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.size == 0
211
+ raise CoordinatesNotFoundError, "Node is either not visible or not an HTMLElement" if quads.size.zero?
212
+
167
213
  quads
168
214
  end
169
215
 
170
- def get_content_quads_with(delay: MOVING_WAIT_DELAY)
171
- previous = get_content_quads
216
+ def content_quads_with(delay: MOVING_WAIT_DELAY)
217
+ previous = content_quads
172
218
  sleep(delay)
173
- current = get_content_quads
219
+ current = content_quads
174
220
  [previous, current]
175
221
  end
176
222
 
@@ -182,28 +228,28 @@ module Ferrum
182
228
  x = point[:x] + offset_x.to_i
183
229
  y = point[:y] + offset_y.to_i
184
230
  else
185
- x, y = points.inject([0, 0]) do |memo, point|
186
- [memo[0] + point[:x],
187
- memo[1] + point[:y]]
231
+ x, y = points.inject([0, 0]) do |memo, coordinate|
232
+ [memo[0] + coordinate[:x],
233
+ memo[1] + coordinate[:y]]
188
234
  end
189
235
 
190
- x = x / 4
191
- y = y / 4
236
+ x /= 4
237
+ y /= 4
192
238
  end
193
239
 
194
240
  if offset_x && offset_y && position == :center
195
- x = x + offset_x.to_i
196
- y = y + offset_y.to_i
241
+ x += offset_x.to_i
242
+ y += offset_y.to_i
197
243
  end
198
244
 
199
245
  [x, y]
200
246
  end
201
247
 
202
248
  def to_points(quad)
203
- [{x: quad[0], y: quad[1]},
204
- {x: quad[2], y: quad[3]},
205
- {x: quad[4], y: quad[5]},
206
- {x: quad[6], y: quad[7]}]
249
+ [{ x: quad[0], y: quad[1] },
250
+ { x: quad[2], y: quad[3] },
251
+ { x: quad[4], y: quad[5] },
252
+ { x: quad[6], y: quad[7] }]
207
253
  end
208
254
  end
209
255
  end
@@ -7,7 +7,6 @@ module Ferrum
7
7
  command("Animation.getPlaybackRate")["playbackRate"]
8
8
  end
9
9
 
10
-
11
10
  def playback_rate=(value)
12
11
  command("Animation.setPlaybackRate", playbackRate: value)
13
12
  end
@@ -11,35 +11,60 @@ module Ferrum
11
11
  @frames.values
12
12
  end
13
13
 
14
- def frame_by(id: nil, name: nil)
14
+ def frame_by(id: nil, name: nil, execution_id: nil)
15
15
  if id
16
16
  @frames[id]
17
17
  elsif name
18
18
  frames.find { |f| f.name == name }
19
+ elsif execution_id
20
+ frames.find { |f| f.execution_id == execution_id }
19
21
  else
20
22
  raise ArgumentError
21
23
  end
22
24
  end
23
25
 
24
26
  def frames_subscribe
27
+ subscribe_frame_attached
28
+ subscribe_frame_started_loading
29
+ subscribe_frame_navigated
30
+ subscribe_frame_stopped_loading
31
+
32
+ subscribe_navigated_within_document
33
+
34
+ subscribe_request_will_be_sent
35
+
36
+ subscribe_execution_context_created
37
+ subscribe_execution_context_destroyed
38
+ subscribe_execution_contexts_cleared
39
+ end
40
+
41
+ private
42
+
43
+ def subscribe_frame_attached
25
44
  on("Page.frameAttached") do |params|
26
45
  parent_frame_id, frame_id = params.values_at("parentFrameId", "frameId")
27
46
  @frames[frame_id] = Frame.new(frame_id, self, parent_frame_id)
28
47
  end
48
+ end
29
49
 
50
+ def subscribe_frame_started_loading
30
51
  on("Page.frameStartedLoading") do |params|
31
52
  frame = @frames[params["frameId"]]
32
53
  frame.state = :started_loading
33
54
  @event.reset
34
55
  end
56
+ end
35
57
 
58
+ def subscribe_frame_navigated
36
59
  on("Page.frameNavigated") do |params|
37
60
  frame_id, name = params["frame"]&.values_at("id", "name")
38
61
  frame = @frames[frame_id]
39
62
  frame.state = :navigated
40
63
  frame.name = name unless name.to_s.empty?
41
64
  end
65
+ end
42
66
 
67
+ def subscribe_frame_stopped_loading
43
68
  on("Page.frameStoppedLoading") do |params|
44
69
  # `DOM.performSearch` doesn't work without getting #document node first.
45
70
  # It returns node with nodeId 1 and nodeType 9 from which descend the
@@ -47,7 +72,7 @@ module Ferrum
47
72
  # node will change the id and all subsequent nodes have to change id too.
48
73
  if @main_frame.id == params["frameId"]
49
74
  @event.set if idling?
50
- get_document_id
75
+ document_node_id
51
76
  end
52
77
 
53
78
  frame = @frames[params["frameId"]]
@@ -55,58 +80,59 @@ module Ferrum
55
80
 
56
81
  @event.set if idling?
57
82
  end
83
+ end
58
84
 
85
+ def subscribe_navigated_within_document
59
86
  on("Page.navigatedWithinDocument") do
60
87
  @event.set if idling?
61
88
  end
89
+ end
62
90
 
91
+ def subscribe_request_will_be_sent
63
92
  on("Network.requestWillBeSent") do |params|
64
- if params["frameId"] == @main_frame.id
65
- # Possible types:
66
- # Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR,
67
- # Fetch, EventSource, WebSocket, Manifest, SignedExchange, Ping,
68
- # CSPViolationReport, Other
69
- @event.reset if params["type"] == "Document"
70
- end
93
+ # Possible types:
94
+ # Document, Stylesheet, Image, Media, Font, Script, TextTrack, XHR,
95
+ # Fetch, EventSource, WebSocket, Manifest, SignedExchange, Ping,
96
+ # CSPViolationReport, Other
97
+ @event.reset if params["frameId"] == @main_frame.id && params["type"] == "Document"
71
98
  end
99
+ end
72
100
 
101
+ def subscribe_execution_context_created
73
102
  on("Runtime.executionContextCreated") do |params|
74
- setting_up_main_frame = false
75
103
  context_id = params.dig("context", "id")
76
104
  frame_id = params.dig("context", "auxData", "frameId")
77
105
 
78
106
  unless @main_frame.id
79
107
  root_frame = command("Page.getFrameTree").dig("frameTree", "frame", "id")
80
108
  if frame_id == root_frame
81
- setting_up_main_frame = true
82
109
  @main_frame.id = frame_id
83
110
  @frames[frame_id] = @main_frame
84
111
  end
85
112
  end
86
113
 
87
114
  frame = @frames[frame_id] || Frame.new(frame_id, self)
88
- frame.set_execution_id(context_id)
89
-
90
- # Set event because `execution_id` might raise NoExecutionContextError
91
- @event.set if setting_up_main_frame
115
+ frame.execution_id = context_id
92
116
 
93
117
  @frames[frame_id] ||= frame
94
118
  end
119
+ end
95
120
 
121
+ def subscribe_execution_context_destroyed
96
122
  on("Runtime.executionContextDestroyed") do |params|
97
123
  execution_id = params["executionContextId"]
98
- frame = frames.find { |f| f.execution_id?(execution_id) }
99
- frame.reset_execution_id
124
+ frame = frame_by(execution_id: execution_id)
125
+ frame&.execution_id = nil
100
126
  end
127
+ end
101
128
 
129
+ def subscribe_execution_contexts_cleared
102
130
  on("Runtime.executionContextsCleared") do
103
131
  @frames.delete_if { |_, f| !f.main? }
104
- @main_frame.reset_execution_id
132
+ @main_frame.execution_id = nil
105
133
  end
106
134
  end
107
135
 
108
- private
109
-
110
136
  def idling?
111
137
  @frames.all? { |_, f| f.state == :stopped_loading }
112
138
  end