ferrum 0.19.2 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +60 -0
  3. data/lib/ferrum/browser/binary.rb +1 -2
  4. data/lib/ferrum/browser/options.rb +33 -12
  5. data/lib/ferrum/browser/process/output.rb +143 -0
  6. data/lib/ferrum/browser/process.rb +42 -69
  7. data/lib/ferrum/browser.rb +9 -23
  8. data/lib/ferrum/client/web_socket.rb +4 -15
  9. data/lib/ferrum/client.rb +1 -1
  10. data/lib/ferrum/console/location.rb +42 -0
  11. data/lib/ferrum/console/message.rb +230 -0
  12. data/lib/ferrum/console.rb +157 -0
  13. data/lib/ferrum/context.rb +23 -15
  14. data/lib/ferrum/contexts.rb +41 -18
  15. data/lib/ferrum/errors.rb +7 -5
  16. data/lib/ferrum/frame/dom.rb +50 -44
  17. data/lib/ferrum/frame/runtime/legacy.rb +336 -0
  18. data/lib/ferrum/frame/runtime.rb +416 -186
  19. data/lib/ferrum/frame/script.rb +289 -0
  20. data/lib/ferrum/frame.rb +33 -4
  21. data/lib/ferrum/interceptable.rb +19 -9
  22. data/lib/ferrum/isolated_world.rb +94 -0
  23. data/lib/ferrum/keyboard.rb +95 -8
  24. data/lib/ferrum/log/annotated.rb +77 -0
  25. data/lib/ferrum/log/default.rb +55 -0
  26. data/lib/ferrum/log/json.rb +32 -0
  27. data/lib/ferrum/log/paint.rb +28 -0
  28. data/lib/ferrum/log.rb +247 -0
  29. data/lib/ferrum/mouse.rb +16 -8
  30. data/lib/ferrum/node.rb +127 -52
  31. data/lib/ferrum/page/frames.rb +11 -3
  32. data/lib/ferrum/page/screenshot.rb +8 -7
  33. data/lib/ferrum/page.rb +112 -61
  34. data/lib/ferrum/proxy.rb +8 -7
  35. data/lib/ferrum/remote_object.rb +75 -0
  36. data/lib/ferrum/target.rb +7 -0
  37. data/lib/ferrum/utils/deprecate.rb +77 -0
  38. data/lib/ferrum/version.rb +1 -1
  39. data/lib/ferrum/worker.rb +8 -0
  40. data/lib/ferrum.rb +3 -0
  41. metadata +16 -2
data/lib/ferrum/node.rb CHANGED
@@ -14,45 +14,86 @@ module Ferrum
14
14
  MOVING_WAIT_DELAY = ENV.fetch("FERRUM_NODE_MOVING_WAIT", 0.01).to_f
15
15
  MOVING_WAIT_ATTEMPTS = ENV.fetch("FERRUM_NODE_MOVING_ATTEMPTS", 50).to_i
16
16
 
17
- attr_reader :page, :target_id, :description, :tag_name
17
+ # @return [Page] the page the node belongs to.
18
+ attr_reader :page
18
19
 
19
- def initialize(frame, target_id, description, object_id: nil, node_id: nil)
20
+ # @return [String] the id of the page's target.
21
+ attr_reader :target_id
22
+
23
+ # @return [Integer] Chrome's id of the node, stable for the node's lifetime.
24
+ attr_reader :backend_node_id
25
+
26
+ # @return [Integer] the DOM `nodeType`, e.g. `1` for an element, `3` for a text node.
27
+ attr_reader :node_type
28
+
29
+ # @return [String, nil] the id of the frame an `iframe` or `frame` element contains, `nil` for other nodes.
30
+ attr_reader :frame_id
31
+
32
+ #
33
+ # @param [Frame, IsolatedWorld] frame
34
+ # The runtime the node was returned from.
35
+ #
36
+ # @param [Hash{String => Object}] properties
37
+ # The node's properties from Chrome's deep serialization.
38
+ #
39
+ def initialize(frame, properties)
40
+ @runtime = frame
20
41
  @page = frame.page
21
- @target_id = target_id
22
- @description = description
23
- @tag_name = description["nodeName"].downcase
24
- @object_id = object_id
25
- @node_id = node_id
42
+ @target_id = @page.target_id
43
+ @backend_node_id = properties["backendNodeId"]
44
+ @node_type = properties["nodeType"]
45
+ @tag_name = properties["localName"]&.downcase
46
+ @frame_id = properties["frameId"]
47
+ @document_version = @page.document_version
48
+ end
49
+
50
+ #
51
+ # Chrome's description of the node from `DOM.describeNode`, fetched on
52
+ # first use.
53
+ #
54
+ # @return [Hash{String => Object}]
55
+ #
56
+ def description
57
+ @description ||= page.command("DOM.describeNode", backendNodeId: backend_node_id)["node"]
58
+ end
59
+
60
+ #
61
+ # The lowercased tag name of an element, or the node name of any other
62
+ # node, e.g. `"#text"`.
63
+ #
64
+ # @return [String]
65
+ #
66
+ def tag_name
67
+ @tag_name ||= description["nodeName"].downcase
26
68
  end
27
69
 
28
70
  # Frontend node id is resolved lazily, on first actual need (focus, click, scroll_into_view, etc.)
29
71
  # We can try to subscribe to `DOM.childNodeRemoved` and `DOM.childNodeInserted` in the future
30
72
  # to keep track of nodes.
31
73
  def node_id
32
- @node_id ||= begin
33
- id = page.command("DOM.requestNode", objectId: @object_id)["nodeId"]
34
- raise NodeNotFoundError, "node is not trackable" if id.zero?
74
+ return @node_id if @node_id && @document_version == page.document_version
75
+
76
+ document_version = page.document_version
77
+ @node_id = begin
78
+ id = page.command("DOM.pushNodesByBackendIdsToFrontend",
79
+ backendNodeIds: [backend_node_id])["nodeIds"].first
80
+ raise NodeNotFoundError, { "message" => "node is not trackable" } if id.zero?
35
81
 
36
82
  id
37
- rescue NoExecutionContextError
38
- raise NodeNotFoundError, "node is not trackable"
83
+ rescue BrowserError => e
84
+ raise unless e.message == "Document needs to be requested first"
85
+
86
+ raise NodeNotFoundError, { "message" => "node is not trackable, the page is still loading a new document" }
39
87
  end
88
+ @document_version = document_version
89
+ @node_id
40
90
  end
41
91
 
42
92
  # Whether this is an element node, as opposed to e.g. a text node.
43
93
  #
44
94
  # @return [Boolean]
45
95
  def node?
46
- description["nodeType"] == 1 # nodeType: 3, nodeName: "#text" e.g.
47
- end
48
-
49
- #
50
- # The id of the frame this node belongs to.
51
- #
52
- # @return [String]
53
- #
54
- def frame_id
55
- description["frameId"]
96
+ node_type == 1 # nodeType: 3, nodeName: "#text" e.g.
56
97
  end
57
98
 
58
99
  #
@@ -202,9 +243,9 @@ module Ferrum
202
243
  #
203
244
  # @return [Boolean]
204
245
  def in_viewport?(of: nil)
205
- function = <<~JS
206
- function(element, scope) {
207
- const rect = element.getBoundingClientRect();
246
+ evaluate(<<~JS, of)
247
+ function(scope) {
248
+ const rect = this.getBoundingClientRect();
208
249
  const [height, width] = scope
209
250
  ? [scope.offsetHeight, scope.offsetWidth]
210
251
  : [window.innerHeight, window.innerWidth];
@@ -214,7 +255,6 @@ module Ferrum
214
255
  rect.right <= width;
215
256
  }
216
257
  JS
217
- page.evaluate_func(function, self, of)
218
258
  end
219
259
 
220
260
  #
@@ -232,7 +272,7 @@ module Ferrum
232
272
  page.command(
233
273
  "DOM.setFileInputFiles",
234
274
  slowmoable: true,
235
- backendNodeId: description["backendNodeId"],
275
+ backendNodeId: backend_node_id,
236
276
  files: Array(value)
237
277
  )
238
278
  end
@@ -335,7 +375,7 @@ module Ferrum
335
375
  # page.at_css("input").property("value") # => "Foo"
336
376
  #
337
377
  def property(name)
338
- evaluate("this['#{name}']")
378
+ evaluate("this[name]", name: name)
339
379
  end
340
380
  alias [] property
341
381
 
@@ -352,7 +392,7 @@ module Ferrum
352
392
  # page.at_css("input").attribute("value") # => "Foo"
353
393
  #
354
394
  def attribute(name)
355
- evaluate("this.getAttribute('#{name}')")
395
+ evaluate("this.getAttribute(name)", name: name)
356
396
  end
357
397
 
358
398
  #
@@ -361,15 +401,14 @@ module Ferrum
361
401
  # @return [Array<Node>]
362
402
  #
363
403
  def selected
364
- function = <<~JS
365
- function(element) {
366
- if (element.nodeName.toLowerCase() !== 'select') {
404
+ evaluate(<<~JS)
405
+ function() {
406
+ if (this.nodeName.toLowerCase() !== 'select') {
367
407
  throw new Error('Element is not a <select> element.');
368
408
  }
369
- return Array.from(element).filter(option => option.selected);
409
+ return Array.from(this).filter(option => option.selected);
370
410
  }
371
411
  JS
372
- page.evaluate_func(function, self, on: self)
373
412
  end
374
413
 
375
414
  #
@@ -397,38 +436,73 @@ module Ferrum
397
436
  #
398
437
  def select(*values, by: :value)
399
438
  tap do
400
- function = <<~JS
401
- function(element, values, by) {
402
- if (element.nodeName.toLowerCase() !== 'select') {
439
+ execute(<<~JS, values.flatten, by)
440
+ function(values, by) {
441
+ if (this.nodeName.toLowerCase() !== 'select') {
403
442
  throw new Error('Element is not a <select> element.');
404
443
  }
405
- const options = Array.from(element.options);
406
- element.value = undefined;
444
+ const options = Array.from(this.options);
445
+ this.value = undefined;
407
446
  for (const option of options) {
408
447
  option.selected = values.some((value) => option[by] === value);
409
- if (option.selected && !element.multiple) break;
448
+ if (option.selected && !this.multiple) break;
410
449
  }
411
- element.dispatchEvent(new Event('input', { bubbles: true }));
412
- element.dispatchEvent(new Event('change', { bubbles: true }));
450
+ this.dispatchEvent(new Event('input', { bubbles: true }));
451
+ this.dispatchEvent(new Event('change', { bubbles: true }));
413
452
  }
414
453
  JS
415
- page.evaluate_func(function, self, values.flatten, by, on: self)
416
454
  end
417
455
  end
418
456
 
419
457
  #
420
- # Evaluates the given JavaScript expression with `this` bound to the
421
- # node.
458
+ # Evaluates JavaScript with `this` bound to the node, and returns the
459
+ # result serialized into Ruby. Takes the same script shapes and arguments
460
+ # as {Frame::Runtime#evaluate}. Arrow functions don't get `this`, so use a
461
+ # `function` declaration when the script needs it.
422
462
  #
423
- # @param [String] expression
463
+ # @param (see Frame::Runtime#evaluate)
464
+ # @option (see Frame::Runtime#evaluate)
424
465
  #
425
466
  # @return [Object]
426
467
  #
427
468
  # @example
428
469
  # page.at_css("input").evaluate("this.value")
470
+ # page.at_css("input").evaluate("this.value + suffix", suffix: "!")
471
+ #
472
+ def evaluate(expression, *positional, **keywords)
473
+ @runtime.evaluate_in(self, expression, positional, keywords, mode: :value)
474
+ end
475
+
476
+ #
477
+ # Same as {#evaluate}, but returns a {RemoteObject} that stays in the
478
+ # browser instead of being serialized.
479
+ #
480
+ # @param (see Frame::Runtime#evaluate)
481
+ # @option (see Frame::Runtime#evaluate)
482
+ #
483
+ # @return [RemoteObject, Node, Object]
429
484
  #
430
- def evaluate(expression)
431
- page.evaluate_on(node: self, expression: expression)
485
+ def evaluate_handle(expression, *positional, **keywords)
486
+ @runtime.evaluate_in(self, expression, positional, keywords, mode: :handle)
487
+ end
488
+
489
+ #
490
+ # Runs JavaScript with `this` bound to the node, for its side effects.
491
+ # The script is used as a function body, so multiple statements need no
492
+ # wrapping.
493
+ #
494
+ # @param (see Frame::Runtime#evaluate)
495
+ # @option (see Frame::Runtime#evaluate)
496
+ #
497
+ # @return [Boolean]
498
+ # Always `true`.
499
+ #
500
+ # @example
501
+ # page.at_css("input").execute("this.value = text", text: "hello")
502
+ #
503
+ def execute(expression, *positional, **keywords)
504
+ @runtime.evaluate_in(self, expression, positional, keywords, mode: :none)
505
+ true
432
506
  end
433
507
 
434
508
  #
@@ -445,7 +519,7 @@ module Ferrum
445
519
  # We compare backendNodeId because once nodeId is sent to frontend backend
446
520
  # never returns same nodeId sending 0. In other words frontend is
447
521
  # responsible for keeping track of node ids.
448
- target_id == other.target_id && description["backendNodeId"] == other.description["backendNodeId"]
522
+ target_id == other.target_id && backend_node_id == other.backend_node_id
449
523
  end
450
524
 
451
525
  #
@@ -454,7 +528,8 @@ module Ferrum
454
528
  # @return [String]
455
529
  #
456
530
  def inspect
457
- %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} @description=#{@description.inspect}>)
531
+ %(#<#{self.class} @target_id=#{@target_id.inspect} @node_id=#{@node_id} ) +
532
+ %(@backend_node_id=#{@backend_node_id} @tag_name=#{@tag_name.inspect}>)
458
533
  end
459
534
 
460
535
  #
@@ -523,8 +598,8 @@ module Ferrum
523
598
 
524
599
  def bounding_rect_coordinates
525
600
  evaluate <<~JS
526
- [this.getBoundingClientRect().left + window.pageXOffset + (this.offsetWidth / 2),
527
- this.getBoundingClientRect().top + window.pageYOffset + (this.offsetHeight / 2)]
601
+ [this.getBoundingClientRect().left + (this.offsetWidth / 2),
602
+ this.getBoundingClientRect().top + (this.offsetHeight / 2)]
528
603
  JS
529
604
  end
530
605
 
@@ -104,6 +104,7 @@ module Ferrum
104
104
  frame.execution_id = nil
105
105
  else
106
106
  @frames.delete(params["frameId"])
107
+ @event.set if @main_frame.idle? && frame&.idle? == false && frames.size == 1
107
108
  end
108
109
  end
109
110
  end
@@ -203,8 +204,9 @@ module Ferrum
203
204
 
204
205
  def subscribe_execution_context_created
205
206
  on("Runtime.executionContextCreated") do |params|
206
- context_id = params.dig("context", "id")
207
- frame_id = params.dig("context", "auxData", "frameId")
207
+ context = params["context"]
208
+ context_id = context["id"]
209
+ frame_id = context.dig("auxData", "frameId")
208
210
 
209
211
  unless @main_frame.id
210
212
  root_frame = command("Page.getFrameTree").dig("frameTree", "frame", "id")
@@ -215,7 +217,11 @@ module Ferrum
215
217
  end
216
218
 
217
219
  frame = @frames.fetch_or_store(frame_id, Frame.new(frame_id, self))
218
- frame.execution_id = context_id
220
+ if context.dig("auxData", "isDefault")
221
+ frame.execution_id = context_id
222
+ elsif !context["name"].to_s.empty?
223
+ frame.isolated_world(context["name"]).execution_id = context_id
224
+ end
219
225
  end
220
226
  end
221
227
 
@@ -225,6 +231,7 @@ module Ferrum
225
231
  frame = frame_by(execution_id: execution_id)
226
232
  frame&.execution_id = nil
227
233
  frame&.state = :canceled
234
+ frames.flat_map(&:isolated_worlds).find { |w| w.execution_id == execution_id }&.execution_id = nil
228
235
  end
229
236
  end
230
237
 
@@ -240,6 +247,7 @@ module Ferrum
240
247
  @frames.each do |frame_id, f|
241
248
  if f.main?
242
249
  f.execution_id = nil
250
+ f.isolated_worlds.each { |w| w.execution_id = nil }
243
251
  f.loader_id = nil
244
252
  f.lifecycle_events.clear
245
253
  f.state = :canceled
@@ -307,13 +307,14 @@ module Ferrum
307
307
  end
308
308
 
309
309
  def bounding_rect(selector)
310
- rect = evaluate_async(%(
311
- const rect = document
312
- .querySelector(arguments[0])
313
- .getBoundingClientRect();
314
- const {x, y, width, height} = rect;
315
- arguments[1]([x, y, width, height])
316
- ), timeout, selector)
310
+ rect = evaluate(<<~JS, selector)
311
+ function(selector) {
312
+ const {x, y, width, height} = document
313
+ .querySelector(selector)
314
+ .getBoundingClientRect();
315
+ return [x, y, width, height];
316
+ }
317
+ JS
317
318
 
318
319
  { x: rect[0], y: rect[1], width: rect[2], height: rect[3] }
319
320
  end
data/lib/ferrum/page.rb CHANGED
@@ -8,6 +8,7 @@ require "ferrum/headers"
8
8
  require "ferrum/cookies"
9
9
  require "ferrum/dialog"
10
10
  require "ferrum/network"
11
+ require "ferrum/console"
11
12
  require "ferrum/accessibility"
12
13
  require "ferrum/interceptable"
13
14
  require "ferrum/downloads"
@@ -33,7 +34,8 @@ module Ferrum
33
34
 
34
35
  delegate %i[at_css at_xpath css xpath wait_for_selector
35
36
  current_url current_title url title body doctype content=
36
- execution_id execution_id! evaluate evaluate_on evaluate_async execute evaluate_func
37
+ execution_id execution_id! evaluate evaluate_handle execute
38
+ evaluate_on evaluate_async evaluate_func isolated_world
37
39
  add_script_tag add_style_tag] => :main_frame
38
40
  delegate %i[base_url default_user_agent timeout timeout=] => :@options
39
41
 
@@ -67,6 +69,11 @@ module Ferrum
67
69
  # @return [Network]
68
70
  attr_reader :network
69
71
 
72
+ # Console messages.
73
+ #
74
+ # @return [Console]
75
+ attr_reader :console
76
+
70
77
  # Accessibility object.
71
78
  #
72
79
  # @return [Accessibility]
@@ -87,6 +94,12 @@ module Ferrum
87
94
  # @return [Downloads]
88
95
  attr_reader :downloads
89
96
 
97
+ # Bumped every time Chrome reports `DOM.documentUpdated` and after every
98
+ # `DOM.getDocument`, both invalidate every node id handed out before them.
99
+ #
100
+ # @return [Integer]
101
+ attr_reader :document_version
102
+
90
103
  def initialize(client, context_id:, target_id:, proxy: nil)
91
104
  @client = client
92
105
  @context_id = context_id
@@ -96,6 +109,7 @@ module Ferrum
96
109
  @frames = Concurrent::Map.new
97
110
  @main_frame = Frame.new(nil, self)
98
111
  @event = Utils::Event.new.tap(&:set)
112
+ @document_version = 0
99
113
  self.proxy = proxy
100
114
 
101
115
  @mouse = Mouse.new(self)
@@ -103,6 +117,7 @@ module Ferrum
103
117
  @headers = Headers.new(self)
104
118
  @cookies = Cookies.new(self)
105
119
  @network = Network.new(self)
120
+ @console = Console.new(self)
106
121
  @accessibility = Accessibility.new(self)
107
122
  @tracing = Tracing.new(self)
108
123
  @downloads = Downloads.new(self)
@@ -176,14 +191,12 @@ module Ferrum
176
191
  #
177
192
  # @param [Float] scale_factor device scale factor value. 0 disables the override
178
193
  #
179
- # @param [Boolean] mobile whether to emulate mobile device and enable touch
194
+ # @param [Boolean, nil] mobile `true` emulates a mobile device with touch, `false` a desktop without touch, `nil`
195
+ # a desktop and leaves touch emulation as is. Turning touch off resets hover and pointer to the detected input
196
+ # devices, which undoes `--blink-settings` overrides until the next navigation
180
197
  #
181
- def set_viewport(width: 0, height: 0, scale_factor: 0, mobile: false)
182
- command(
183
- "Emulation.setTouchEmulationEnabled",
184
- enabled: mobile,
185
- maxTouchPoints: 1
186
- )
198
+ def set_viewport(width: 0, height: 0, scale_factor: 0, mobile: nil)
199
+ command("Emulation.setTouchEmulationEnabled", enabled: mobile, maxTouchPoints: 1) unless mobile.nil?
187
200
 
188
201
  command(
189
202
  "Emulation.setDeviceMetricsOverride",
@@ -191,7 +204,7 @@ module Ferrum
191
204
  width: width,
192
205
  height: height,
193
206
  deviceScaleFactor: scale_factor,
194
- mobile: mobile
207
+ mobile: mobile || false
195
208
  )
196
209
  end
197
210
 
@@ -384,6 +397,40 @@ module Ferrum
384
397
  @event.set
385
398
  end
386
399
 
400
+ #
401
+ # Runs the block and, if it starts a navigation of the main frame within
402
+ # `wait` seconds, waits for that navigation to finish, e.g. after a click
403
+ # on a link made from JavaScript.
404
+ #
405
+ # @param [Numeric] wait
406
+ # How many seconds to watch for a navigation after the block returns.
407
+ #
408
+ # @yield The code that may start a navigation.
409
+ #
410
+ # @return [Boolean]
411
+ # `false` when a navigation started but didn't finish within the page's
412
+ # `timeout`.
413
+ #
414
+ # @example
415
+ # page.wait_for_navigation { page.at_css("a").evaluate("this.click()") } # => true
416
+ #
417
+ def wait_for_navigation(wait = Mouse::CLICK_WAIT)
418
+ iteration = @event.reset
419
+ yield
420
+ @event.wait(wait)
421
+ iteration == @event.iteration || @event.wait(timeout)
422
+ end
423
+
424
+ #
425
+ # Waits for a navigation the block starts or raises {Ferrum::TimeoutError}.
426
+ # Accepts the same arguments as {#wait_for_navigation}.
427
+ #
428
+ # @raise [Ferrum::TimeoutError]
429
+ #
430
+ def wait_for_navigation!(...)
431
+ raise TimeoutError unless wait_for_navigation(...)
432
+ end
433
+
387
434
  #
388
435
  # Enables/disables CSP bypass.
389
436
  #
@@ -403,6 +450,45 @@ module Ferrum
403
450
  enabled
404
451
  end
405
452
 
453
+ #
454
+ # Adds JavaScript to run in every new document of this page and its in-process frames, before the document's own
455
+ # scripts. For every page and iframe of the browser, see {Browser::Options#add_extension}.
456
+ #
457
+ # @param [String] source
458
+ #
459
+ # @param [String, nil] world_name
460
+ # Runs the script in an isolated world with this name instead of the page's main world.
461
+ #
462
+ # @param [Boolean] run_immediately
463
+ # Runs the script in the current document as well.
464
+ #
465
+ # @return [String]
466
+ # The script identifier, for {#remove_script_on_new_document}.
467
+ #
468
+ # @example
469
+ # page.evaluate_on_new_document <<~JS
470
+ # Object.defineProperty(navigator, "languages", {
471
+ # get: function() { return ["tlh"]; }
472
+ # });
473
+ # JS
474
+ #
475
+ def evaluate_on_new_document(source, world_name: nil, run_immediately: false)
476
+ params = { source: source, worldName: world_name, runImmediately: run_immediately }.compact
477
+ command("Page.addScriptToEvaluateOnNewDocument", **params)["identifier"]
478
+ end
479
+
480
+ #
481
+ # Stops running a script added with {#evaluate_on_new_document} in new documents.
482
+ #
483
+ # @param [String] identifier
484
+ #
485
+ # @return [true]
486
+ #
487
+ def remove_script_on_new_document(identifier)
488
+ command("Page.removeScriptToEvaluateOnNewDocument", identifier: identifier)
489
+ true
490
+ end
491
+
406
492
  #
407
493
  # Activates (focuses) the target for the given page.
408
494
  # When you have multiple tabs you work with, and you need to switch a given one.
@@ -443,21 +529,14 @@ module Ferrum
443
529
  # page.command("Page.navigate", url: "https://github.com/")
444
530
  #
445
531
  def command(method, wait: 0, slowmoable: false, timeout: nil, **params)
446
- iteration = @event.reset if wait.positive?
447
- sleep(@options.slowmo) if slowmoable && @options.slowmo.positive?
448
- result = client.command(method, timeout: timeout || self.timeout, **params)
449
-
450
532
  if wait.positive?
451
- # Wait a bit after command and check if iteration has
452
- # changed, which means there was some network event for
453
- # the main frame, and it started to load new content.
454
- @event.wait(wait)
455
- if iteration != @event.iteration
456
- set = @event.wait(self.timeout)
457
- raise TimeoutError unless set
458
- end
533
+ result = nil
534
+ wait_for_navigation!(wait) { result = command(method, slowmoable:, timeout:, **params) }
535
+ return result
459
536
  end
460
- result
537
+
538
+ sleep(@options.slowmo) if slowmoable && @options.slowmo.positive?
539
+ client.command(method, timeout: timeout || self.timeout, **params)
461
540
  end
462
541
 
463
542
  # Subscribes to a CDP event, or to `:dialog`, `:request`, `:response`,
@@ -514,9 +593,13 @@ module Ferrum
514
593
  # The root node id, or `true` when sent asynchronously.
515
594
  #
516
595
  def document_node_id(async: false)
517
- return client.command("DOM.getDocument", async: true, depth: 0) if async
518
-
519
- command("DOM.getDocument", depth: 0).dig("root", "nodeId")
596
+ result = if async
597
+ client.command("DOM.getDocument", async: true, depth: 0)
598
+ else
599
+ command("DOM.getDocument", depth: 0).dig("root", "nodeId")
600
+ end
601
+ @document_version += 1
602
+ result
520
603
  end
521
604
 
522
605
  private
@@ -524,13 +607,10 @@ module Ferrum
524
607
  def subscribe
525
608
  frames_subscribe
526
609
  network.subscribe
610
+ console.subscribe
527
611
  downloads.subscribe
528
612
 
529
- if @options.logger
530
- on("Runtime.consoleAPICalled") do |params|
531
- log_console_api(params)
532
- end
533
- end
613
+ on("DOM.documentUpdated") { @document_version += 1 }
534
614
 
535
615
  if @options.js_errors
536
616
  on("Runtime.exceptionThrown") do |params|
@@ -560,19 +640,13 @@ module Ferrum
560
640
  if use_authorized_proxy?
561
641
  network.authorize(user: @proxy_user,
562
642
  password: @proxy_password,
563
- type: :proxy) do |request, _index, _total|
564
- request.continue
643
+ type: :proxy) do |request, _index, total|
644
+ request.continue if total == 1
565
645
  end
566
646
  end
567
647
 
568
648
  downloads.set_behavior(save_path: @options.save_path) if @options.save_path
569
649
 
570
- @options.extensions.each do |extension|
571
- command("Page.addScriptToEvaluateOnNewDocument", source: extension)
572
- end
573
-
574
- inject_extensions
575
-
576
650
  response = command("Page.getNavigationHistory")
577
651
  transition_type = response.dig("entries", 0, "transitionType")
578
652
  return if transition_type == "auto_toplevel"
@@ -585,29 +659,6 @@ module Ferrum
585
659
  document_node_id
586
660
  end
587
661
 
588
- def log_console_api(params)
589
- message = params.fetch("args", []).filter_map { |arg| arg["value"] || arg["description"] }.join(" ")
590
- @options.logger.puts("[#{params['type']}] #{message}")
591
-
592
- params.dig("stackTrace", "callFrames")&.each do |frame|
593
- location = "#{frame['url']}:#{frame['lineNumber'].to_i + 1}:#{frame['columnNumber'].to_i + 1}"
594
- @options.logger.puts(" at #{frame['functionName']} (#{location})")
595
- end
596
- end
597
-
598
- def inject_extensions
599
- @options.extensions.each do |extension|
600
- # https://github.com/GoogleChrome/puppeteer/issues/1443
601
- # https://github.com/ChromeDevTools/devtools-protocol/issues/77
602
- # https://github.com/cyrus-and/chrome-remote-interface/issues/319
603
- # We also evaluate script just in case because
604
- # `Page.addScriptToEvaluateOnNewDocument` doesn't work in popups.
605
- command("Runtime.evaluate", expression: extension,
606
- executionContextId: execution_id!,
607
- returnByValue: true)
608
- end
609
- end
610
-
611
662
  def history_navigate(delta:)
612
663
  history = command("Page.getNavigationHistory")
613
664
  index, entries = history.values_at("currentIndex", "entries")
data/lib/ferrum/proxy.rb CHANGED
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "logger"
3
4
  require "tempfile"
4
5
 
5
6
  begin
@@ -133,19 +134,19 @@ module Ferrum
133
134
 
134
135
  if proxy
135
136
  @logger.debug("CONNECT: sending a Request-Line")
136
- os << proxy_request_line << CRLF
137
+ os << proxy_request_line << WEBrick::CRLF
137
138
  @logger.debug("CONNECT: > #{proxy_request_line}")
138
139
  if credentials
139
140
  @logger.debug("CONNECT: sending credentials")
140
- os << "Proxy-Authorization: " << credentials << CRLF
141
+ os << "Proxy-Authorization: " << credentials << WEBrick::CRLF
141
142
  end
142
- os << CRLF
143
- proxy_status_line = os.gets(LF)
143
+ os << WEBrick::CRLF
144
+ proxy_status_line = os.gets(WEBrick::LF)
144
145
  @logger.debug("CONNECT: read Status-Line from the upstream server")
145
146
  @logger.debug("CONNECT: < #{proxy_status_line}")
146
147
  if %r{^HTTP/\d+\.\d+\s+200\s*} =~ proxy_status_line
147
- while line = os.gets(LF)
148
- break if /\A(#{CRLF}|#{LF})\z/om =~ line
148
+ while line = os.gets(WEBrick::LF)
149
+ break if /\A(#{WEBrick::CRLF}|#{WEBrick::LF})\z/om =~ line
149
150
  end
150
151
  else
151
152
  raise WEBrick::HTTPStatus::BadGateway
@@ -169,7 +170,7 @@ module Ferrum
169
170
 
170
171
  # Should clear request-line not to send the response twice.
171
172
  # see: HTTPServer#run
172
- req.parse(NullReader) rescue nil
173
+ req.parse(WEBrick::NullReader) rescue nil
173
174
  end
174
175
 
175
176
  begin