ferrum 0.19.1 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +60 -0
  3. data/lib/ferrum/browser/binary.rb +1 -2
  4. data/lib/ferrum/browser/options.rb +33 -12
  5. data/lib/ferrum/browser/process/output.rb +143 -0
  6. data/lib/ferrum/browser/process.rb +42 -69
  7. data/lib/ferrum/browser.rb +9 -23
  8. data/lib/ferrum/client/web_socket.rb +93 -73
  9. data/lib/ferrum/client.rb +9 -5
  10. data/lib/ferrum/console/location.rb +42 -0
  11. data/lib/ferrum/console/message.rb +230 -0
  12. data/lib/ferrum/console.rb +157 -0
  13. data/lib/ferrum/context.rb +23 -15
  14. data/lib/ferrum/contexts.rb +41 -18
  15. data/lib/ferrum/errors.rb +7 -5
  16. data/lib/ferrum/frame/dom.rb +50 -44
  17. data/lib/ferrum/frame/runtime/legacy.rb +336 -0
  18. data/lib/ferrum/frame/runtime.rb +416 -186
  19. data/lib/ferrum/frame/script.rb +289 -0
  20. data/lib/ferrum/frame.rb +33 -4
  21. data/lib/ferrum/interceptable.rb +19 -9
  22. data/lib/ferrum/isolated_world.rb +94 -0
  23. data/lib/ferrum/keyboard.rb +95 -8
  24. data/lib/ferrum/log/annotated.rb +77 -0
  25. data/lib/ferrum/log/default.rb +55 -0
  26. data/lib/ferrum/log/json.rb +32 -0
  27. data/lib/ferrum/log/paint.rb +28 -0
  28. data/lib/ferrum/log.rb +247 -0
  29. data/lib/ferrum/mouse.rb +16 -8
  30. data/lib/ferrum/node.rb +127 -52
  31. data/lib/ferrum/page/frames.rb +10 -3
  32. data/lib/ferrum/page/screenshot.rb +8 -7
  33. data/lib/ferrum/page.rb +106 -53
  34. data/lib/ferrum/proxy.rb +8 -7
  35. data/lib/ferrum/remote_object.rb +75 -0
  36. data/lib/ferrum/target.rb +7 -0
  37. data/lib/ferrum/utils/deprecate.rb +77 -0
  38. data/lib/ferrum/version.rb +1 -1
  39. data/lib/ferrum/worker.rb +8 -0
  40. data/lib/ferrum.rb +3 -0
  41. metadata +16 -2
@@ -0,0 +1,157 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ferrum/console/message"
4
+
5
+ module Ferrum
6
+ #
7
+ # Collects what the DevTools console would show for a page or worker:
8
+ # `console.*` calls, browser log entries and uncaught exceptions, as
9
+ # {Console::Message}s. Keeps the last {LIMIT} messages and is `Enumerable`
10
+ # over them, oldest first.
11
+ #
12
+ # @example
13
+ # browser.go_to("https://example.com")
14
+ # browser.console.select(&:error?) # => [#<Ferrum::Console::Message type=:error ...>]
15
+ # browser.console.clear
16
+ #
17
+ class Console
18
+ include Enumerable
19
+
20
+ LIMIT = 1000
21
+
22
+ def initialize(page)
23
+ @page = page
24
+ @messages = []
25
+ @listeners = Concurrent::Map.new
26
+ @listener_id = Concurrent::AtomicFixnum.new
27
+ @mutex = Mutex.new
28
+ end
29
+
30
+ #
31
+ # Yields each collected message, oldest first.
32
+ #
33
+ # @yieldparam [Message] message
34
+ #
35
+ # @return [Enumerator, self]
36
+ #
37
+ def each(&)
38
+ return to_enum(:each) { size } unless block_given?
39
+
40
+ to_a.each(&)
41
+ self
42
+ end
43
+
44
+ #
45
+ # The collected messages, oldest first.
46
+ #
47
+ # @return [Array<Message>]
48
+ #
49
+ def to_a
50
+ @mutex.synchronize { @messages.dup }
51
+ end
52
+
53
+ #
54
+ # The most recent message.
55
+ #
56
+ # @return [Message, nil]
57
+ #
58
+ def last
59
+ @mutex.synchronize { @messages.last }
60
+ end
61
+
62
+ #
63
+ # Number of collected messages.
64
+ #
65
+ # @return [Integer]
66
+ #
67
+ def size
68
+ @mutex.synchronize { @messages.size }
69
+ end
70
+
71
+ #
72
+ # Whether no messages were collected.
73
+ #
74
+ # @return [Boolean]
75
+ #
76
+ def empty?
77
+ size.zero?
78
+ end
79
+
80
+ #
81
+ # Forgets the collected messages.
82
+ #
83
+ # @return [self]
84
+ #
85
+ def clear
86
+ @mutex.synchronize { @messages.clear }
87
+ self
88
+ end
89
+
90
+ #
91
+ # Calls the block for every new message. Prefer `page.on(:console)`.
92
+ #
93
+ # @yieldparam [Message] message
94
+ #
95
+ # @return [Integer]
96
+ # The listener id, used to remove it via {#off}.
97
+ #
98
+ def on(&block)
99
+ @listener_id.increment.tap { |id| @listeners[id] = block }
100
+ end
101
+
102
+ #
103
+ # Removes a listener added via {#on}.
104
+ #
105
+ # @param [Integer] id
106
+ #
107
+ # @return [void]
108
+ #
109
+ def off(id)
110
+ @listeners.delete(id)
111
+ end
112
+
113
+ #
114
+ # Subscribes to the CDP events messages are built from.
115
+ #
116
+ # @return [void]
117
+ #
118
+ def subscribe
119
+ @page.on("Runtime.consoleAPICalled") do |params|
120
+ add(Message.from_console_api(params)) unless replayed?(params)
121
+ end
122
+
123
+ @page.on("Log.entryAdded") do |params|
124
+ entry = params["entry"]
125
+ add(Message.from_log_entry(entry)) unless entry["source"] == "worker"
126
+ end
127
+
128
+ @page.on("Runtime.exceptionThrown") do |params|
129
+ add(Message.from_exception(params))
130
+ end
131
+ end
132
+
133
+ #
134
+ # Debug representation of the console.
135
+ #
136
+ # @return [String]
137
+ #
138
+ def inspect
139
+ "#<#{self.class.name} size=#{size}>"
140
+ end
141
+
142
+ private
143
+
144
+ def replayed?(params)
145
+ params["executionContextId"].to_i.zero?
146
+ end
147
+
148
+ def add(message)
149
+ @mutex.synchronize do
150
+ @messages << message
151
+ @messages.shift if @messages.size > LIMIT
152
+ end
153
+
154
+ @listeners.each_value { |listener| listener.call(message) }
155
+ end
156
+ end
157
+ end
@@ -20,7 +20,7 @@ module Ferrum
20
20
  @implicit = implicit
21
21
  @client = client
22
22
  @contexts = contexts
23
- @targets = Concurrent::Map.new
23
+ @targets = Concurrent::Hash.new
24
24
  @pendings = Concurrent::Map.new
25
25
  end
26
26
 
@@ -56,11 +56,12 @@ module Ferrum
56
56
  @targets.values.select(&:page?).map(&:page)
57
57
  end
58
58
 
59
- # Dedicated and shared workers spawned by any page in this context.
59
+ # Dedicated and shared workers spawned by any page in this context,
60
+ # leaving out the ones we have no session with, e.g. detached ones.
60
61
  #
61
62
  # @return [Array<Worker>]
62
63
  def workers
63
- @targets.values.select { |t| t.worker? || t.shared_worker? }.map(&:worker)
64
+ @targets.values.select { |t| (t.worker? || t.shared_worker?) && t.session_id }.map(&:worker)
64
65
  end
65
66
 
66
67
  # Service worker targets registered in this context. Unlike {#workers},
@@ -124,13 +125,15 @@ module Ferrum
124
125
  #
125
126
  # @param [String, nil] session_id
126
127
  #
128
+ # @param [Boolean] attached
129
+ # The session comes from `Target.attachedToTarget`, so it replaces the one a target got from its frame owner.
130
+ #
127
131
  # @return [Target]
128
- def add_target(params:, session_id: nil)
132
+ def add_target(params:, session_id: nil, attached: false)
129
133
  new_target = Target.new(@client, session_id, params, implicit: implicit?)
130
- # `put_if_absent` returns nil if added a new value or existing if there was one already
131
- target = @targets.put_if_absent(new_target.id, new_target) || new_target
134
+ target = @targets[new_target.id] ||= new_target
132
135
  # on first iteration session_id may be nil, then if session is present here we must set it to the target
133
- target.session_id = session_id if session_id && target.session_id.nil?
136
+ target.session_id = session_id if session_id && (attached || target.session_id.nil?)
134
137
  @default_target ||= target
135
138
 
136
139
  new_pending = Concurrent::IVar.new
@@ -161,25 +164,34 @@ module Ferrum
161
164
 
162
165
  # Manually attaches to a target, e.g. a service worker discovered via
163
166
  # {#service_workers}. Once attached, `target.worker`/`target.page`
164
- # returns a connected {Worker}/{Page} for it.
167
+ # returns a connected {Worker}/{Page} for it. A target that already has
168
+ # a session keeps it.
165
169
  #
166
170
  # Note: attaching to a service worker's session prevents Chrome from
167
171
  # ever terminating it while the connection is open.
172
+ #
173
+ # @param [String] target_id
174
+ #
175
+ # @return [Target]
176
+ #
177
+ # @raise [NoSuchTargetError] if the context has no such target.
168
178
  def attach_target(target_id)
169
179
  target = @targets[target_id]
170
180
  raise NoSuchTargetError unless target
181
+ return target if target.session_id
171
182
 
172
183
  @contexts.manually_attached(target_id)
173
184
  session = @client.command("Target.attachToTarget", targetId: target_id, flatten: true)
174
185
  target.session_id = session["sessionId"]
175
- true
186
+ target
176
187
  end
177
188
 
178
189
  # Returns the first target for which the block returns truthy.
179
190
  #
180
191
  # @return [Target, nil]
181
192
  def find_target
182
- @targets.each_value { |t| return t if yield(t) }
193
+ targets = @targets.values
194
+ targets.each { |t| return t if yield(t) }
183
195
 
184
196
  nil
185
197
  end
@@ -189,11 +201,7 @@ module Ferrum
189
201
  #
190
202
  # @return [void]
191
203
  def close_targets_connection
192
- @targets.each_value do |target|
193
- next unless target.connected?
194
-
195
- target.close_connection
196
- end
204
+ @targets.values.select(&:connected?).each(&:close_connection)
197
205
  end
198
206
 
199
207
  # Disposes this browser context and all of its targets. The browser's
@@ -159,14 +159,18 @@ module Ferrum
159
159
  def subscribe_attached_target(client)
160
160
  client.on("Target.attachedToTarget") do |params|
161
161
  info, session_id = params.values_at("targetInfo", "sessionId")
162
- next detach_untracked(session_id) unless ALLOWED_TARGET_TYPES.include?(info["type"])
162
+ next detach_untracked(session_id, client) unless ALLOWED_TARGET_TYPES.include?(info["type"])
163
163
 
164
164
  context_id = info["browserContextId"]
165
165
  add_context(context_id)
166
- target = @contexts[context_id]&.add_target(session_id: session_id, params: info)
166
+ target = @contexts[context_id]&.add_target(session_id: session_id, params: info, attached: true)
167
167
 
168
168
  rearm_auto_attach(session_id, info["type"])
169
- handle_attach(target, session_id, params)
169
+ handle_attach(target, session_id, params, client)
170
+ end
171
+
172
+ client.on("Target.detachedFromTarget") do |params|
173
+ forget_session(params["targetId"], params["sessionId"])
170
174
  end
171
175
  end
172
176
 
@@ -218,37 +222,47 @@ module Ferrum
218
222
  subscribe_attached_target(client)
219
223
  end
220
224
 
221
- def handle_attach(target, session_id, params)
225
+ def handle_attach(target, session_id, params, parent)
222
226
  return unless target
223
227
 
224
228
  if target.service_worker?
225
- detach_unless_manually_attached(target, session_id)
229
+ detach_unless_manually_attached(target, session_id, parent)
226
230
  elsif target.worker? || target.shared_worker?
227
231
  connect_worker(target)
228
- elsif params["waitingForDebugger"]
229
- resume(session_id)
232
+ else
233
+ add_extensions_and_resume(session_id, params["waitingForDebugger"])
234
+ end
235
+ end
236
+
237
+ # A page or iframe is paused until we resume it, so the extensions we
238
+ # register now run before any script of its first document. An
239
+ # out-of-process iframe ignores them until the Page domain is enabled.
240
+ def add_extensions_and_resume(session_id, waiting)
241
+ extensions = @client.options.extensions
242
+ session = @client.session(session_id)
243
+ session.command("Page.enable", async: true) if extensions.any?
244
+ extensions.each do |source|
245
+ session.command("Page.addScriptToEvaluateOnNewDocument", async: true, source: source)
230
246
  end
247
+ resume(session_id) if waiting
231
248
  end
232
249
 
233
250
  # Auto-attach pauses every target it attaches to, including the types
234
251
  # we don't track: Chrome's own `browser_ui` targets, for instance. We
235
252
  # never resume those, so left alone they stay attached and paused for
236
253
  # the lifetime of the browser. Resume them and drop the session.
237
- def detach_untracked(session_id)
238
- detach(session_id)
239
- rescue BrowserError, TimeoutError
240
- nil
254
+ def detach_untracked(session_id, parent)
255
+ silent_detach(session_id, parent)
241
256
  end
242
257
 
243
258
  # Attaching keeps a service worker alive forever, so unless the caller
244
259
  # explicitly asked to connect to it (via Context#attach_target), we
245
260
  # just resume it and let go.
246
- def detach_unless_manually_attached(target, session_id)
261
+ def detach_unless_manually_attached(target, session_id, parent)
247
262
  return if @manually_attached.delete(target.id)
248
263
 
249
- detach(session_id)
250
- rescue BrowserError, TimeoutError
251
- nil
264
+ forget_session(target.id, session_id)
265
+ silent_detach(session_id, parent)
252
266
  end
253
267
 
254
268
  # Workers have no events to notify us when they're ready, so we
@@ -264,9 +278,18 @@ module Ferrum
264
278
  @client.session(session_id).command("Runtime.runIfWaitingForDebugger", async: true)
265
279
  end
266
280
 
267
- def detach(session_id)
268
- resume(session_id)
269
- @client.command("Target.detachFromTarget", sessionId: session_id)
281
+ def forget_session(target_id, session_id)
282
+ target = find_by(target_id: target_id)&.targets&.[](target_id)
283
+ target.session_id = nil if target&.session_id == session_id
284
+ end
285
+
286
+ def silent_detach(session_id, parent)
287
+ Utils::Thread.spawn do
288
+ @client.session(session_id).command("Runtime.runIfWaitingForDebugger")
289
+ parent.command("Target.detachFromTarget", sessionId: session_id)
290
+ rescue Error
291
+ nil
292
+ end
270
293
  end
271
294
 
272
295
  def discover
data/lib/ferrum/errors.rb CHANGED
@@ -87,7 +87,7 @@ module Ferrum
87
87
  end
88
88
 
89
89
  # Raised when an evaluated script takes too long to return a value.
90
- class ScriptTimeoutError < Error
90
+ class ScriptTimeoutError < TimeoutError
91
91
  #
92
92
  # Explains that the evaluated script timed out.
93
93
  #
@@ -112,11 +112,13 @@ module Ferrum
112
112
  # Raised when connecting to an already running browser, but its
113
113
  # `/json/version` endpoint doesn't answer with the websocket url.
114
114
  class NoWebSocketUrlError < Error
115
- attr_reader :url
115
+ attr_reader :url, :status
116
116
 
117
- def initialize(url)
117
+ def initialize(url, status = nil)
118
118
  @url = url
119
- super("Couldn't get a websocket url out of #{url}, `/json/version` there didn't answer with JSON. " \
119
+ @status = status
120
+ reason = status ? "answered #{status}" : "didn't answer with JSON"
121
+ super("Couldn't get a websocket url out of #{url}, `/json/version` there #{reason}. " \
120
122
  "Check the browser is reachable at that url and that any credentials it needs are part of it")
121
123
  end
122
124
  end
@@ -214,7 +216,7 @@ module Ferrum
214
216
 
215
217
  def initialize(response)
216
218
  @class_name, @message = response["exception"]&.values_at("className", "description")
217
- @message ||= response["text"]
219
+ @message ||= [response["text"], response.dig("exception", "value")].compact.join(" ")
218
220
  @stack_trace = response["stackTrace"]
219
221
  super(response.merge("message" => @message))
220
222
  end
@@ -27,32 +27,38 @@ module Ferrum
27
27
  #
28
28
  module DOM
29
29
  SCRIPT_SRC_TAG = <<~JS
30
- const script = document.createElement("script");
31
- script.src = arguments[0];
32
- script.type = arguments[1];
33
- script.onload = arguments[2];
34
- document.head.appendChild(script);
30
+ function(url, type) {
31
+ const script = document.createElement("script");
32
+ script.src = url;
33
+ script.type = type;
34
+ document.head.appendChild(script);
35
+ return new Promise(resolve => script.onload = resolve);
36
+ }
35
37
  JS
36
38
  SCRIPT_TEXT_TAG = <<~JS
37
- const script = document.createElement("script");
38
- script.text = arguments[0];
39
- script.type = arguments[1];
40
- document.head.appendChild(script);
41
- arguments[2]();
39
+ function(content, type) {
40
+ const script = document.createElement("script");
41
+ script.text = content;
42
+ script.type = type;
43
+ document.head.appendChild(script);
44
+ }
42
45
  JS
43
46
  STYLE_TAG = <<~JS
44
- const style = document.createElement("style");
45
- style.type = "text/css";
46
- style.appendChild(document.createTextNode(arguments[0]));
47
- document.head.appendChild(style);
48
- arguments[1]();
47
+ function(content) {
48
+ const style = document.createElement("style");
49
+ style.type = "text/css";
50
+ style.appendChild(document.createTextNode(content));
51
+ document.head.appendChild(style);
52
+ }
49
53
  JS
50
54
  LINK_TAG = <<~JS
51
- const link = document.createElement("link");
52
- link.rel = "stylesheet";
53
- link.href = arguments[0];
54
- link.onload = arguments[1];
55
- document.head.appendChild(link);
55
+ function(url) {
56
+ const link = document.createElement("link");
57
+ link.rel = "stylesheet";
58
+ link.href = url;
59
+ document.head.appendChild(link);
60
+ return new Promise(resolve => link.onload = resolve);
61
+ }
56
62
  JS
57
63
 
58
64
  #
@@ -160,7 +166,7 @@ module Ferrum
160
166
  }
161
167
  JS
162
168
 
163
- evaluate_func(expr, selector, within)
169
+ evaluate(expr, selector, within)
164
170
  end
165
171
 
166
172
  #
@@ -187,7 +193,7 @@ module Ferrum
187
193
  return xpath.snapshotItem(0);
188
194
  }
189
195
  JS
190
- evaluate_func(expr, selector, within)
196
+ evaluate(expr, selector, within)
191
197
  end
192
198
 
193
199
  #
@@ -214,7 +220,7 @@ module Ferrum
214
220
  }
215
221
  JS
216
222
 
217
- evaluate_func(expr, selector, within)
223
+ evaluate(expr, selector, within)
218
224
  end
219
225
 
220
226
  #
@@ -241,7 +247,7 @@ module Ferrum
241
247
  }
242
248
  JS
243
249
 
244
- evaluate_func(expr, selector, within)
250
+ evaluate(expr, selector, within)
245
251
  end
246
252
 
247
253
  #
@@ -307,17 +313,17 @@ module Ferrum
307
313
  # browser.add_script_tag(url: "http://example.com/stylesheet.css") # => true
308
314
  #
309
315
  def add_script_tag(url: nil, path: nil, content: nil, type: "text/javascript")
310
- expr, *args = if url
311
- [SCRIPT_SRC_TAG, url, type]
312
- elsif path || content
313
- if path
314
- content = File.read(path)
315
- content += "\n//# sourceURL=#{path}"
316
- end
317
- [SCRIPT_TEXT_TAG, content, type]
318
- end
316
+ if url
317
+ evaluate(SCRIPT_SRC_TAG, url, type)
318
+ elsif path || content
319
+ if path
320
+ content = File.read(path)
321
+ content += "\n//# sourceURL=#{path}"
322
+ end
323
+ evaluate(SCRIPT_TEXT_TAG, content, type)
324
+ end
319
325
 
320
- evaluate_async(expr, @page.timeout, *args)
326
+ true
321
327
  end
322
328
 
323
329
  #
@@ -333,17 +339,17 @@ module Ferrum
333
339
  # browser.add_style_tag(content: "h1 { font-size: 40px; }") # => true
334
340
  #
335
341
  def add_style_tag(url: nil, path: nil, content: nil)
336
- expr, *args = if url
337
- [LINK_TAG, url]
338
- elsif path || content
339
- if path
340
- content = File.read(path)
341
- content += "\n//# sourceURL=#{path}"
342
- end
343
- [STYLE_TAG, content]
344
- end
342
+ if url
343
+ evaluate(LINK_TAG, url)
344
+ elsif path || content
345
+ if path
346
+ content = File.read(path)
347
+ content += "\n//# sourceURL=#{path}"
348
+ end
349
+ evaluate(STYLE_TAG, content)
350
+ end
345
351
 
346
- evaluate_async(expr, @page.timeout, *args)
352
+ true
347
353
  end
348
354
  end
349
355
  end