ferrum 0.17.1 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/LICENSE +1 -1
  3. data/README.md +9 -1369
  4. data/lib/ferrum/accessibility/ax_node.rb +108 -0
  5. data/lib/ferrum/accessibility.rb +106 -0
  6. data/lib/ferrum/browser/binary.rb +41 -0
  7. data/lib/ferrum/browser/command.rb +20 -0
  8. data/lib/ferrum/browser/options/base.rb +69 -2
  9. data/lib/ferrum/browser/options/chrome.rb +88 -26
  10. data/lib/ferrum/browser/options/firefox.rb +32 -0
  11. data/lib/ferrum/browser/options.rb +43 -3
  12. data/lib/ferrum/browser/process.rb +56 -4
  13. data/lib/ferrum/browser/xvfb.rb +24 -0
  14. data/lib/ferrum/browser.rb +35 -9
  15. data/lib/ferrum/client/subscriber.rb +58 -0
  16. data/lib/ferrum/client/web_socket.rb +63 -12
  17. data/lib/ferrum/client.rb +206 -14
  18. data/lib/ferrum/context.rb +99 -4
  19. data/lib/ferrum/contexts.rb +136 -22
  20. data/lib/ferrum/cookies/cookie.rb +7 -1
  21. data/lib/ferrum/cookies.rb +5 -0
  22. data/lib/ferrum/dialog.rb +18 -2
  23. data/lib/ferrum/downloads.rb +52 -0
  24. data/lib/ferrum/errors.rb +62 -6
  25. data/lib/ferrum/frame/dom.rb +17 -0
  26. data/lib/ferrum/frame/runtime.rb +49 -8
  27. data/lib/ferrum/frame.rb +58 -1
  28. data/lib/ferrum/headers.rb +6 -0
  29. data/lib/ferrum/interceptable.rb +62 -0
  30. data/lib/ferrum/keyboard.rb +25 -0
  31. data/lib/ferrum/mouse.rb +6 -0
  32. data/lib/ferrum/network/auth_request.rb +81 -2
  33. data/lib/ferrum/network/error.rb +15 -0
  34. data/lib/ferrum/network/exchange.rb +10 -0
  35. data/lib/ferrum/network/intercepted_request.rb +94 -2
  36. data/lib/ferrum/network/request.rb +1 -1
  37. data/lib/ferrum/network/response.rb +16 -1
  38. data/lib/ferrum/network.rb +122 -10
  39. data/lib/ferrum/node.rb +305 -15
  40. data/lib/ferrum/page/animation.rb +7 -2
  41. data/lib/ferrum/page/frames.rb +69 -7
  42. data/lib/ferrum/page/screencast.rb +5 -0
  43. data/lib/ferrum/page/screenshot.rb +52 -20
  44. data/lib/ferrum/page/stream.rb +56 -0
  45. data/lib/ferrum/page/tracing.rb +6 -0
  46. data/lib/ferrum/page.rb +141 -46
  47. data/lib/ferrum/proxy.rb +52 -2
  48. data/lib/ferrum/rgba.rb +10 -0
  49. data/lib/ferrum/target.rb +124 -1
  50. data/lib/ferrum/utils/attempt.rb +20 -0
  51. data/lib/ferrum/utils/elapsed_time.rb +38 -0
  52. data/lib/ferrum/utils/event.rb +14 -0
  53. data/lib/ferrum/utils/platform.rb +22 -1
  54. data/lib/ferrum/utils/thread.rb +12 -0
  55. data/lib/ferrum/version.rb +1 -1
  56. data/lib/ferrum/worker.rb +125 -0
  57. data/lib/ferrum.rb +7 -0
  58. metadata +8 -21
data/lib/ferrum/target.rb CHANGED
@@ -1,6 +1,12 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Ferrum
4
+ #
5
+ # Represents a CDP target, e.g. a page, iframe, dedicated/shared worker or
6
+ # service worker. Wraps the raw `targetInfo` params and lazily connects to
7
+ # the target as a {Page} or {Worker} over its own {Client}/{SessionClient}
8
+ # session, so a `Target` can exist without being connected to yet.
9
+ #
4
10
  class Target
5
11
  NEW_WINDOW_WAIT = ENV.fetch("FERRUM_NEW_WINDOW_WAIT", 0.3).to_f
6
12
 
@@ -13,70 +19,187 @@ module Ferrum
13
19
 
14
20
  def initialize(browser_client, session_id = nil, params = nil)
15
21
  @page = nil
22
+ @worker = nil
16
23
  @session_id = session_id
17
24
  @params = params
18
25
  @browser_client = browser_client
19
26
  @options = browser_client.options
20
27
  end
21
28
 
29
+ # Merges freshly received CDP target info into this target's params,
30
+ # e.g. on `Target.targetInfoChanged`.
31
+ #
32
+ # @param [Hash] params
33
+ #
34
+ # @return [Hash]
22
35
  def update(params)
23
36
  @params.merge!(params)
24
37
  end
25
38
 
39
+ # Whether this target has already been connected to as a {Page} or
40
+ # {Worker}.
41
+ #
42
+ # @return [Boolean]
26
43
  def connected?
27
- !!@page
44
+ !!@page || !!@worker
28
45
  end
29
46
 
47
+ # Connects to and returns this target's {Page}.
48
+ #
49
+ # @return [Page]
30
50
  def page
31
51
  @page ||= build_page
32
52
  end
33
53
 
54
+ # Connects to and returns this target's {Worker}.
55
+ #
56
+ # @return [Worker]
57
+ def worker
58
+ @worker ||= build_worker
59
+ end
60
+
61
+ # The client connected to this target's own session (or a dedicated
62
+ # WebSocket connection if `options.flatten` is disabled).
63
+ #
64
+ # @return [Client, SessionClient]
34
65
  def client
35
66
  @client ||= build_client
36
67
  end
37
68
 
69
+ # Builds a {Page} for this target. Called lazily by `page`, or eagerly
70
+ # by {Context#create_page}.
71
+ #
72
+ # @param [Hash] options
73
+ #
74
+ # @return [Page]
38
75
  def build_page(**options)
39
76
  maybe_sleep_if_new_window
40
77
  Page.new(client, context_id: context_id, target_id: id, **options)
41
78
  end
42
79
 
80
+ # Builds a {Worker} for this target. Called lazily by {#worker}.
81
+ #
82
+ # @return [Worker]
83
+ def build_worker
84
+ Worker.new(client, target_id: id, url: url)
85
+ end
86
+
87
+ # Closes the WebSocket connection, without closing the target itself
88
+ # in the browser.
89
+ #
90
+ # @return [void]
91
+ def close_connection
92
+ @page&.close_connection
93
+ @worker&.close_connection
94
+ end
95
+
96
+ # The target's id.
97
+ #
98
+ # @return [String]
43
99
  def id
44
100
  @params["targetId"]
45
101
  end
46
102
 
103
+ # The target's type, e.g. `"page"`, `"iframe"`, `"worker"`,
104
+ # `"shared_worker"`, `"service_worker"`.
105
+ #
106
+ # @return [String]
47
107
  def type
48
108
  @params["type"]
49
109
  end
50
110
 
111
+ # The target's title.
112
+ #
113
+ # @return [String]
51
114
  def title
52
115
  @params["title"]
53
116
  end
54
117
 
118
+ # The target's URL.
119
+ #
120
+ # @return [String]
55
121
  def url
56
122
  @params["url"]
57
123
  end
58
124
 
125
+ # The id of the target that opened this one, set only for
126
+ # windows/tabs opened via `window.open`/links/etc.
127
+ #
128
+ # @return [String, nil]
59
129
  def opener_id
60
130
  @params["openerId"]
61
131
  end
62
132
 
133
+ # The id of the target that spawned this one, set for iframes and
134
+ # workers. Unlike `opener_id`, which is only set for windows/tabs opened
135
+ # via `window.open`/links/etc.
136
+ def parent_id
137
+ @params["parentId"]
138
+ end
139
+
140
+ # The id of the browser context this target belongs to.
141
+ #
142
+ # @return [String, nil]
63
143
  def context_id
64
144
  @params["browserContextId"]
65
145
  end
66
146
 
147
+ # Whether this target is a window/tab, i.e. was opened via
148
+ # `window.open`/a link/etc. and thus has an {#opener_id}.
149
+ #
150
+ # @return [Boolean]
67
151
  def window?
68
152
  !!opener_id
69
153
  end
70
154
 
155
+ # Whether this target is an iframe.
156
+ #
157
+ # @return [Boolean]
71
158
  def iframe?
72
159
  type == "iframe"
73
160
  end
74
161
 
162
+ # Whether this target is a page.
163
+ #
164
+ # @return [Boolean]
165
+ def page?
166
+ type == "page"
167
+ end
168
+
169
+ # Whether this target is a dedicated worker.
170
+ #
171
+ # @return [Boolean]
172
+ def worker?
173
+ type == "worker"
174
+ end
175
+
176
+ # Whether this target is a shared worker.
177
+ #
178
+ # @return [Boolean]
179
+ def shared_worker?
180
+ type == "shared_worker"
181
+ end
182
+
183
+ # Whether this target is a service worker.
184
+ #
185
+ # @return [Boolean]
186
+ def service_worker?
187
+ type == "service_worker"
188
+ end
189
+
190
+ # Chrome fires no events for a newly opened window, so we sleep a bit
191
+ # to give it a chance to load before connecting.
192
+ #
193
+ # @return [void]
75
194
  def maybe_sleep_if_new_window
76
195
  # Dirty hack because new window doesn't have events at all
77
196
  sleep(NEW_WINDOW_WAIT) if window?
78
197
  end
79
198
 
199
+ # Sends a CDP command through {#client}.
200
+ #
201
+ # @return [Boolean, Hash]
202
+ # `true` when sent asynchronously, otherwise the command's result.
80
203
  def command(...)
81
204
  client.command(...)
82
205
  end
@@ -2,9 +2,29 @@
2
2
 
3
3
  module Ferrum
4
4
  module Utils
5
+ #
6
+ # A retry-with-backoff helper for re-running a block a fixed number of
7
+ # times when it raises one of a given set of exceptions, sleeping
8
+ # between attempts.
9
+ #
5
10
  module Attempt
6
11
  module_function
7
12
 
13
+ #
14
+ # Retries the block up to `max` times when one of `errors` is raised,
15
+ # sleeping `wait` seconds between attempts.
16
+ #
17
+ # @param [Array<Class>, Class] errors
18
+ # Exception classes that trigger a retry.
19
+ #
20
+ # @param [Integer] max
21
+ # Maximum number of attempts.
22
+ #
23
+ # @param [Numeric] wait
24
+ # Seconds to sleep between attempts.
25
+ #
26
+ # @return [Object]
27
+ #
8
28
  def with_retry(errors:, max:, wait:)
9
29
  attempts ||= 1
10
30
  yield
@@ -2,25 +2,63 @@
2
2
 
3
3
  module Ferrum
4
4
  module Utils
5
+ #
6
+ # A monotonic-clock helper for tracking elapsed time and checking
7
+ # timeouts, backed by `Concurrent.monotonic_time`.
8
+ #
5
9
  module ElapsedTime
6
10
  module_function
7
11
 
12
+ #
13
+ # Sets the start point to the current monotonic time unless already set.
14
+ #
15
+ # @return [Float]
16
+ #
8
17
  def start
9
18
  @start ||= monotonic_time
10
19
  end
11
20
 
21
+ #
22
+ # Resets the start point to the current monotonic time.
23
+ #
24
+ # @return [Float]
25
+ #
12
26
  def reset
13
27
  @start = monotonic_time
14
28
  end
15
29
 
30
+ #
31
+ # Returns the time elapsed since `start` (or since {.start}/{.reset} was called).
32
+ #
33
+ # @param [Float, nil] start
34
+ # Monotonic time to measure from, defaults to the stored start point.
35
+ #
36
+ # @return [Float]
37
+ #
16
38
  def elapsed_time(start = nil)
17
39
  monotonic_time - (start || @start)
18
40
  end
19
41
 
42
+ #
43
+ # Returns the current monotonic clock time in seconds.
44
+ #
45
+ # @return [Float]
46
+ #
20
47
  def monotonic_time
21
48
  Concurrent.monotonic_time
22
49
  end
23
50
 
51
+ #
52
+ # Whether more than `timeout` seconds have elapsed since `start`.
53
+ #
54
+ # @param [Float] start
55
+ # Monotonic time to measure from.
56
+ #
57
+ # @param [Float, Integer] timeout
58
+ # The timeout, in seconds.
59
+ #
60
+ # @return [Boolean]
61
+ #
24
62
  def timeout?(start, timeout)
25
63
  elapsed_time(start) > timeout
26
64
  end
@@ -2,11 +2,25 @@
2
2
 
3
3
  module Ferrum
4
4
  module Utils
5
+ #
6
+ # A {Concurrent::Event} subclass that additionally tracks how many
7
+ # times it has been reset, via {#iteration}.
8
+ #
5
9
  class Event < Concurrent::Event
10
+ #
11
+ # Number of times the event has been reset.
12
+ #
13
+ # @return [Integer]
14
+ #
6
15
  def iteration
7
16
  synchronize { @iteration }
8
17
  end
9
18
 
19
+ #
20
+ # Marks the event as unset and increments the iteration counter.
21
+ #
22
+ # @return [Integer]
23
+ #
10
24
  def reset
11
25
  synchronize do
12
26
  @iteration += 1
@@ -2,28 +2,49 @@
2
2
 
3
3
  module Ferrum
4
4
  module Utils
5
+ #
6
+ # OS and Ruby engine detection helpers, used to pick the right browser
7
+ # path/flags and behavior for the current platform.
8
+ #
5
9
  module Platform
6
10
  module_function
7
11
 
8
- def name
12
+ #
13
+ # Detects the current platform.
14
+ #
15
+ # @return [:mac, :windows, :linux]
16
+ #
17
+ def platform_name
9
18
  return :mac if mac?
10
19
  return :windows if windows?
11
20
 
12
21
  :linux
13
22
  end
14
23
 
24
+ # Whether the current platform is Windows.
25
+ #
26
+ # @return [Boolean]
15
27
  def windows?
16
28
  RbConfig::CONFIG["host_os"] =~ /mingw|mswin|cygwin/
17
29
  end
18
30
 
31
+ # Whether the current platform is macOS.
32
+ #
33
+ # @return [Boolean]
19
34
  def mac?
20
35
  RbConfig::CONFIG["host_os"] =~ /darwin/
21
36
  end
22
37
 
38
+ # Whether the current platform is macOS running on Apple Silicon (arm64).
39
+ #
40
+ # @return [Boolean]
23
41
  def mac_arm?
24
42
  mac? && RbConfig::CONFIG["host_cpu"] =~ /arm/
25
43
  end
26
44
 
45
+ # Whether the current Ruby engine is MRI (as opposed to e.g. JRuby, TruffleRuby).
46
+ #
47
+ # @return [Boolean]
27
48
  def mri?
28
49
  defined?(RUBY_ENGINE) && RUBY_ENGINE == "ruby"
29
50
  end
@@ -2,9 +2,21 @@
2
2
 
3
3
  module Ferrum
4
4
  module Utils
5
+ #
6
+ # A helper for spawning threads with consistent exception-handling
7
+ # behavior (abort-on-exception and report-on-exception).
8
+ #
5
9
  module Thread
6
10
  module_function
7
11
 
12
+ #
13
+ # Spawns a new thread running the given block.
14
+ #
15
+ # @param [Boolean] abort_on_exception
16
+ # Whether the thread aborts the process if it raises an unhandled exception.
17
+ #
18
+ # @return [Thread]
19
+ #
8
20
  def spawn(abort_on_exception: true)
9
21
  ::Thread.new(abort_on_exception) do |whether_abort_on_exception|
10
22
  ::Thread.current.abort_on_exception = whether_abort_on_exception
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Ferrum
4
- VERSION = "0.17.1"
4
+ VERSION = "0.18.0"
5
5
  end
@@ -0,0 +1,125 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "ferrum/frame/runtime"
4
+ require "ferrum/interceptable"
5
+
6
+ module Ferrum
7
+ # A dedicated or shared Worker spawned by a page. Unlike {Page} it has no
8
+ # DOM, frames, mouse/keyboard, or navigation history -- just a single
9
+ # global execution context and its own network activity.
10
+ class Worker
11
+ include Frame::Runtime
12
+ include Interceptable
13
+
14
+ # Client connection.
15
+ #
16
+ # @return [Client, SessionClient]
17
+ attr_reader :client
18
+
19
+ attr_reader :target_id, :url
20
+
21
+ def initialize(client, target_id:, url:)
22
+ @client = client
23
+ @target_id = target_id
24
+ @url = url
25
+ @options = client.options
26
+ @page = self
27
+ @execution_id = Concurrent::MVar.new
28
+ @network = Network.new(self)
29
+
30
+ subscribe
31
+ prepare
32
+ end
33
+
34
+ # Network object.
35
+ #
36
+ # @return [Network]
37
+ attr_reader :network
38
+
39
+ # How long to wait for CDP responses and JS evaluation to complete.
40
+ #
41
+ # @return [Numeric]
42
+ def timeout
43
+ @options.timeout
44
+ end
45
+
46
+ # Sends a CDP command through {#client}.
47
+ #
48
+ # @return [Boolean, Hash]
49
+ # `true` when sent asynchronously, otherwise the command's result.
50
+ def command(...)
51
+ client.command(...)
52
+ end
53
+
54
+ # Workers have a single execution context and no navigable document, so
55
+ # there's nothing for {Network} to check requests against.
56
+ def main_frame
57
+ nil
58
+ end
59
+
60
+ # Closes the target in the browser, and its connection.
61
+ #
62
+ # @return [Boolean]
63
+ def close
64
+ client.command("Target.closeTarget", async: true, targetId: target_id)
65
+ close_connection
66
+
67
+ true
68
+ end
69
+
70
+ # Closes the WebSocket connection only, without asking the browser to
71
+ # close the underlying target. Kept separate from {#close} so a whole
72
+ # context can be dropped (the browser closes its targets anyway) without
73
+ # every worker having to be closed one by one.
74
+ #
75
+ # @return [void]
76
+ def close_connection
77
+ client&.close
78
+ end
79
+
80
+ # Debug representation of the worker.
81
+ #
82
+ # @return [String]
83
+ def inspect
84
+ "#<#{self.class} @target_id=#{@target_id.inspect} @url=#{@url.inspect}>"
85
+ end
86
+
87
+ private
88
+
89
+ def subscribe
90
+ network.subscribe
91
+
92
+ on("Runtime.executionContextCreated") do |params|
93
+ self.execution_id = params.dig("context", "id")
94
+ end
95
+
96
+ return unless @options.js_errors
97
+
98
+ on("Runtime.exceptionThrown") do |params|
99
+ # FIXME: https://jvns.ca/blog/2015/11/27/why-rubys-timeout-is-dangerous-and-thread-dot-raise-is-terrifying/
100
+ Thread.main.raise JavaScriptError, params["exceptionDetails"]
101
+ end
102
+ end
103
+
104
+ def prepare
105
+ command("Runtime.enable")
106
+ command("Network.enable")
107
+ command("Runtime.runIfWaitingForDebugger")
108
+ end
109
+
110
+ def execution_id!
111
+ value = @execution_id.borrow(timeout, &:itself)
112
+ raise NoExecutionContextError if value.instance_of?(Object)
113
+
114
+ value
115
+ end
116
+
117
+ def execution_id=(value)
118
+ if value.nil?
119
+ @execution_id.try_take!
120
+ else
121
+ @execution_id.try_put!(value)
122
+ end
123
+ end
124
+ end
125
+ end
data/lib/ferrum.rb CHANGED
@@ -10,5 +10,12 @@ require "ferrum/errors"
10
10
  require "ferrum/browser"
11
11
  require "ferrum/node"
12
12
 
13
+ #
14
+ # Ferrum is a pure Ruby driver for headless Chrome and Firefox. It talks
15
+ # directly to the browser over the Chrome DevTools Protocol (CDP), so it
16
+ # doesn't depend on Selenium, WebDriver or any other third party gem.
17
+ #
18
+ # {Ferrum::Browser} is the entry point most applications start from.
19
+ #
13
20
  module Ferrum
14
21
  end
metadata CHANGED
@@ -1,14 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ferrum
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.17.1
4
+ version: 0.18.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dmitry Vorotilin
8
- autorequire:
9
8
  bindir: bin
10
9
  cert_chain: []
11
- date: 2025-05-11 00:00:00.000000000 Z
10
+ date: 1980-01-02 00:00:00.000000000 Z
12
11
  dependencies:
13
12
  - !ruby/object:Gem::Dependency
14
13
  name: addressable
@@ -52,20 +51,6 @@ dependencies:
52
51
  - - "~>"
53
52
  - !ruby/object:Gem::Version
54
53
  version: '1.1'
55
- - !ruby/object:Gem::Dependency
56
- name: webrick
57
- requirement: !ruby/object:Gem::Requirement
58
- requirements:
59
- - - "~>"
60
- - !ruby/object:Gem::Version
61
- version: '1.7'
62
- type: :runtime
63
- prerelease: false
64
- version_requirements: !ruby/object:Gem::Requirement
65
- requirements:
66
- - - "~>"
67
- - !ruby/object:Gem::Version
68
- version: '1.7'
69
54
  - !ruby/object:Gem::Dependency
70
55
  name: websocket-driver
71
56
  requirement: !ruby/object:Gem::Requirement
@@ -90,6 +75,8 @@ files:
90
75
  - LICENSE
91
76
  - README.md
92
77
  - lib/ferrum.rb
78
+ - lib/ferrum/accessibility.rb
79
+ - lib/ferrum/accessibility/ax_node.rb
93
80
  - lib/ferrum/browser.rb
94
81
  - lib/ferrum/browser/binary.rb
95
82
  - lib/ferrum/browser/command.rb
@@ -114,6 +101,7 @@ files:
114
101
  - lib/ferrum/frame/dom.rb
115
102
  - lib/ferrum/frame/runtime.rb
116
103
  - lib/ferrum/headers.rb
104
+ - lib/ferrum/interceptable.rb
117
105
  - lib/ferrum/keyboard.json
118
106
  - lib/ferrum/keyboard.rb
119
107
  - lib/ferrum/mouse.rb
@@ -142,6 +130,7 @@ files:
142
130
  - lib/ferrum/utils/platform.rb
143
131
  - lib/ferrum/utils/thread.rb
144
132
  - lib/ferrum/version.rb
133
+ - lib/ferrum/worker.rb
145
134
  homepage: https://github.com/rubycdp/ferrum
146
135
  licenses:
147
136
  - MIT
@@ -152,7 +141,6 @@ metadata:
152
141
  changelog_uri: https://github.com/rubycdp/ferrum/blob/main/CHANGELOG.md
153
142
  source_code_uri: https://github.com/rubycdp/ferrum
154
143
  rubygems_mfa_required: 'true'
155
- post_install_message:
156
144
  rdoc_options: []
157
145
  require_paths:
158
146
  - lib
@@ -160,15 +148,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
160
148
  requirements:
161
149
  - - ">="
162
150
  - !ruby/object:Gem::Version
163
- version: 2.7.0
151
+ version: '3.1'
164
152
  required_rubygems_version: !ruby/object:Gem::Requirement
165
153
  requirements:
166
154
  - - ">="
167
155
  - !ruby/object:Gem::Version
168
156
  version: '0'
169
157
  requirements: []
170
- rubygems_version: 3.5.22
171
- signing_key:
158
+ rubygems_version: 4.0.16
172
159
  specification_version: 4
173
160
  summary: Ruby headless Chrome driver
174
161
  test_files: []