ferrum 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +63 -1
  3. data/lib/ferrum/browser/binary.rb +1 -3
  4. data/lib/ferrum/browser/command.rb +0 -2
  5. data/lib/ferrum/browser/options/base.rb +5 -6
  6. data/lib/ferrum/browser/options/chrome.rb +0 -3
  7. data/lib/ferrum/browser/options.rb +33 -12
  8. data/lib/ferrum/browser/process/killer.rb +188 -0
  9. data/lib/ferrum/browser/process/output.rb +143 -0
  10. data/lib/ferrum/browser/process.rb +122 -144
  11. data/lib/ferrum/browser.rb +24 -30
  12. data/lib/ferrum/client/subscriber.rb +7 -2
  13. data/lib/ferrum/client/web_socket.rb +124 -43
  14. data/lib/ferrum/client.rb +36 -9
  15. data/lib/ferrum/console/location.rb +42 -0
  16. data/lib/ferrum/console/message.rb +230 -0
  17. data/lib/ferrum/console.rb +157 -0
  18. data/lib/ferrum/context.rb +41 -20
  19. data/lib/ferrum/contexts.rb +110 -28
  20. data/lib/ferrum/downloads.rb +2 -1
  21. data/lib/ferrum/errors.rb +50 -4
  22. data/lib/ferrum/frame/dom.rb +100 -45
  23. data/lib/ferrum/frame/runtime/legacy.rb +336 -0
  24. data/lib/ferrum/frame/runtime.rb +417 -186
  25. data/lib/ferrum/frame/script.rb +289 -0
  26. data/lib/ferrum/frame.rb +33 -4
  27. data/lib/ferrum/interceptable.rb +35 -4
  28. data/lib/ferrum/isolated_world.rb +94 -0
  29. data/lib/ferrum/keyboard.rb +95 -8
  30. data/lib/ferrum/log/annotated.rb +77 -0
  31. data/lib/ferrum/log/default.rb +55 -0
  32. data/lib/ferrum/log/json.rb +32 -0
  33. data/lib/ferrum/log/paint.rb +28 -0
  34. data/lib/ferrum/log.rb +247 -0
  35. data/lib/ferrum/mouse.rb +22 -8
  36. data/lib/ferrum/node.rb +127 -52
  37. data/lib/ferrum/page/frames.rb +10 -3
  38. data/lib/ferrum/page/screenshot.rb +11 -10
  39. data/lib/ferrum/page/stream.rb +16 -1
  40. data/lib/ferrum/page.rb +117 -58
  41. data/lib/ferrum/proxy.rb +8 -7
  42. data/lib/ferrum/remote_object.rb +75 -0
  43. data/lib/ferrum/target.rb +14 -3
  44. data/lib/ferrum/utils/deprecate.rb +77 -0
  45. data/lib/ferrum/utils/thread.rb +3 -1
  46. data/lib/ferrum/version.rb +1 -1
  47. data/lib/ferrum/worker.rb +8 -0
  48. data/lib/ferrum.rb +4 -1
  49. metadata +18 -4
  50. data/lib/ferrum/browser/options/firefox.rb +0 -70
@@ -7,10 +7,12 @@ require "tmpdir"
7
7
  require "forwardable"
8
8
  require "ferrum/browser/options/base"
9
9
  require "ferrum/browser/options/chrome"
10
- require "ferrum/browser/options/firefox"
10
+ require "ferrum/browser/process/killer"
11
+ require "ferrum/browser/process/output"
11
12
  require "ferrum/browser/command"
12
13
  require "ferrum/utils/elapsed_time"
13
14
  require "ferrum/utils/platform"
15
+ require "ferrum/utils/thread"
14
16
 
15
17
  module Ferrum
16
18
  class Browser
@@ -21,11 +23,13 @@ module Ferrum
21
23
  # stopping/restarting the process and cleaning up its user data directory.
22
24
  #
23
25
  class Process
24
- KILL_TIMEOUT = 2
25
- WAIT_KILLED = 0.05
26
-
27
26
  extend Forwardable
28
27
 
28
+ HTTP_SCHEMES = { "ws" => "http", "wss" => "https" }.freeze
29
+ WS_SCHEMES = { "http" => "ws", "https" => "wss" }.freeze
30
+ LOCAL_HOSTS = %w[0.0.0.0 :: 127.0.0.1 ::1 localhost].freeze
31
+ WS_URL = %r{DevTools listening on (ws://.*[a-zA-Z0-9-]{36})}
32
+
29
33
  delegate path: :command
30
34
 
31
35
  #
@@ -40,77 +44,30 @@ module Ferrum
40
44
  new(*args).tap(&:start)
41
45
  end
42
46
 
43
- #
44
- # Builds a finalizer proc that kills the process with the given pid.
45
- #
46
- # @param [Integer] pid
47
- # Process id to kill.
48
- #
49
- # @return [Proc]
50
- #
51
- def self.process_killer(pid)
52
- proc do
53
- if Utils::Platform.windows?
54
- # Process.kill is unreliable on Windows
55
- ::Process.kill("KILL", pid) unless system("taskkill /f /t /pid #{pid} >NUL 2>NUL")
56
- else
57
- ::Process.kill("USR1", pid)
58
- start = Utils::ElapsedTime.monotonic_time
59
- while ::Process.wait(pid, ::Process::WNOHANG).nil?
60
- sleep(WAIT_KILLED)
61
- next unless Utils::ElapsedTime.timeout?(start, KILL_TIMEOUT)
62
-
63
- ::Process.kill("KILL", pid)
64
- ::Process.wait(pid)
65
- break
66
- end
67
- end
68
- rescue Errno::ESRCH, Errno::ECHILD
69
- # nop
70
- end
71
- end
47
+ attr_reader :host, :port, :ws_url, :pid, :command, :xvfb
72
48
 
49
+ # What the browser process wrote to stdout and stderr, empty for a
50
+ # browser Ferrum didn't launch.
73
51
  #
74
- # Builds a finalizer proc that removes the directory at the given path.
75
- #
76
- # @param [String] path
77
- # Directory to remove.
78
- #
79
- # @return [Proc]
80
- #
81
- def self.directory_remover(path)
82
- proc {
83
- begin
84
- FileUtils.remove_entry(path)
85
- rescue StandardError
86
- Errno::ENOENT
87
- end
88
- }
89
- end
90
-
91
- attr_reader :host, :port, :ws_url, :pid, :command,
92
- :default_user_agent, :browser_version, :protocol_version,
93
- :v8_version, :webkit_version, :xvfb
52
+ # @return [Output]
53
+ attr_reader :output
94
54
 
95
55
  def initialize(options)
96
56
  @pid = @xvfb = @user_data_dir = nil
57
+ @requested_host = options.host
58
+ @log = options.log
59
+ @output = Output.new(@log)
97
60
 
98
61
  if options.ws_url || options.url
99
- # `:ws_url` option is higher priority than `:url`, parse versions
100
- # and use it as a ws_url, otherwise use what has been parsed.
101
- response = parse_json_version(options.ws_url || options.url)
102
- self.ws_url = options.ws_url || response&.[]("webSocketDebuggerUrl")
62
+ self.ws_url = options.ws_url || fetch_ws_url(options.url)
103
63
  return
104
64
  end
105
65
 
106
- @logger = options.logger
107
66
  @process_timeout = options.process_timeout
108
67
  @env = Hash(options.env)
109
68
 
110
- tmpdir = Dir.mktmpdir("ferrum_user_data_dir_")
111
- ObjectSpace.define_finalizer(self, self.class.directory_remover(tmpdir))
112
- @user_data_dir = tmpdir
113
- @command = Command.build(options, tmpdir)
69
+ @user_data_dir = Dir.mktmpdir("ferrum_user_data_dir_")
70
+ @command = Command.build(options, @user_data_dir)
114
71
  end
115
72
 
116
73
  #
@@ -123,24 +80,13 @@ module Ferrum
123
80
  return if ws_url
124
81
 
125
82
  begin
126
- read_io, write_io = IO.pipe
127
- process_options = { in: File::NULL }
128
- process_options[:pgroup] = true unless Utils::Platform.windows?
129
- process_options[:out] = process_options[:err] = write_io
130
-
131
- if @command.xvfb?
132
- @xvfb = Xvfb.start(@command.options)
133
- ObjectSpace.define_finalizer(self, self.class.process_killer(@xvfb.pid))
134
- end
135
-
136
- env = Hash(@xvfb&.to_env).merge(@env)
137
- @pid = ::Process.spawn(env, *@command.to_a, process_options)
138
- ObjectSpace.define_finalizer(self, self.class.process_killer(@pid))
139
-
140
- parse_ws_url(read_io, @process_timeout)
141
- parse_json_version(ws_url)
83
+ @xvfb = Xvfb.start(@command.options) if @command.xvfb?
84
+ @output = Output.new(@log)
85
+ @output.pipe { |io| spawn_browser(io) }
86
+ parse_ws_url
87
+ @output.follow
142
88
  ensure
143
- close_io(read_io, write_io)
89
+ @output.close unless ws_url
144
90
  end
145
91
  end
146
92
 
@@ -148,17 +94,26 @@ module Ferrum
148
94
  # Kills the browser process (and Xvfb, if running) and removes the user
149
95
  # data directory.
150
96
  #
151
- # @return [void]
152
- #
153
- def stop
154
- if @pid
155
- kill(@pid)
156
- kill(@xvfb.pid) if @xvfb&.pid
157
- @pid = nil
158
- end
159
-
160
- remove_user_data_dir if @user_data_dir
161
- ObjectSpace.undefine_finalizer(self)
97
+ # @param [Boolean] wait
98
+ # Whether to block until the process is confirmed dead and its user
99
+ # data directory removed (the default), or return immediately and
100
+ # run that cleanup on a background thread instead. Killing a
101
+ # stubborn process group can block for up to {Killer::KILL_TIMEOUT}
102
+ # seconds, plus retries removing its directory, which matters when quitting
103
+ # many browsers in a hot path. `wait: false` is not used by
104
+ # {#restart}, which always waits so the old process is fully gone
105
+ # before the new one starts.
106
+ #
107
+ # @return [Thread, nil]
108
+ # The background cleanup thread when `wait: false`; the caller can
109
+ # `#join` it if they need cleanup to have finished, e.g. before
110
+ # process exit or before reusing a fixed port. `nil` when `wait:
111
+ # true`.
112
+ #
113
+ def stop(wait: true)
114
+ return sync_stop if wait
115
+
116
+ async_stop
162
117
  end
163
118
 
164
119
  #
@@ -180,46 +135,65 @@ module Ferrum
180
135
  "#<#{self.class} " \
181
136
  "@user_data_dir=#{@user_data_dir.inspect} " \
182
137
  "@command=#<#{@command.class}:#{@command.object_id}> " \
183
- "@default_user_agent=#{@default_user_agent.inspect} " \
184
- "@ws_url=#{@ws_url.inspect} " \
185
- "@v8_version=#{@v8_version.inspect} " \
186
- "@browser_version=#{@browser_version.inspect} " \
187
- "@webkit_version=#{@webkit_version.inspect}>"
138
+ "@ws_url=#{@ws_url.inspect}>"
188
139
  end
189
140
 
190
141
  private
191
142
 
192
- def kill(pid)
193
- self.class.process_killer(pid).call
143
+ def spawn_browser(io)
144
+ process_options = { in: File::NULL, out: io, err: io }
145
+ process_options[:pgroup] = true unless Utils::Platform.windows?
146
+ env = Hash(@xvfb&.to_env).merge(@env)
147
+
148
+ @pid = ::Process.spawn(env, *@command.to_a, process_options)
149
+ ObjectSpace.define_finalizer(self, Killer.finalizer([@pid, @xvfb&.pid], @user_data_dir))
194
150
  end
195
151
 
196
- def remove_user_data_dir
197
- self.class.directory_remover(@user_data_dir).call
198
- @user_data_dir = nil
152
+ def sync_stop
153
+ if @pid
154
+ Killer.kill(@pid)
155
+ Killer.kill(@xvfb.pid) if @xvfb&.pid
156
+ @pid = nil
157
+ end
158
+
159
+ @output.close
160
+ remove_user_data_dir if @user_data_dir
161
+ ObjectSpace.undefine_finalizer(self)
162
+ nil
199
163
  end
200
164
 
201
- def parse_ws_url(read_io, timeout)
202
- output = ""
203
- start = Utils::ElapsedTime.monotonic_time
204
- max_time = start + timeout
205
- regexp = %r{DevTools listening on (ws://.*[a-zA-Z0-9-]{36})}
206
- while (now = Utils::ElapsedTime.monotonic_time) < max_time
207
- begin
208
- output += read_io.read_nonblock(512)
209
- rescue IO::WaitReadable
210
- read_io.wait_readable(max_time - now)
211
- else
212
- if output.match(regexp)
213
- self.ws_url = output.match(regexp)[1].strip
214
- break
215
- end
216
- end
165
+ #
166
+ # Snapshots what needs killing/removing, clears instance state so the
167
+ # object looks stopped right away, and does the actual work on a
168
+ # background thread. The finalizer is left in place as a backup until
169
+ # that thread finishes, in case the process exits before it does.
170
+ #
171
+ def async_stop
172
+ pid = @pid
173
+ xvfb_pid = @xvfb&.pid
174
+ user_data_dir = @user_data_dir
175
+ output = @output
176
+ @pid = @user_data_dir = nil
177
+
178
+ Utils::Thread.spawn do
179
+ Killer.kill(pid) if pid
180
+ Killer.kill(xvfb_pid) if xvfb_pid
181
+ output.close
182
+ Killer.remove_directory(user_data_dir) if user_data_dir
183
+ ObjectSpace.undefine_finalizer(self)
217
184
  end
185
+ end
218
186
 
219
- return if ws_url
187
+ def remove_user_data_dir
188
+ Killer.remove_directory(@user_data_dir)
189
+ @user_data_dir = nil
190
+ end
220
191
 
221
- @logger&.puts(output)
222
- raise ProcessTimeoutError.new(timeout, output)
192
+ def parse_ws_url
193
+ match = @output.wait_for(WS_URL, @process_timeout)
194
+ raise ProcessTimeoutError.new(@process_timeout, @output.to_s) unless match
195
+
196
+ self.ws_url = rewrite_host(match[1].strip)
223
197
  end
224
198
 
225
199
  def ws_url=(url)
@@ -228,37 +202,41 @@ module Ferrum
228
202
  @port = @ws_url.port
229
203
  end
230
204
 
231
- def close_io(*ios)
232
- ios.each do |io|
233
- io.close if io && !io.closed?
234
- rescue IOError
235
- raise unless RUBY_ENGINE == "jruby"
236
- end
205
+ def rewrite_host(url)
206
+ uri = Addressable::URI.parse(url)
207
+ uri.host = @requested_host
208
+ uri.to_s
237
209
  end
238
210
 
239
- def parse_json_version(url)
240
- url = URI.join(url, "/json/version")
241
-
242
- if %w[wss ws].include?(url.scheme)
243
- url.scheme = case url.scheme
244
- when "ws"
245
- "http"
246
- when "wss"
247
- "https"
248
- end
211
+ def fetch_ws_url(url)
212
+ response = ::Net::HTTP.get_response(URI(json_version_url(url).to_s))
213
+ status = "#{response.code} #{response.message}" unless response.is_a?(::Net::HTTPSuccess)
214
+ ws_url = begin
215
+ JSON.parse(response.body.to_s)["webSocketDebuggerUrl"]
216
+ rescue JSON::ParserError
217
+ nil
249
218
  end
250
219
 
251
- response = JSON.parse(::Net::HTTP.get(URI(url.to_s)))
220
+ reachable_ws_url(ws_url, url) || raise(NoWebSocketUrlError.new(url, status))
221
+ end
222
+
223
+ def json_version_url(url)
224
+ url = Addressable::URI.parse(url).dup
225
+ url.scheme = HTTP_SCHEMES.fetch(url.scheme, url.scheme)
226
+ url.path = "/json/version"
227
+ url
228
+ end
229
+
230
+ def reachable_ws_url(ws_url, endpoint)
231
+ return unless ws_url
252
232
 
253
- @v8_version = response["V8-Version"]
254
- @browser_version = response["Browser"]
255
- @webkit_version = response["WebKit-Version"]
256
- @default_user_agent = response["User-Agent"]
257
- @protocol_version = response["Protocol-Version"]
233
+ ws_url = Addressable::URI.parse(ws_url)
234
+ return ws_url.to_s unless LOCAL_HOSTS.include?(ws_url.host)
258
235
 
259
- response
260
- rescue JSON::ParserError
261
- # nop
236
+ url = Addressable::URI.parse(endpoint).dup
237
+ url.scheme = WS_SCHEMES.fetch(url.scheme, url.scheme)
238
+ url.path = ws_url.path
239
+ url.to_s
262
240
  end
263
241
  end
264
242
  end
@@ -29,15 +29,18 @@ module Ferrum
29
29
  delegate %i[default_context] => :contexts
30
30
  delegate %i[targets create_target page pages windows workers service_workers attach_target] => :default_context
31
31
  delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
32
- at_css at_xpath css xpath current_url current_title url title
32
+ wait_for_navigation wait_for_navigation!
33
+ at_css at_xpath css xpath wait_for_selector current_url current_title url title
33
34
  body doctype content=
34
- headers cookies network accessibility downloads
35
+ headers cookies network console accessibility downloads
35
36
  mouse keyboard
36
37
  screenshot pdf mhtml viewport_size device_pixel_ratio
37
38
  start_screencast stop_screencast
38
39
  frames frame_by main_frame
39
- evaluate evaluate_on evaluate_async execute evaluate_func
40
+ evaluate evaluate_handle execute
41
+ evaluate_on evaluate_async evaluate_func isolated_world
40
42
  add_script_tag add_style_tag bypass_csp
43
+ evaluate_on_new_document remove_script_on_new_document
41
44
  on position position=
42
45
  playback_rate playback_rate=
43
46
  disable_javascript set_viewport resize] => :page
@@ -45,7 +48,7 @@ module Ferrum
45
48
  attr_reader :client, :process, :contexts, :options
46
49
 
47
50
  delegate %i[timeout timeout= protocol_timeout protocol_timeout=
48
- base_url base_url= default_user_agent default_user_agent= extensions] => :options
51
+ base_url base_url= default_user_agent default_user_agent= extensions add_extension] => :options
49
52
  delegate %i[command] => :client
50
53
 
51
54
  #
@@ -77,8 +80,8 @@ module Ferrum
77
80
  # An array of paths to files or JS source code to be preloaded into the
78
81
  # browser e.g.: `["/path/to/script.js", { source: "window.secret = 'top'" }]`
79
82
  #
80
- # @option options [#puts] :logger
81
- # When present, debug output is written to this object.
83
+ # @option options [#puts, Logger] :logger
84
+ # When present, debug output is written to this object, see {Log}.
82
85
  #
83
86
  # @option options [Integer, Float] :slowmo
84
87
  # Set a delay in seconds to wait before sending a command.
@@ -103,9 +106,8 @@ module Ferrum
103
106
  # why you have slow responses and fix or block them rather than turn this
104
107
  # setting off.
105
108
  #
106
- # @option options [:chrome, :firefox] :browser_name (:chrome)
107
- # Sets the browser's name. **Note:** only experimental support for
108
- # `:firefox` for now.
109
+ # @option options [:chrome, :opera, :edge] :browser_name (:chrome)
110
+ # Sets the browser's name.
109
111
  #
110
112
  # @option options [String] :browser_path
111
113
  # Path to Chrome binary, you can also set ENV variable as
@@ -192,23 +194,6 @@ module Ferrum
192
194
  end
193
195
  end
194
196
 
195
- #
196
- # Evaluate JavaScript to modify things before a page load.
197
- #
198
- # @param [String] expression
199
- # The JavaScript to add to each new document.
200
- #
201
- # @example
202
- # browser.evaluate_on_new_document <<~JS
203
- # Object.defineProperty(navigator, "languages", {
204
- # get: function() { return ["tlh"]; }
205
- # });
206
- # JS
207
- #
208
- def evaluate_on_new_document(expression)
209
- extensions << expression
210
- end
211
-
212
197
  #
213
198
  # Closes browser tabs opened by the `Browser` instance.
214
199
  #
@@ -238,14 +223,23 @@ module Ferrum
238
223
  #
239
224
  # Terminates the browser process and closes the client connection.
240
225
  #
241
- def quit
226
+ # @param [Boolean] wait
227
+ # Whether to block until the process is confirmed dead and its user
228
+ # data directory removed (the default), or return immediately and run
229
+ # that cleanup on a background thread instead. See {Process#stop}.
230
+ #
231
+ # @return [Thread, nil]
232
+ # The background cleanup thread when `wait: false`, `nil` otherwise.
233
+ #
234
+ def quit(wait: true)
242
235
  return unless @client
243
236
 
244
- contexts.close_connections
237
+ contexts&.close_connections
245
238
 
246
239
  @client.close
247
- @process.stop
240
+ thread = @process.stop(wait: wait)
248
241
  @client = @process = @contexts = nil
242
+ thread
249
243
  end
250
244
 
251
245
  #
@@ -301,9 +295,9 @@ module Ferrum
301
295
 
302
296
  begin
303
297
  @process.start
304
- @options.default_user_agent = @process.default_user_agent
305
298
 
306
299
  @client = Client.new(@process.ws_url, options)
300
+ @options.default_user_agent = version.user_agent
307
301
  @contexts = Contexts.new(@client)
308
302
  rescue StandardError
309
303
  @process.stop
@@ -106,7 +106,7 @@ module Ferrum
106
106
  private
107
107
 
108
108
  def start
109
- @regular_thread = Utils::Thread.spawn(abort_on_exception: false) do
109
+ @regular_thread = Utils::Thread.spawn do
110
110
  loop do
111
111
  message = @regular.pop
112
112
  break unless message
@@ -115,7 +115,7 @@ module Ferrum
115
115
  end
116
116
  end
117
117
 
118
- @priority_thread = Utils::Thread.spawn(abort_on_exception: false) do
118
+ @priority_thread = Utils::Thread.spawn do
119
119
  loop do
120
120
  message = @priority.pop
121
121
  break unless message
@@ -133,6 +133,11 @@ module Ferrum
133
133
  @on[event]&.each_with_index do |block, index|
134
134
  # In case of multiple callbacks we provide current index and total
135
135
  block.call(params, index, total)
136
+ rescue StandardError => e
137
+ # A raising callback used to terminate the dispatch thread, and with it every
138
+ # event for the rest of the browser's life: targets stopped being registered
139
+ # and any page created later raised `NoSuchTargetError`. Report and carry on.
140
+ warn("Ferrum: #{event} callback raised #{e.class}: #{e.message}\n #{e.backtrace&.first}")
136
141
  end
137
142
  end
138
143
  end