ferrum 0.19.2 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +60 -0
  3. data/lib/ferrum/browser/binary.rb +1 -2
  4. data/lib/ferrum/browser/options.rb +33 -12
  5. data/lib/ferrum/browser/process/output.rb +143 -0
  6. data/lib/ferrum/browser/process.rb +42 -69
  7. data/lib/ferrum/browser.rb +9 -23
  8. data/lib/ferrum/client/web_socket.rb +4 -15
  9. data/lib/ferrum/client.rb +1 -1
  10. data/lib/ferrum/console/location.rb +42 -0
  11. data/lib/ferrum/console/message.rb +230 -0
  12. data/lib/ferrum/console.rb +157 -0
  13. data/lib/ferrum/context.rb +23 -15
  14. data/lib/ferrum/contexts.rb +41 -18
  15. data/lib/ferrum/errors.rb +7 -5
  16. data/lib/ferrum/frame/dom.rb +50 -44
  17. data/lib/ferrum/frame/runtime/legacy.rb +336 -0
  18. data/lib/ferrum/frame/runtime.rb +416 -186
  19. data/lib/ferrum/frame/script.rb +289 -0
  20. data/lib/ferrum/frame.rb +33 -4
  21. data/lib/ferrum/interceptable.rb +19 -9
  22. data/lib/ferrum/isolated_world.rb +94 -0
  23. data/lib/ferrum/keyboard.rb +95 -8
  24. data/lib/ferrum/log/annotated.rb +77 -0
  25. data/lib/ferrum/log/default.rb +55 -0
  26. data/lib/ferrum/log/json.rb +32 -0
  27. data/lib/ferrum/log/paint.rb +28 -0
  28. data/lib/ferrum/log.rb +247 -0
  29. data/lib/ferrum/mouse.rb +16 -8
  30. data/lib/ferrum/node.rb +127 -52
  31. data/lib/ferrum/page/frames.rb +11 -3
  32. data/lib/ferrum/page/screenshot.rb +8 -7
  33. data/lib/ferrum/page.rb +112 -61
  34. data/lib/ferrum/proxy.rb +8 -7
  35. data/lib/ferrum/remote_object.rb +75 -0
  36. data/lib/ferrum/target.rb +7 -0
  37. data/lib/ferrum/utils/deprecate.rb +77 -0
  38. data/lib/ferrum/version.rb +1 -1
  39. data/lib/ferrum/worker.rb +8 -0
  40. data/lib/ferrum.rb +3 -0
  41. metadata +16 -2
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b44af007f48e698ad75285cccfcb44da6254de2cb5e9b94965691130ae2ec8e0
4
- data.tar.gz: 576166fff45f907937bcc23e28a202a101f1d0c16e55772bbf50ec239ab5f627
3
+ metadata.gz: 1429a2443d8b287c0bdb7db1589c702e2d808acce61d6341cb7f29c3de84dcfd
4
+ data.tar.gz: a48d3e136ebbf422b5efdbbe5516f7b93d33150158e337c32afc92c55db26bb7
5
5
  SHA512:
6
- metadata.gz: 2951cc2f1e098ddf0ea6df16d4abd68936548667fc191f0d828955c7bcbb719d4d00c260ba4b9941906387b66cef42d6d4cf5e625bb678d2460555c8c1c21ebe
7
- data.tar.gz: 0fd32129fe61aa693163d41b3986dba56ad7a3729e852e96bf78d9b49cc0a820707cb0cd77c6c91a84faeb1c6e042dca7115144de6da72d1e7d9a52da41c6dec
6
+ metadata.gz: c927ede51c9c40a54918410fb217b5f7a26e4a1d39574268443fee0cb9966f1dd7d00508d892fb2190b3cf6b23b4c0f05f7ad5221755414edc8c105dd2083511
7
+ data.tar.gz: afbde35727cad16c84bce0420d6c5e26244bb9efeee9fef7e6324534cfc7e2a3b878016520dc697788976ab4e4b595feffebda6acc6fe71f50153e9a67185ae3
data/README.md CHANGED
@@ -35,6 +35,17 @@ Add this to your `Gemfile` and run `bundle install`.
35
35
  gem "ferrum"
36
36
  ```
37
37
 
38
+ Or skip installing Chrome and run it in a container with our images, see
39
+ [Docker](https://docs.rubycdp.com/docs/ferrum/docker#our-images-recommended):
40
+
41
+ ```shell
42
+ docker run --rm -p 3000:3000 ghcr.io/rubycdp/chromium
43
+ ```
44
+
45
+ ```ruby
46
+ browser = Ferrum::Browser.new(url: "http://localhost:3000")
47
+ ```
48
+
38
49
  ## Quick Start
39
50
 
40
51
  Navigate to a website and save a screenshot:
@@ -74,6 +85,55 @@ JS
74
85
  browser.quit
75
86
  ```
76
87
 
88
+ Pass arguments by name, so the script can use parameters instead of digging
89
+ through `arguments[0]`:
90
+
91
+ ```ruby
92
+ page.evaluate("a + b", a: 1, b: 2) # => 3
93
+ page.evaluate("el.getAttribute(name)", el: page.at_css("a"), name: "href")
94
+ ```
95
+
96
+ A script that is a function or arrow declaration is used as-is and takes its
97
+ arguments positionally, in the order it declares them:
98
+
99
+ ```ruby
100
+ page.evaluate(<<~JS, 1, 2)
101
+ function(a, b) {
102
+ const sum = a + b;
103
+ return sum * 3;
104
+ }
105
+ JS
106
+ ```
107
+
108
+ `await` works directly, and a function declaration's returned promise is
109
+ awaited:
110
+
111
+ ```ruby
112
+ page.evaluate("await fetch(url).then(r => r.text())", url: "/api")
113
+ ```
114
+
115
+ Pass `timeout:` (seconds, defaulting to the page timeout) to bound how long the
116
+ script may take before `Ferrum::ScriptTimeoutError` is raised. `timeout:` is the
117
+ only reserved keyword.
118
+
119
+ Use `#execute` when you only want the side effects, `#evaluate_handle` when you
120
+ want to keep a value in the browser and pass it back in later, and
121
+ `Ferrum::Node#evaluate` to run a script with `this` bound to an element:
122
+
123
+ ```ruby
124
+ page.execute("window.scrollBy(0, 100)") # => true
125
+
126
+ list = page.evaluate_handle("document.querySelectorAll('li')")
127
+ page.evaluate("Array.from(nodes).map(n => n.textContent)", nodes: list)
128
+
129
+ page.at_css("input").evaluate("this.value")
130
+ ```
131
+
132
+ Scripts written for the old API keep working, and Ferrum warns, with a suggested
133
+ rewrite, about any whose result will change. The next release, 0.21, removes the
134
+ old behavior, so run your suite with `FERRUM_JS_API=new` to check it's ready. See
135
+ [upgrading to 0.21](docs/13-javascript.md#upgrading-to-021).
136
+
77
137
  Do any mouse movements you like:
78
138
 
79
139
  ```ruby
@@ -68,7 +68,6 @@ module Ferrum
68
68
  [paths, exts]
69
69
  end
70
70
 
71
- # rubocop:disable-next Style/CollectionCompact
72
71
  def lazy_find(cmds)
73
72
  cmds.lazy.map do |cmd, path, ext|
74
73
  absolute_path = File.absolute_path(cmd)
@@ -79,7 +78,7 @@ module Ferrum
79
78
  next if File.directory?(cmd)
80
79
 
81
80
  cmd
82
- end.reject(&:nil?) # .compact isn't defined on Enumerator::Lazy
81
+ end.compact
83
82
  end
84
83
  end
85
84
  end
@@ -1,5 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "ferrum/log"
4
+
3
5
  module Ferrum
4
6
  class Browser
5
7
  #
@@ -16,13 +18,14 @@ module Ferrum
16
18
  DEFAULT_TIMEOUT = ENV.fetch("FERRUM_DEFAULT_TIMEOUT", 5).to_i
17
19
  DEFAULT_PROTOCOL_TIMEOUT = ENV.fetch("FERRUM_PROTOCOL_TIMEOUT", DEFAULT_TIMEOUT).to_i
18
20
  PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
19
- DEBUG_MODE = !ENV.fetch("FERRUM_DEBUG", nil).nil?
20
21
 
21
- attr_reader :window_size, :logger, :ws_max_receive_size,
22
+ warn "Ferrum: FERRUM_DEBUG is deprecated, use FERRUM_LOG=* instead" if ENV.key?("FERRUM_DEBUG")
23
+
24
+ attr_reader :window_size, :logger, :log, :ws_max_receive_size,
22
25
  :js_errors, :base_url, :slowmo, :pending_connection_errors,
23
26
  :url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
24
27
  :save_path, :proxy, :port, :host, :headless, :incognito, :dockerize, :browser_options,
25
- :ignore_default_browser_options, :xvfb, :flatten
28
+ :ignore_default_browser_options, :xvfb, :flatten, :extensions
26
29
  attr_accessor :timeout, :protocol_timeout, :default_user_agent
27
30
 
28
31
  def initialize(options = nil)
@@ -51,10 +54,11 @@ module Ferrum
51
54
  @ignore_default_browser_options = @options[:ignore_default_browser_options]
52
55
 
53
56
  @proxy = validate_proxy(@options[:proxy])
54
- @logger = parse_logger(@options[:logger])
57
+ @logger, @log = parse_logger(@options[:logger])
55
58
  @base_url = parse_base_url(@options[:base_url]) if @options[:base_url]
56
59
  @url = @options[:url].to_s if @options[:url]
57
60
  @ws_url = @options[:ws_url].to_s if @options[:ws_url]
61
+ @extensions = Array(@options[:extensions]).map { |e| read_extension(e) }.freeze
58
62
 
59
63
  @options = @options.merge(window_size: @window_size).freeze
60
64
  @browser_options = @options.fetch(:browser_options, {}).freeze
@@ -73,15 +77,24 @@ module Ferrum
73
77
  end
74
78
 
75
79
  #
76
- # JS source to preload into the browser, read from `:extensions` option.
80
+ # Adds an extension to run in every new document, before the document's own scripts. Applies to pages and
81
+ # iframes attached after the call: pages already open keep the extensions they were attached with.
82
+ #
83
+ # @param [String, Hash] extension
84
+ # A path to a JS file, or `{ source: "..." }`.
77
85
  #
78
86
  # @return [Array<String>]
79
- # JS source code for each configured extension.
87
+ # JS source code of every extension.
80
88
  #
81
- def extensions
82
- @extensions ||= Array(@options[:extensions]).map do |extension|
83
- (extension.is_a?(Hash) && extension[:source]) || File.read(extension)
84
- end
89
+ # @raise [Errno::ENOENT]
90
+ # If the file doesn't exist.
91
+ #
92
+ # @example
93
+ # browser.add_extension("/path/to/stub.js")
94
+ # browser.add_extension(source: "window.secret = 'top'")
95
+ #
96
+ def add_extension(extension)
97
+ @extensions = [*@extensions, read_extension(extension)].freeze
85
98
  end
86
99
 
87
100
  #
@@ -119,10 +132,18 @@ module Ferrum
119
132
 
120
133
  private
121
134
 
135
+ def read_extension(extension)
136
+ (extension.is_a?(Hash) && extension[:source]) || File.read(extension)
137
+ end
138
+
122
139
  def parse_logger(logger)
123
- return logger if logger
140
+ value = ENV.fetch("FERRUM_LOG") { "*" if ENV.key?("FERRUM_DEBUG") }
141
+ settings = value ? Log.parse(value) : {}
142
+ logger ||= $stdout.tap { |s| s.sync = true } if value && settings
143
+ return [logger, nil] unless logger && settings
124
144
 
125
- !logger && DEBUG_MODE ? $stdout.tap { |s| s.sync = true } : logger
145
+ [logger, Log.new(logger, **settings, format: Log.parse_format(ENV.fetch("FERRUM_LOG_FORMAT", nil)),
146
+ color: Log.parse_color(ENV.fetch("FERRUM_LOG_COLOR", nil)))]
126
147
  end
127
148
 
128
149
  def parse_base_url(value)
@@ -0,0 +1,143 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ferrum
4
+ class Browser
5
+ class Process
6
+ #
7
+ # What the browser process wrote to stdout and stderr: keeps the last
8
+ # {LIMIT} lines and writes each one to the `:browser` log channel. Owns
9
+ # the pipe the process writes to, which is read on its own thread for as
10
+ # long as the process lives, so it never fills up and blocks the browser.
11
+ #
12
+ class Output
13
+ LIMIT = 100
14
+ DRAIN_TIMEOUT = 0.5
15
+
16
+ def initialize(log = nil)
17
+ @log = log
18
+ @lines = []
19
+ @partial = String.new(encoding: Encoding::BINARY)
20
+ @mutex = Mutex.new
21
+ @io = @reader = nil
22
+ end
23
+
24
+ #
25
+ # Opens the pipe and yields its write end, to be given to the process
26
+ # as stdout and stderr. The write end is closed here afterwards, so
27
+ # reading hits the end of the pipe once the process exits.
28
+ #
29
+ # @yieldparam [IO] io
30
+ #
31
+ # @return [void]
32
+ #
33
+ def pipe
34
+ @io, writer = IO.pipe
35
+ yield writer
36
+ ensure
37
+ close_io(writer)
38
+ end
39
+
40
+ #
41
+ # Reads the pipe until the output matches, the process exits or the
42
+ # time is up.
43
+ #
44
+ # @param [Regexp] regexp
45
+ #
46
+ # @param [Numeric] timeout
47
+ #
48
+ # @return [MatchData, nil]
49
+ #
50
+ def wait_for(regexp, timeout)
51
+ deadline = Utils::ElapsedTime.monotonic_time + timeout
52
+ while (now = Utils::ElapsedTime.monotonic_time) < deadline
53
+ begin
54
+ self << @io.read_nonblock(512)
55
+ rescue IO::WaitReadable
56
+ @io.wait_readable(deadline - now)
57
+ rescue EOFError
58
+ return
59
+ else
60
+ match = to_s.match(regexp)
61
+ return match if match
62
+ end
63
+ end
64
+ end
65
+
66
+ #
67
+ # Keeps reading the pipe on a new thread until it's closed.
68
+ #
69
+ # @return [Thread]
70
+ #
71
+ def follow
72
+ @reader = Utils::Thread.spawn do
73
+ loop { self << @io.readpartial(4096) }
74
+ rescue IOError
75
+ self << "\n" unless @partial.empty?
76
+ end
77
+ end
78
+
79
+ #
80
+ # Gives the reading thread {DRAIN_TIMEOUT} seconds to take what the
81
+ # process wrote last, then closes the pipe. The lines are kept.
82
+ #
83
+ # @return [void]
84
+ #
85
+ def close
86
+ @reader&.join(DRAIN_TIMEOUT)
87
+ close_io(@io)
88
+ @io = @reader = nil
89
+ end
90
+
91
+ #
92
+ # Adds a chunk of output, which can end in the middle of a line.
93
+ #
94
+ # @param [String] chunk
95
+ #
96
+ # @return [self]
97
+ #
98
+ def <<(chunk)
99
+ lines = @mutex.synchronize do
100
+ *complete, @partial = (@partial + chunk.b).split("\n", -1)
101
+ complete.map! { |line| utf8(line) }
102
+ @lines.concat(complete)
103
+ @lines.shift(@lines.size - LIMIT) if @lines.size > LIMIT
104
+ complete
105
+ end
106
+
107
+ lines.each { |line| @log&.browser(line) }
108
+ self
109
+ end
110
+
111
+ #
112
+ # The last lines, oldest first.
113
+ #
114
+ # @return [Array<String>]
115
+ #
116
+ def lines
117
+ @mutex.synchronize { @partial.empty? ? @lines.dup : [*@lines, utf8(@partial)] }
118
+ end
119
+
120
+ #
121
+ # The last lines joined by newlines.
122
+ #
123
+ # @return [String]
124
+ #
125
+ def to_s
126
+ lines.join("\n")
127
+ end
128
+
129
+ private
130
+
131
+ def utf8(line)
132
+ line.dup.force_encoding(Encoding::UTF_8).scrub.delete_suffix("\r")
133
+ end
134
+
135
+ def close_io(io)
136
+ io.close if io && !io.closed?
137
+ rescue IOError
138
+ raise unless RUBY_ENGINE == "jruby"
139
+ end
140
+ end
141
+ end
142
+ end
143
+ end
@@ -8,6 +8,7 @@ require "forwardable"
8
8
  require "ferrum/browser/options/base"
9
9
  require "ferrum/browser/options/chrome"
10
10
  require "ferrum/browser/process/killer"
11
+ require "ferrum/browser/process/output"
11
12
  require "ferrum/browser/command"
12
13
  require "ferrum/utils/elapsed_time"
13
14
  require "ferrum/utils/platform"
@@ -27,6 +28,7 @@ module Ferrum
27
28
  HTTP_SCHEMES = { "ws" => "http", "wss" => "https" }.freeze
28
29
  WS_SCHEMES = { "http" => "ws", "https" => "wss" }.freeze
29
30
  LOCAL_HOSTS = %w[0.0.0.0 :: 127.0.0.1 ::1 localhost].freeze
31
+ WS_URL = %r{DevTools listening on (ws://.*[a-zA-Z0-9-]{36})}
30
32
 
31
33
  delegate path: :command
32
34
 
@@ -42,27 +44,25 @@ module Ferrum
42
44
  new(*args).tap(&:start)
43
45
  end
44
46
 
45
- attr_reader :host, :port, :ws_url, :pid, :command,
46
- :default_user_agent, :browser_version, :protocol_version,
47
- :v8_version, :webkit_version, :xvfb
47
+ attr_reader :host, :port, :ws_url, :pid, :command, :xvfb
48
+
49
+ # What the browser process wrote to stdout and stderr, empty for a
50
+ # browser Ferrum didn't launch.
51
+ #
52
+ # @return [Output]
53
+ attr_reader :output
48
54
 
49
55
  def initialize(options)
50
56
  @pid = @xvfb = @user_data_dir = nil
51
57
  @requested_host = options.host
58
+ @log = options.log
59
+ @output = Output.new(@log)
52
60
 
53
61
  if options.ws_url || options.url
54
- # `:ws_url` option is higher priority than `:url`, parse versions
55
- # and use it as a ws_url, otherwise use what has been parsed.
56
- endpoint = options.ws_url || options.url
57
- response = parse_json_version(endpoint)
58
- ws_url = options.ws_url || reachable_ws_url(response&.[]("webSocketDebuggerUrl"), endpoint)
59
- raise NoWebSocketUrlError, endpoint unless ws_url
60
-
61
- self.ws_url = ws_url
62
+ self.ws_url = options.ws_url || fetch_ws_url(options.url)
62
63
  return
63
64
  end
64
65
 
65
- @logger = options.logger
66
66
  @process_timeout = options.process_timeout
67
67
  @env = Hash(options.env)
68
68
 
@@ -80,21 +80,13 @@ module Ferrum
80
80
  return if ws_url
81
81
 
82
82
  begin
83
- read_io, write_io = IO.pipe
84
- process_options = { in: File::NULL }
85
- process_options[:pgroup] = true unless Utils::Platform.windows?
86
- process_options[:out] = process_options[:err] = write_io
87
-
88
83
  @xvfb = Xvfb.start(@command.options) if @command.xvfb?
89
-
90
- env = Hash(@xvfb&.to_env).merge(@env)
91
- @pid = ::Process.spawn(env, *@command.to_a, process_options)
92
- ObjectSpace.define_finalizer(self, Killer.finalizer([@pid, @xvfb&.pid], @user_data_dir))
93
-
94
- parse_ws_url(read_io, @process_timeout)
95
- parse_json_version(ws_url)
84
+ @output = Output.new(@log)
85
+ @output.pipe { |io| spawn_browser(io) }
86
+ parse_ws_url
87
+ @output.follow
96
88
  ensure
97
- close_io(read_io, write_io)
89
+ @output.close unless ws_url
98
90
  end
99
91
  end
100
92
 
@@ -143,15 +135,20 @@ module Ferrum
143
135
  "#<#{self.class} " \
144
136
  "@user_data_dir=#{@user_data_dir.inspect} " \
145
137
  "@command=#<#{@command.class}:#{@command.object_id}> " \
146
- "@default_user_agent=#{@default_user_agent.inspect} " \
147
- "@ws_url=#{@ws_url.inspect} " \
148
- "@v8_version=#{@v8_version.inspect} " \
149
- "@browser_version=#{@browser_version.inspect} " \
150
- "@webkit_version=#{@webkit_version.inspect}>"
138
+ "@ws_url=#{@ws_url.inspect}>"
151
139
  end
152
140
 
153
141
  private
154
142
 
143
+ def spawn_browser(io)
144
+ process_options = { in: File::NULL, out: io, err: io }
145
+ process_options[:pgroup] = true unless Utils::Platform.windows?
146
+ env = Hash(@xvfb&.to_env).merge(@env)
147
+
148
+ @pid = ::Process.spawn(env, *@command.to_a, process_options)
149
+ ObjectSpace.define_finalizer(self, Killer.finalizer([@pid, @xvfb&.pid], @user_data_dir))
150
+ end
151
+
155
152
  def sync_stop
156
153
  if @pid
157
154
  Killer.kill(@pid)
@@ -159,6 +156,7 @@ module Ferrum
159
156
  @pid = nil
160
157
  end
161
158
 
159
+ @output.close
162
160
  remove_user_data_dir if @user_data_dir
163
161
  ObjectSpace.undefine_finalizer(self)
164
162
  nil
@@ -174,11 +172,13 @@ module Ferrum
174
172
  pid = @pid
175
173
  xvfb_pid = @xvfb&.pid
176
174
  user_data_dir = @user_data_dir
175
+ output = @output
177
176
  @pid = @user_data_dir = nil
178
177
 
179
178
  Utils::Thread.spawn do
180
179
  Killer.kill(pid) if pid
181
180
  Killer.kill(xvfb_pid) if xvfb_pid
181
+ output.close
182
182
  Killer.remove_directory(user_data_dir) if user_data_dir
183
183
  ObjectSpace.undefine_finalizer(self)
184
184
  end
@@ -189,28 +189,11 @@ module Ferrum
189
189
  @user_data_dir = nil
190
190
  end
191
191
 
192
- def parse_ws_url(read_io, timeout)
193
- output = ""
194
- start = Utils::ElapsedTime.monotonic_time
195
- max_time = start + timeout
196
- regexp = %r{DevTools listening on (ws://.*[a-zA-Z0-9-]{36})}
197
- while (now = Utils::ElapsedTime.monotonic_time) < max_time
198
- begin
199
- output += read_io.read_nonblock(512)
200
- rescue IO::WaitReadable
201
- read_io.wait_readable(max_time - now)
202
- else
203
- if output.match(regexp)
204
- self.ws_url = rewrite_host(output.match(regexp)[1].strip)
205
- break
206
- end
207
- end
208
- end
209
-
210
- return if ws_url
192
+ def parse_ws_url
193
+ match = @output.wait_for(WS_URL, @process_timeout)
194
+ raise ProcessTimeoutError.new(@process_timeout, @output.to_s) unless match
211
195
 
212
- @logger&.puts(output)
213
- raise ProcessTimeoutError.new(timeout, output)
196
+ self.ws_url = rewrite_host(match[1].strip)
214
197
  end
215
198
 
216
199
  def ws_url=(url)
@@ -225,26 +208,16 @@ module Ferrum
225
208
  uri.to_s
226
209
  end
227
210
 
228
- def close_io(*ios)
229
- ios.each do |io|
230
- io.close if io && !io.closed?
231
- rescue IOError
232
- raise unless RUBY_ENGINE == "jruby"
211
+ def fetch_ws_url(url)
212
+ response = ::Net::HTTP.get_response(URI(json_version_url(url).to_s))
213
+ status = "#{response.code} #{response.message}" unless response.is_a?(::Net::HTTPSuccess)
214
+ ws_url = begin
215
+ JSON.parse(response.body.to_s)["webSocketDebuggerUrl"]
216
+ rescue JSON::ParserError
217
+ nil
233
218
  end
234
- end
235
-
236
- def parse_json_version(url)
237
- response = JSON.parse(::Net::HTTP.get(URI(json_version_url(url).to_s)))
238
-
239
- @v8_version = response["V8-Version"]
240
- @browser_version = response["Browser"]
241
- @webkit_version = response["WebKit-Version"]
242
- @default_user_agent = response["User-Agent"]
243
- @protocol_version = response["Protocol-Version"]
244
219
 
245
- response
246
- rescue JSON::ParserError
247
- # nop
220
+ reachable_ws_url(ws_url, url) || raise(NoWebSocketUrlError.new(url, status))
248
221
  end
249
222
 
250
223
  def json_version_url(url)
@@ -29,15 +29,18 @@ module Ferrum
29
29
  delegate %i[default_context] => :contexts
30
30
  delegate %i[targets create_target page pages windows workers service_workers attach_target] => :default_context
31
31
  delegate %i[go_to goto go back forward refresh reload stop wait_for_reload
32
+ wait_for_navigation wait_for_navigation!
32
33
  at_css at_xpath css xpath wait_for_selector current_url current_title url title
33
34
  body doctype content=
34
- headers cookies network accessibility downloads
35
+ headers cookies network console accessibility downloads
35
36
  mouse keyboard
36
37
  screenshot pdf mhtml viewport_size device_pixel_ratio
37
38
  start_screencast stop_screencast
38
39
  frames frame_by main_frame
39
- evaluate evaluate_on evaluate_async execute evaluate_func
40
+ evaluate evaluate_handle execute
41
+ evaluate_on evaluate_async evaluate_func isolated_world
40
42
  add_script_tag add_style_tag bypass_csp
43
+ evaluate_on_new_document remove_script_on_new_document
41
44
  on position position=
42
45
  playback_rate playback_rate=
43
46
  disable_javascript set_viewport resize] => :page
@@ -45,7 +48,7 @@ module Ferrum
45
48
  attr_reader :client, :process, :contexts, :options
46
49
 
47
50
  delegate %i[timeout timeout= protocol_timeout protocol_timeout=
48
- base_url base_url= default_user_agent default_user_agent= extensions] => :options
51
+ base_url base_url= default_user_agent default_user_agent= extensions add_extension] => :options
49
52
  delegate %i[command] => :client
50
53
 
51
54
  #
@@ -77,8 +80,8 @@ module Ferrum
77
80
  # An array of paths to files or JS source code to be preloaded into the
78
81
  # browser e.g.: `["/path/to/script.js", { source: "window.secret = 'top'" }]`
79
82
  #
80
- # @option options [#puts] :logger
81
- # When present, debug output is written to this object.
83
+ # @option options [#puts, Logger] :logger
84
+ # When present, debug output is written to this object, see {Log}.
82
85
  #
83
86
  # @option options [Integer, Float] :slowmo
84
87
  # Set a delay in seconds to wait before sending a command.
@@ -191,23 +194,6 @@ module Ferrum
191
194
  end
192
195
  end
193
196
 
194
- #
195
- # Evaluate JavaScript to modify things before a page load.
196
- #
197
- # @param [String] expression
198
- # The JavaScript to add to each new document.
199
- #
200
- # @example
201
- # browser.evaluate_on_new_document <<~JS
202
- # Object.defineProperty(navigator, "languages", {
203
- # get: function() { return ["tlh"]; }
204
- # });
205
- # JS
206
- #
207
- def evaluate_on_new_document(expression)
208
- extensions << expression
209
- end
210
-
211
197
  #
212
198
  # Closes browser tabs opened by the `Browser` instance.
213
199
  #
@@ -309,9 +295,9 @@ module Ferrum
309
295
 
310
296
  begin
311
297
  @process.start
312
- @options.default_user_agent = @process.default_user_agent
313
298
 
314
299
  @client = Client.new(@process.ws_url, options)
300
+ @options.default_user_agent = version.user_agent
315
301
  @contexts = Contexts.new(@client)
316
302
  rescue StandardError
317
303
  @process.stop
@@ -24,7 +24,6 @@ module Ferrum
24
24
  class WebSocket
25
25
  WEBSOCKET_BUG_SLEEP = 0.05
26
26
  DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
27
- SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
28
27
  WRITE_CHUNK_SIZE = 64 * 1024
29
28
  ESCAPE_OR_LONE_SURROGATE = /\\u[dD][89abAB]\h{2}\\u[dD][c-fC-F]\h{2}|\\u([dD][89a-fA-F]\h{2})|\\./
30
29
 
@@ -34,9 +33,9 @@ module Ferrum
34
33
  # Why Ferrum closed the connection itself, `nil` if it didn't.
35
34
  attr_reader :close_reason
36
35
 
37
- def initialize(url, max_receive_size, logger, write_timeout = nil)
36
+ def initialize(url, max_receive_size, log, write_timeout = nil)
38
37
  @url = url
39
- @logger = logger
38
+ @log = log
40
39
  @write_timeout = write_timeout
41
40
  uri = URI.parse(@url)
42
41
  port = uri.port || DEFAULT_PORTS[uri.scheme]
@@ -60,8 +59,6 @@ module Ferrum
60
59
  @messages = Queue.new
61
60
  @outgoing = Queue.new
62
61
 
63
- @screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
64
-
65
62
  @driver.on(:open, &method(:on_open))
66
63
  @driver.on(:message, &method(:on_message))
67
64
  @driver.on(:close, &method(:on_close))
@@ -91,13 +88,7 @@ module Ferrum
91
88
  #
92
89
  def on_message(event)
93
90
  data = safely_parse_json(event.data)
94
- output = event.data
95
- if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data&.dig("id")]
96
- @screenshot_commands.delete(data&.dig("id"))
97
- output.sub!(/{"data":"[^"]*"}/, %("Set FERRUM_LOGGING_SCREENSHOTS=true to see screenshots in Base64"))
98
- end
99
-
100
- @logger&.puts(" ◀ #{Utils::ElapsedTime.elapsed_time} #{output}\n")
91
+ @log&.received(data, event.data, self)
101
92
 
102
93
  # If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
103
94
  # don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
@@ -135,12 +126,10 @@ module Ferrum
135
126
  # @return [void]
136
127
  #
137
128
  def send_message(data, timeout = nil)
138
- @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
139
-
140
129
  json = data.to_json
141
130
  deadline = Utils::ElapsedTime.monotonic_time + timeout if timeout
142
131
  drive(deadline) { @driver.text(json) }
143
- @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
132
+ @log&.sent(data, self)
144
133
  end
145
134
 
146
135
  #
data/lib/ferrum/client.rb CHANGED
@@ -160,7 +160,7 @@ module Ferrum
160
160
  @ws_url = ws_url
161
161
  @options = options
162
162
  @pendings = Concurrent::Hash.new
163
- @ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger, options.protocol_timeout)
163
+ @ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.log, options.protocol_timeout)
164
164
  @subscriber = Subscriber.new
165
165
 
166
166
  start