ferrum 0.19.0 → 0.19.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4a719b34dd1893e7c0ba2b71e121e1a2b7cc7950b96a1e462ac6a3a9f36c04e0
4
- data.tar.gz: 4c96a549ff3aa00d76420b43c2f7b4870cacb712d390efe7394ad2a8245f6825
3
+ metadata.gz: b44af007f48e698ad75285cccfcb44da6254de2cb5e9b94965691130ae2ec8e0
4
+ data.tar.gz: 576166fff45f907937bcc23e28a202a101f1d0c16e55772bbf50ec239ab5f627
5
5
  SHA512:
6
- metadata.gz: 1a4e9981f2e6334b2862a073eb645b893bbebd42399f0e4b3090d98ea3d69ab0d3104bebe34f86e66b3ec2ea01b827efa28b75c2be55482de6f89a8b082b2113
7
- data.tar.gz: acab89439a3655e51c3173bc30199bbccb6511567803fa16f066b26fbd6e5934f75492b6ed6383c8c59a27e397e23c781226abd439c551bc6be63c001868bc3a
6
+ metadata.gz: 2951cc2f1e098ddf0ea6df16d4abd68936548667fc191f0d828955c7bcbb719d4d00c260ba4b9941906387b66cef42d6d4cf5e625bb678d2460555c8c1c21ebe
7
+ data.tar.gz: 0fd32129fe61aa693163d41b3986dba56ad7a3729e852e96bf78d9b49cc0a820707cb0cd77c6c91a84faeb1c6e042dca7115144de6da72d1e7d9a52da41c6dec
@@ -48,7 +48,6 @@ module Ferrum
48
48
  "disable-smooth-scrolling" => nil,
49
49
  "disable-sync" => nil,
50
50
  "disable-translate" => nil,
51
- "disable-web-security" => nil,
52
51
  "enable-automation" => nil,
53
52
  "enable-features" => %w[
54
53
  NetworkService
@@ -12,18 +12,34 @@ module Ferrum
12
12
  # framing, and exposes a queue of parsed incoming messages alongside
13
13
  # methods to send commands and close the connection.
14
14
  #
15
+ # The socket is only ever touched by two threads: the reader reads it and
16
+ # the writer writes it, both blocking. Callers and the driver only encode
17
+ # frames and queue them for the writer, so no lock is held while waiting
18
+ # on the socket and a browser that stops reading blocks nobody but the
19
+ # writer. A frame chunk the browser doesn't take within the write timeout
20
+ # closes the connection. JRuby can't mix a blocking read with
21
+ # `write_nonblock` or `IO.select` from another thread on the same socket,
22
+ # so neither is used.
23
+ #
15
24
  class WebSocket
16
25
  WEBSOCKET_BUG_SLEEP = 0.05
17
26
  DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
18
27
  SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
28
+ WRITE_CHUNK_SIZE = 64 * 1024
29
+ ESCAPE_OR_LONE_SURROGATE = /\\u[dD][89abAB]\h{2}\\u[dD][c-fC-F]\h{2}|\\u([dD][89a-fA-F]\h{2})|\\./
19
30
 
20
31
  attr_reader :url, :messages
21
32
 
22
- def initialize(url, max_receive_size, logger)
23
- @url = url
33
+ # @return [String, nil]
34
+ # Why Ferrum closed the connection itself, `nil` if it didn't.
35
+ attr_reader :close_reason
36
+
37
+ def initialize(url, max_receive_size, logger, write_timeout = nil)
38
+ @url = url
24
39
  @logger = logger
25
- uri = URI.parse(@url)
26
- port = uri.port || DEFAULT_PORTS[uri.scheme]
40
+ @write_timeout = write_timeout
41
+ uri = URI.parse(@url)
42
+ port = uri.port || DEFAULT_PORTS[uri.scheme]
27
43
 
28
44
  if port == 443 || url.scheme == "wss"
29
45
  tcp = TCPSocket.new(uri.host, port)
@@ -39,8 +55,10 @@ module Ferrum
39
55
  @driver = ::WebSocket::Driver.client(self, max_length: max_receive_size)
40
56
  # websocket-driver holds no locks and is called from many threads: commands from
41
57
  # callers, pong/close replies from the reader. One lock keeps frames from interleaving.
42
- @driver_mutex = Mutex.new
58
+ @mutex = Mutex.new
59
+ @encoded = []
43
60
  @messages = Queue.new
61
+ @outgoing = Queue.new
44
62
 
45
63
  @screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
46
64
 
@@ -49,8 +67,9 @@ module Ferrum
49
67
  @driver.on(:close, &method(:on_close))
50
68
 
51
69
  start
70
+ start_writer
52
71
 
53
- @driver_mutex.synchronize { @driver.start }
72
+ drive { @driver.start }
54
73
  end
55
74
 
56
75
  #
@@ -88,76 +107,54 @@ module Ferrum
88
107
  end
89
108
 
90
109
  #
91
- # Handles the driver's `:close` event: closes the message queue and
92
- # underlying socket, then kills the reader thread.
110
+ # Handles the driver's `:close` event: closes the message queue, lets the
111
+ # writer send what's queued and close the socket, then kills the reader
112
+ # thread.
93
113
  #
94
114
  # @return [void]
95
115
  #
96
116
  def on_close(_event)
97
117
  @messages.close
98
- @sock.close
118
+ queue_encoded
119
+ @outgoing.close
99
120
  @thread.kill
100
121
  end
101
122
 
102
123
  #
103
- # Serializes a CDP command to JSON and sends it as a websocket text
104
- # frame.
124
+ # Serializes a CDP command to JSON and queues it as a websocket text
125
+ # frame. Returns without waiting for the frame to be written.
105
126
  #
106
127
  # @param [Hash] data
107
128
  # The message to send, must include an `:id` key.
108
129
  #
109
130
  # @param [Numeric, nil] timeout
110
- # How long the browser may go without accepting any of the frame's
111
- # bytes. `nil` waits for as long as it takes.
131
+ # How long the frame may wait in the queue. A frame still queued after
132
+ # that is dropped, nobody waits for its response anymore. `nil` keeps it
133
+ # until it's written.
112
134
  #
113
135
  # @return [void]
114
136
  #
115
- # @raise [Ferrum::DeadBrowserError]
116
- # If the browser accepted no data for `timeout` seconds. The connection
117
- # is closed, since the frame was cut off midway.
118
- #
119
137
  def send_message(data, timeout = nil)
120
138
  @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
121
139
 
122
140
  json = data.to_json
123
- @driver_mutex.synchronize do
124
- @write_timeout = timeout
125
- @driver.text(json)
126
- ensure
127
- @write_timeout = nil
128
- end
141
+ deadline = Utils::ElapsedTime.monotonic_time + timeout if timeout
142
+ drive(deadline) { @driver.text(json) }
129
143
  @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
130
144
  end
131
145
 
132
146
  #
133
- # Writes raw bytes to the underlying socket. Called by
134
- # `websocket-driver` to emit frames. Closes {#messages} instead of
135
- # raising if the connection has already been torn down.
136
- #
137
- # A browser that stops reading its socket (hung, or stopped) would block
138
- # a plain write forever, holding the lock every other command needs to be
139
- # sent. So the write gives up once the browser accepts no data for the
140
- # timeout of the command being sent.
147
+ # Collects the raw bytes of a frame. Called by `websocket-driver` to
148
+ # emit frames, one call per frame, always from within the driver call
149
+ # that queues them for the writer thread.
141
150
  #
142
151
  # @param [String] data
143
- # The raw bytes to write.
152
+ # The raw bytes of one frame.
144
153
  #
145
154
  # @return [void]
146
155
  #
147
- # @raise [Ferrum::DeadBrowserError]
148
- # If the browser accepted no data for the command's timeout.
149
- #
150
156
  def write(data)
151
- until data.empty?
152
- written = @sock.write_nonblock(data, exception: false)
153
- if written.is_a?(Integer)
154
- data = data.byteslice(written..)
155
- elsif !wait_for_socket(written)
156
- abandon_stalled_write
157
- end
158
- end
159
- rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
160
- @messages.close
157
+ @encoded << data
161
158
  end
162
159
 
163
160
  #
@@ -166,22 +163,23 @@ module Ferrum
166
163
  # @return [void]
167
164
  #
168
165
  def close
169
- @driver_mutex.synchronize { @driver.close }
166
+ drive { @driver.close }
170
167
  end
171
168
 
172
169
  #
173
- # Closes the underlying socket whether or not the close handshake ever
174
- # completed.
170
+ # Stops the writer and closes the underlying socket whether or not the
171
+ # close handshake ever completed.
175
172
  #
176
- # {#close} only sends a close frame; the socket itself is closed by
177
- # {#on_close}, which runs when the browser answers with a close frame of
178
- # its own. A browser that is already dead never answers, so without this
173
+ # {#close} only queues a close frame; the socket itself is closed by the
174
+ # writer once {#on_close} runs, when the browser answers with a close
175
+ # frame of its own. A browser that is already dead never answers, so without this
179
176
  # the socket stays open until GC finalizes it -- long enough to exhaust
180
177
  # the process's file descriptors when browsers are restarted in a loop.
181
178
  #
182
179
  # @return [void]
183
180
  #
184
181
  def force_close
182
+ @outgoing.close
185
183
  @sock.close unless @sock.closed?
186
184
  rescue IOError, SystemCallError
187
185
  # nop
@@ -189,16 +187,50 @@ module Ferrum
189
187
 
190
188
  private
191
189
 
192
- def wait_for_socket(state)
193
- readers, writers = state == :wait_readable ? [[@sock], nil] : [nil, [@sock]]
194
- IO.select(readers, writers, nil, @write_timeout)
190
+ def drive(deadline = nil)
191
+ @mutex.synchronize do
192
+ yield
193
+ queue_encoded(deadline)
194
+ end
195
+ end
196
+
197
+ def queue_encoded(deadline = nil)
198
+ @encoded.each { |data| @outgoing << [data, deadline] }
199
+ rescue ClosedQueueError
200
+ @messages.close
201
+ ensure
202
+ @encoded.clear
203
+ end
204
+
205
+ def start_writer
206
+ @writer = Utils::Thread.spawn do
207
+ while (frame = @outgoing.pop)
208
+ data, deadline = frame
209
+ write_frame(data) unless deadline && Utils::ElapsedTime.monotonic_time > deadline
210
+ end
211
+ rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EBADF, IOError # rubocop:disable Lint/ShadowedException
212
+ # The browser went away, nothing to report.
213
+ rescue StandardError => e
214
+ warn("Ferrum: websocket writer stopped, #{e.class}: #{e.message}\n #{e.backtrace&.first}")
215
+ ensure
216
+ @messages.close
217
+ force_close
218
+ end
219
+ end
220
+
221
+ def write_frame(data)
222
+ 0.step(data.bytesize - 1, WRITE_CHUNK_SIZE) do |offset|
223
+ watchdog = Concurrent::ScheduledTask.execute(@write_timeout) { abandon_stalled_write } if @write_timeout
224
+ @sock.write(data.byteslice(offset, WRITE_CHUNK_SIZE))
225
+ ensure
226
+ watchdog&.cancel
227
+ end
195
228
  end
196
229
 
197
230
  def abandon_stalled_write
198
- timeout = @write_timeout
231
+ @close_reason = "Browser accepted no data for #{@write_timeout}s and is likely stuck, connection closed"
199
232
  @messages.close
200
- @sock.close
201
- raise DeadBrowserError, "Browser accepted no data for #{timeout}s and is likely stuck, connection closed"
233
+ force_close
202
234
  end
203
235
 
204
236
  def start
@@ -207,9 +239,9 @@ module Ferrum
207
239
  data = @sock.readpartial(512)
208
240
  break unless data
209
241
 
210
- @driver_mutex.synchronize { @driver.parse(data) }
242
+ drive { @driver.parse(data) }
211
243
  end
212
- rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
244
+ rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EBADF, IOError # rubocop:disable Lint/ShadowedException
213
245
  # The browser went away, nothing to report.
214
246
  rescue StandardError => e
215
247
  warn("Ferrum: websocket reader stopped, #{e.class}: #{e.message}\n #{e.backtrace&.first}")
@@ -223,14 +255,12 @@ module Ferrum
223
255
  rescue JSON::NestingError
224
256
  # nop
225
257
  rescue JSON::ParserError
226
- safely_parse_escaped_json(data)
258
+ safely_parse_lone_surrogates(data)
227
259
  end
228
260
 
229
- def safely_parse_escaped_json(data)
230
- unescaped_unicode =
231
- data.gsub(/\\u([\da-fA-F]{4})/) { |_| [::Regexp.last_match(1)].pack("H*").unpack("n*").pack("U*") }
232
- escaped_data = unescaped_unicode.encode("UTF-8", "UTF-8", undef: :replace, invalid: :replace, replace: "?")
233
- JSON.parse(escaped_data, max_nesting: false)
261
+ def safely_parse_lone_surrogates(data)
262
+ replaced = data.gsub(ESCAPE_OR_LONE_SURROGATE) { |escape| ::Regexp.last_match(1) ? "\\ufffd" : escape }
263
+ JSON.parse(replaced, max_nesting: false)
234
264
  rescue JSON::ParserError
235
265
  # nop
236
266
  end
data/lib/ferrum/client.rb CHANGED
@@ -160,7 +160,7 @@ module Ferrum
160
160
  @ws_url = ws_url
161
161
  @options = options
162
162
  @pendings = Concurrent::Hash.new
163
- @ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger)
163
+ @ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger, options.protocol_timeout)
164
164
  @subscriber = Subscriber.new
165
165
 
166
166
  start
@@ -208,8 +208,8 @@ module Ferrum
208
208
  # Whether to return immediately instead of waiting for a response.
209
209
  #
210
210
  # @param [Numeric, nil] timeout
211
- # How long to wait for the browser to take the message, and then for its
212
- # response. Defaults to `protocol_timeout`.
211
+ # How long to wait for the response, counting the time the message waits
212
+ # to be sent. Defaults to `protocol_timeout`.
213
213
  #
214
214
  # @return [Boolean, Hash]
215
215
  # `true` when sent asynchronously, otherwise the parsed `"result"`
@@ -222,7 +222,7 @@ module Ferrum
222
222
  @ws.send_message(message, timeout)
223
223
  true
224
224
  else
225
- raise DeadBrowserError if @ws.messages.closed?
225
+ raise dead_browser_error if @ws.messages.closed?
226
226
 
227
227
  pending = Concurrent::IVar.new
228
228
  @pendings[message[:id]] = pending
@@ -233,7 +233,7 @@ module Ferrum
233
233
  @pendings.delete(message[:id])
234
234
  end
235
235
 
236
- raise DeadBrowserError if data.nil? && @ws.messages.closed?
236
+ raise dead_browser_error if data.nil? && @ws.messages.closed?
237
237
  raise TimeoutError.new(message[:method], session_id: message[:sessionId]) unless data
238
238
 
239
239
  error, response = data.values_at("error", "result")
@@ -365,6 +365,10 @@ module Ferrum
365
365
  @pendings.each_value { |pending| pending.try_set(nil) }
366
366
  end
367
367
 
368
+ def dead_browser_error
369
+ @ws.close_reason ? DeadBrowserError.new(@ws.close_reason) : DeadBrowserError.new
370
+ end
371
+
368
372
  # Locked so two concurrent commands never share an id and read each
369
373
  # other's responses from @pendings.
370
374
  def next_command_id
@@ -376,7 +380,8 @@ module Ferrum
376
380
  # Node has disappeared while we were trying to get it
377
381
  when "No node with given id found",
378
382
  "Could not find node with given id",
379
- "Inspected target navigated or closed"
383
+ "Inspected target navigated or closed",
384
+ "Argument should belong to the same JavaScript world as target object"
380
385
  raise NodeNotFoundError, error
381
386
  # Context is lost, page is reloading
382
387
  when "Cannot find context with specified id"
@@ -116,7 +116,8 @@ module Ferrum
116
116
  # Returns the element in which the window is embedded.
117
117
  #
118
118
  # @return [Node, nil]
119
- # The element in which the window is embedded.
119
+ # The element in which the window is embedded, `nil` for the main frame
120
+ # and for a frame from a different origin than its parent.
120
121
  #
121
122
  # @example
122
123
  # browser.go_to("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
data/lib/ferrum/mouse.rb CHANGED
@@ -152,6 +152,7 @@ module Ferrum
152
152
  from_y = @y
153
153
  @x = x
154
154
  @y = y
155
+ button = pressed_button
155
156
 
156
157
  steps.times do |i|
157
158
  new_x = from_x + ((@x - from_x) * ((i + 1) / steps.to_f))
@@ -162,6 +163,7 @@ module Ferrum
162
163
  type: "mouseMoved",
163
164
  x: new_x,
164
165
  y: new_y,
166
+ button: button,
165
167
  buttons: @buttons)
166
168
  end
167
169
 
@@ -185,6 +187,10 @@ module Ferrum
185
187
  button
186
188
  end
187
189
 
190
+ def pressed_button
191
+ BUTTON_MASKS.find { |_name, mask| mask.positive? && @buttons.anybits?(mask) }&.first || "none"
192
+ end
193
+
188
194
  def register_event_button(type, button)
189
195
  case type
190
196
  when "mousePressed"
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Ferrum
4
- VERSION = "0.19.0"
4
+ VERSION = "0.19.2"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ferrum
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.19.0
4
+ version: 0.19.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dmitry Vorotilin