ferrum 0.19.1 → 0.19.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 07c9e2d0d81f6b0919d937f91e2432ce2a64890b78981d2088d606f8ce088d8f
4
- data.tar.gz: 595a3850a55afed99b8a537b9c544fc706c8ca66f6531b4233b0f5adfa637500
3
+ metadata.gz: b44af007f48e698ad75285cccfcb44da6254de2cb5e9b94965691130ae2ec8e0
4
+ data.tar.gz: 576166fff45f907937bcc23e28a202a101f1d0c16e55772bbf50ec239ab5f627
5
5
  SHA512:
6
- metadata.gz: 2c8f277c645602793e5c9f63e89f12c810668ba9fd8d45585c090a220343abb4ea40769e5c03baa4c59d1aed530282db5bcaa30d151a7662f85142556dcfdd49
7
- data.tar.gz: a4113a1261c436899348992e3514796c3dccffd425003e2d10da7d398bd743f6b864cbc85d8fd48efc33d42f8ce5e4c3546fcc209b813c5417982c4f8dca893e
6
+ metadata.gz: 2951cc2f1e098ddf0ea6df16d4abd68936548667fc191f0d828955c7bcbb719d4d00c260ba4b9941906387b66cef42d6d4cf5e625bb678d2460555c8c1c21ebe
7
+ data.tar.gz: 0fd32129fe61aa693163d41b3986dba56ad7a3729e852e96bf78d9b49cc0a820707cb0cd77c6c91a84faeb1c6e042dca7115144de6da72d1e7d9a52da41c6dec
@@ -12,19 +12,34 @@ module Ferrum
12
12
  # framing, and exposes a queue of parsed incoming messages alongside
13
13
  # methods to send commands and close the connection.
14
14
  #
15
+ # The socket is only ever touched by two threads: the reader reads it and
16
+ # the writer writes it, both blocking. Callers and the driver only encode
17
+ # frames and queue them for the writer, so no lock is held while waiting
18
+ # on the socket and a browser that stops reading blocks nobody but the
19
+ # writer. A frame chunk the browser doesn't take within the write timeout
20
+ # closes the connection. JRuby can't mix a blocking read with
21
+ # `write_nonblock` or `IO.select` from another thread on the same socket,
22
+ # so neither is used.
23
+ #
15
24
  class WebSocket
16
25
  WEBSOCKET_BUG_SLEEP = 0.05
17
26
  DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
18
27
  SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
28
+ WRITE_CHUNK_SIZE = 64 * 1024
19
29
  ESCAPE_OR_LONE_SURROGATE = /\\u[dD][89abAB]\h{2}\\u[dD][c-fC-F]\h{2}|\\u([dD][89a-fA-F]\h{2})|\\./
20
30
 
21
31
  attr_reader :url, :messages
22
32
 
23
- def initialize(url, max_receive_size, logger)
24
- @url = url
33
+ # @return [String, nil]
34
+ # Why Ferrum closed the connection itself, `nil` if it didn't.
35
+ attr_reader :close_reason
36
+
37
+ def initialize(url, max_receive_size, logger, write_timeout = nil)
38
+ @url = url
25
39
  @logger = logger
26
- uri = URI.parse(@url)
27
- port = uri.port || DEFAULT_PORTS[uri.scheme]
40
+ @write_timeout = write_timeout
41
+ uri = URI.parse(@url)
42
+ port = uri.port || DEFAULT_PORTS[uri.scheme]
28
43
 
29
44
  if port == 443 || url.scheme == "wss"
30
45
  tcp = TCPSocket.new(uri.host, port)
@@ -40,8 +55,10 @@ module Ferrum
40
55
  @driver = ::WebSocket::Driver.client(self, max_length: max_receive_size)
41
56
  # websocket-driver holds no locks and is called from many threads: commands from
42
57
  # callers, pong/close replies from the reader. One lock keeps frames from interleaving.
43
- @driver_mutex = Mutex.new
58
+ @mutex = Mutex.new
59
+ @encoded = []
44
60
  @messages = Queue.new
61
+ @outgoing = Queue.new
45
62
 
46
63
  @screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
47
64
 
@@ -50,8 +67,9 @@ module Ferrum
50
67
  @driver.on(:close, &method(:on_close))
51
68
 
52
69
  start
70
+ start_writer
53
71
 
54
- @driver_mutex.synchronize { @driver.start }
72
+ drive { @driver.start }
55
73
  end
56
74
 
57
75
  #
@@ -89,76 +107,54 @@ module Ferrum
89
107
  end
90
108
 
91
109
  #
92
- # Handles the driver's `:close` event: closes the message queue and
93
- # underlying socket, then kills the reader thread.
110
+ # Handles the driver's `:close` event: closes the message queue, lets the
111
+ # writer send what's queued and close the socket, then kills the reader
112
+ # thread.
94
113
  #
95
114
  # @return [void]
96
115
  #
97
116
  def on_close(_event)
98
117
  @messages.close
99
- @sock.close
118
+ queue_encoded
119
+ @outgoing.close
100
120
  @thread.kill
101
121
  end
102
122
 
103
123
  #
104
- # Serializes a CDP command to JSON and sends it as a websocket text
105
- # frame.
124
+ # Serializes a CDP command to JSON and queues it as a websocket text
125
+ # frame. Returns without waiting for the frame to be written.
106
126
  #
107
127
  # @param [Hash] data
108
128
  # The message to send, must include an `:id` key.
109
129
  #
110
130
  # @param [Numeric, nil] timeout
111
- # How long the browser may go without accepting any of the frame's
112
- # bytes. `nil` waits for as long as it takes.
131
+ # How long the frame may wait in the queue. A frame still queued after
132
+ # that is dropped, nobody waits for its response anymore. `nil` keeps it
133
+ # until it's written.
113
134
  #
114
135
  # @return [void]
115
136
  #
116
- # @raise [Ferrum::DeadBrowserError]
117
- # If the browser accepted no data for `timeout` seconds. The connection
118
- # is closed, since the frame was cut off midway.
119
- #
120
137
  def send_message(data, timeout = nil)
121
138
  @screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
122
139
 
123
140
  json = data.to_json
124
- @driver_mutex.synchronize do
125
- @write_timeout = timeout
126
- @driver.text(json)
127
- ensure
128
- @write_timeout = nil
129
- end
141
+ deadline = Utils::ElapsedTime.monotonic_time + timeout if timeout
142
+ drive(deadline) { @driver.text(json) }
130
143
  @logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
131
144
  end
132
145
 
133
146
  #
134
- # Writes raw bytes to the underlying socket. Called by
135
- # `websocket-driver` to emit frames. Closes {#messages} instead of
136
- # raising if the connection has already been torn down.
137
- #
138
- # A browser that stops reading its socket (hung, or stopped) would block
139
- # a plain write forever, holding the lock every other command needs to be
140
- # sent. So the write gives up once the browser accepts no data for the
141
- # timeout of the command being sent.
147
+ # Collects the raw bytes of a frame. Called by `websocket-driver` to
148
+ # emit frames, one call per frame, always from within the driver call
149
+ # that queues them for the writer thread.
142
150
  #
143
151
  # @param [String] data
144
- # The raw bytes to write.
152
+ # The raw bytes of one frame.
145
153
  #
146
154
  # @return [void]
147
155
  #
148
- # @raise [Ferrum::DeadBrowserError]
149
- # If the browser accepted no data for the command's timeout.
150
- #
151
156
  def write(data)
152
- until data.empty?
153
- written = @sock.write_nonblock(data, exception: false)
154
- if written.is_a?(Integer)
155
- data = data.byteslice(written..)
156
- elsif !wait_for_socket(written)
157
- abandon_stalled_write
158
- end
159
- end
160
- rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
161
- @messages.close
157
+ @encoded << data
162
158
  end
163
159
 
164
160
  #
@@ -167,22 +163,23 @@ module Ferrum
167
163
  # @return [void]
168
164
  #
169
165
  def close
170
- @driver_mutex.synchronize { @driver.close }
166
+ drive { @driver.close }
171
167
  end
172
168
 
173
169
  #
174
- # Closes the underlying socket whether or not the close handshake ever
175
- # completed.
170
+ # Stops the writer and closes the underlying socket whether or not the
171
+ # close handshake ever completed.
176
172
  #
177
- # {#close} only sends a close frame; the socket itself is closed by
178
- # {#on_close}, which runs when the browser answers with a close frame of
179
- # its own. A browser that is already dead never answers, so without this
173
+ # {#close} only queues a close frame; the socket itself is closed by the
174
+ # writer once {#on_close} runs, when the browser answers with a close
175
+ # frame of its own. A browser that is already dead never answers, so without this
180
176
  # the socket stays open until GC finalizes it -- long enough to exhaust
181
177
  # the process's file descriptors when browsers are restarted in a loop.
182
178
  #
183
179
  # @return [void]
184
180
  #
185
181
  def force_close
182
+ @outgoing.close
186
183
  @sock.close unless @sock.closed?
187
184
  rescue IOError, SystemCallError
188
185
  # nop
@@ -190,16 +187,50 @@ module Ferrum
190
187
 
191
188
  private
192
189
 
193
- def wait_for_socket(state)
194
- readers, writers = state == :wait_readable ? [[@sock], nil] : [nil, [@sock]]
195
- IO.select(readers, writers, nil, @write_timeout)
190
+ def drive(deadline = nil)
191
+ @mutex.synchronize do
192
+ yield
193
+ queue_encoded(deadline)
194
+ end
195
+ end
196
+
197
+ def queue_encoded(deadline = nil)
198
+ @encoded.each { |data| @outgoing << [data, deadline] }
199
+ rescue ClosedQueueError
200
+ @messages.close
201
+ ensure
202
+ @encoded.clear
203
+ end
204
+
205
+ def start_writer
206
+ @writer = Utils::Thread.spawn do
207
+ while (frame = @outgoing.pop)
208
+ data, deadline = frame
209
+ write_frame(data) unless deadline && Utils::ElapsedTime.monotonic_time > deadline
210
+ end
211
+ rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EBADF, IOError # rubocop:disable Lint/ShadowedException
212
+ # The browser went away, nothing to report.
213
+ rescue StandardError => e
214
+ warn("Ferrum: websocket writer stopped, #{e.class}: #{e.message}\n #{e.backtrace&.first}")
215
+ ensure
216
+ @messages.close
217
+ force_close
218
+ end
219
+ end
220
+
221
+ def write_frame(data)
222
+ 0.step(data.bytesize - 1, WRITE_CHUNK_SIZE) do |offset|
223
+ watchdog = Concurrent::ScheduledTask.execute(@write_timeout) { abandon_stalled_write } if @write_timeout
224
+ @sock.write(data.byteslice(offset, WRITE_CHUNK_SIZE))
225
+ ensure
226
+ watchdog&.cancel
227
+ end
196
228
  end
197
229
 
198
230
  def abandon_stalled_write
199
- timeout = @write_timeout
231
+ @close_reason = "Browser accepted no data for #{@write_timeout}s and is likely stuck, connection closed"
200
232
  @messages.close
201
- @sock.close
202
- raise DeadBrowserError, "Browser accepted no data for #{timeout}s and is likely stuck, connection closed"
233
+ force_close
203
234
  end
204
235
 
205
236
  def start
@@ -208,9 +239,9 @@ module Ferrum
208
239
  data = @sock.readpartial(512)
209
240
  break unless data
210
241
 
211
- @driver_mutex.synchronize { @driver.parse(data) }
242
+ drive { @driver.parse(data) }
212
243
  end
213
- rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
244
+ rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EBADF, IOError # rubocop:disable Lint/ShadowedException
214
245
  # The browser went away, nothing to report.
215
246
  rescue StandardError => e
216
247
  warn("Ferrum: websocket reader stopped, #{e.class}: #{e.message}\n #{e.backtrace&.first}")
data/lib/ferrum/client.rb CHANGED
@@ -160,7 +160,7 @@ module Ferrum
160
160
  @ws_url = ws_url
161
161
  @options = options
162
162
  @pendings = Concurrent::Hash.new
163
- @ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger)
163
+ @ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger, options.protocol_timeout)
164
164
  @subscriber = Subscriber.new
165
165
 
166
166
  start
@@ -208,8 +208,8 @@ module Ferrum
208
208
  # Whether to return immediately instead of waiting for a response.
209
209
  #
210
210
  # @param [Numeric, nil] timeout
211
- # How long to wait for the browser to take the message, and then for its
212
- # response. Defaults to `protocol_timeout`.
211
+ # How long to wait for the response, counting the time the message waits
212
+ # to be sent. Defaults to `protocol_timeout`.
213
213
  #
214
214
  # @return [Boolean, Hash]
215
215
  # `true` when sent asynchronously, otherwise the parsed `"result"`
@@ -222,7 +222,7 @@ module Ferrum
222
222
  @ws.send_message(message, timeout)
223
223
  true
224
224
  else
225
- raise DeadBrowserError if @ws.messages.closed?
225
+ raise dead_browser_error if @ws.messages.closed?
226
226
 
227
227
  pending = Concurrent::IVar.new
228
228
  @pendings[message[:id]] = pending
@@ -233,7 +233,7 @@ module Ferrum
233
233
  @pendings.delete(message[:id])
234
234
  end
235
235
 
236
- raise DeadBrowserError if data.nil? && @ws.messages.closed?
236
+ raise dead_browser_error if data.nil? && @ws.messages.closed?
237
237
  raise TimeoutError.new(message[:method], session_id: message[:sessionId]) unless data
238
238
 
239
239
  error, response = data.values_at("error", "result")
@@ -365,6 +365,10 @@ module Ferrum
365
365
  @pendings.each_value { |pending| pending.try_set(nil) }
366
366
  end
367
367
 
368
+ def dead_browser_error
369
+ @ws.close_reason ? DeadBrowserError.new(@ws.close_reason) : DeadBrowserError.new
370
+ end
371
+
368
372
  # Locked so two concurrent commands never share an id and read each
369
373
  # other's responses from @pendings.
370
374
  def next_command_id
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Ferrum
4
- VERSION = "0.19.1"
4
+ VERSION = "0.19.2"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ferrum
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.19.1
4
+ version: 0.19.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dmitry Vorotilin