ferrum 0.19.1 → 0.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/ferrum/client/web_socket.rb +90 -59
- data/lib/ferrum/client.rb +9 -5
- data/lib/ferrum/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b44af007f48e698ad75285cccfcb44da6254de2cb5e9b94965691130ae2ec8e0
|
|
4
|
+
data.tar.gz: 576166fff45f907937bcc23e28a202a101f1d0c16e55772bbf50ec239ab5f627
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2951cc2f1e098ddf0ea6df16d4abd68936548667fc191f0d828955c7bcbb719d4d00c260ba4b9941906387b66cef42d6d4cf5e625bb678d2460555c8c1c21ebe
|
|
7
|
+
data.tar.gz: 0fd32129fe61aa693163d41b3986dba56ad7a3729e852e96bf78d9b49cc0a820707cb0cd77c6c91a84faeb1c6e042dca7115144de6da72d1e7d9a52da41c6dec
|
|
@@ -12,19 +12,34 @@ module Ferrum
|
|
|
12
12
|
# framing, and exposes a queue of parsed incoming messages alongside
|
|
13
13
|
# methods to send commands and close the connection.
|
|
14
14
|
#
|
|
15
|
+
# The socket is only ever touched by two threads: the reader reads it and
|
|
16
|
+
# the writer writes it, both blocking. Callers and the driver only encode
|
|
17
|
+
# frames and queue them for the writer, so no lock is held while waiting
|
|
18
|
+
# on the socket and a browser that stops reading blocks nobody but the
|
|
19
|
+
# writer. A frame chunk the browser doesn't take within the write timeout
|
|
20
|
+
# closes the connection. JRuby can't mix a blocking read with
|
|
21
|
+
# `write_nonblock` or `IO.select` from another thread on the same socket,
|
|
22
|
+
# so neither is used.
|
|
23
|
+
#
|
|
15
24
|
class WebSocket
|
|
16
25
|
WEBSOCKET_BUG_SLEEP = 0.05
|
|
17
26
|
DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
|
|
18
27
|
SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
|
|
28
|
+
WRITE_CHUNK_SIZE = 64 * 1024
|
|
19
29
|
ESCAPE_OR_LONE_SURROGATE = /\\u[dD][89abAB]\h{2}\\u[dD][c-fC-F]\h{2}|\\u([dD][89a-fA-F]\h{2})|\\./
|
|
20
30
|
|
|
21
31
|
attr_reader :url, :messages
|
|
22
32
|
|
|
23
|
-
|
|
24
|
-
|
|
33
|
+
# @return [String, nil]
|
|
34
|
+
# Why Ferrum closed the connection itself, `nil` if it didn't.
|
|
35
|
+
attr_reader :close_reason
|
|
36
|
+
|
|
37
|
+
def initialize(url, max_receive_size, logger, write_timeout = nil)
|
|
38
|
+
@url = url
|
|
25
39
|
@logger = logger
|
|
26
|
-
|
|
27
|
-
|
|
40
|
+
@write_timeout = write_timeout
|
|
41
|
+
uri = URI.parse(@url)
|
|
42
|
+
port = uri.port || DEFAULT_PORTS[uri.scheme]
|
|
28
43
|
|
|
29
44
|
if port == 443 || url.scheme == "wss"
|
|
30
45
|
tcp = TCPSocket.new(uri.host, port)
|
|
@@ -40,8 +55,10 @@ module Ferrum
|
|
|
40
55
|
@driver = ::WebSocket::Driver.client(self, max_length: max_receive_size)
|
|
41
56
|
# websocket-driver holds no locks and is called from many threads: commands from
|
|
42
57
|
# callers, pong/close replies from the reader. One lock keeps frames from interleaving.
|
|
43
|
-
@
|
|
58
|
+
@mutex = Mutex.new
|
|
59
|
+
@encoded = []
|
|
44
60
|
@messages = Queue.new
|
|
61
|
+
@outgoing = Queue.new
|
|
45
62
|
|
|
46
63
|
@screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
|
|
47
64
|
|
|
@@ -50,8 +67,9 @@ module Ferrum
|
|
|
50
67
|
@driver.on(:close, &method(:on_close))
|
|
51
68
|
|
|
52
69
|
start
|
|
70
|
+
start_writer
|
|
53
71
|
|
|
54
|
-
|
|
72
|
+
drive { @driver.start }
|
|
55
73
|
end
|
|
56
74
|
|
|
57
75
|
#
|
|
@@ -89,76 +107,54 @@ module Ferrum
|
|
|
89
107
|
end
|
|
90
108
|
|
|
91
109
|
#
|
|
92
|
-
# Handles the driver's `:close` event: closes the message queue
|
|
93
|
-
#
|
|
110
|
+
# Handles the driver's `:close` event: closes the message queue, lets the
|
|
111
|
+
# writer send what's queued and close the socket, then kills the reader
|
|
112
|
+
# thread.
|
|
94
113
|
#
|
|
95
114
|
# @return [void]
|
|
96
115
|
#
|
|
97
116
|
def on_close(_event)
|
|
98
117
|
@messages.close
|
|
99
|
-
|
|
118
|
+
queue_encoded
|
|
119
|
+
@outgoing.close
|
|
100
120
|
@thread.kill
|
|
101
121
|
end
|
|
102
122
|
|
|
103
123
|
#
|
|
104
|
-
# Serializes a CDP command to JSON and
|
|
105
|
-
# frame.
|
|
124
|
+
# Serializes a CDP command to JSON and queues it as a websocket text
|
|
125
|
+
# frame. Returns without waiting for the frame to be written.
|
|
106
126
|
#
|
|
107
127
|
# @param [Hash] data
|
|
108
128
|
# The message to send, must include an `:id` key.
|
|
109
129
|
#
|
|
110
130
|
# @param [Numeric, nil] timeout
|
|
111
|
-
# How long the
|
|
112
|
-
#
|
|
131
|
+
# How long the frame may wait in the queue. A frame still queued after
|
|
132
|
+
# that is dropped, nobody waits for its response anymore. `nil` keeps it
|
|
133
|
+
# until it's written.
|
|
113
134
|
#
|
|
114
135
|
# @return [void]
|
|
115
136
|
#
|
|
116
|
-
# @raise [Ferrum::DeadBrowserError]
|
|
117
|
-
# If the browser accepted no data for `timeout` seconds. The connection
|
|
118
|
-
# is closed, since the frame was cut off midway.
|
|
119
|
-
#
|
|
120
137
|
def send_message(data, timeout = nil)
|
|
121
138
|
@screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
|
|
122
139
|
|
|
123
140
|
json = data.to_json
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
@driver.text(json)
|
|
127
|
-
ensure
|
|
128
|
-
@write_timeout = nil
|
|
129
|
-
end
|
|
141
|
+
deadline = Utils::ElapsedTime.monotonic_time + timeout if timeout
|
|
142
|
+
drive(deadline) { @driver.text(json) }
|
|
130
143
|
@logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
|
|
131
144
|
end
|
|
132
145
|
|
|
133
146
|
#
|
|
134
|
-
#
|
|
135
|
-
#
|
|
136
|
-
#
|
|
137
|
-
#
|
|
138
|
-
# A browser that stops reading its socket (hung, or stopped) would block
|
|
139
|
-
# a plain write forever, holding the lock every other command needs to be
|
|
140
|
-
# sent. So the write gives up once the browser accepts no data for the
|
|
141
|
-
# timeout of the command being sent.
|
|
147
|
+
# Collects the raw bytes of a frame. Called by `websocket-driver` to
|
|
148
|
+
# emit frames, one call per frame, always from within the driver call
|
|
149
|
+
# that queues them for the writer thread.
|
|
142
150
|
#
|
|
143
151
|
# @param [String] data
|
|
144
|
-
# The raw bytes
|
|
152
|
+
# The raw bytes of one frame.
|
|
145
153
|
#
|
|
146
154
|
# @return [void]
|
|
147
155
|
#
|
|
148
|
-
# @raise [Ferrum::DeadBrowserError]
|
|
149
|
-
# If the browser accepted no data for the command's timeout.
|
|
150
|
-
#
|
|
151
156
|
def write(data)
|
|
152
|
-
|
|
153
|
-
written = @sock.write_nonblock(data, exception: false)
|
|
154
|
-
if written.is_a?(Integer)
|
|
155
|
-
data = data.byteslice(written..)
|
|
156
|
-
elsif !wait_for_socket(written)
|
|
157
|
-
abandon_stalled_write
|
|
158
|
-
end
|
|
159
|
-
end
|
|
160
|
-
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
|
|
161
|
-
@messages.close
|
|
157
|
+
@encoded << data
|
|
162
158
|
end
|
|
163
159
|
|
|
164
160
|
#
|
|
@@ -167,22 +163,23 @@ module Ferrum
|
|
|
167
163
|
# @return [void]
|
|
168
164
|
#
|
|
169
165
|
def close
|
|
170
|
-
|
|
166
|
+
drive { @driver.close }
|
|
171
167
|
end
|
|
172
168
|
|
|
173
169
|
#
|
|
174
|
-
#
|
|
175
|
-
# completed.
|
|
170
|
+
# Stops the writer and closes the underlying socket whether or not the
|
|
171
|
+
# close handshake ever completed.
|
|
176
172
|
#
|
|
177
|
-
# {#close} only
|
|
178
|
-
# {#on_close}
|
|
179
|
-
# its own. A browser that is already dead never answers, so without this
|
|
173
|
+
# {#close} only queues a close frame; the socket itself is closed by the
|
|
174
|
+
# writer once {#on_close} runs, when the browser answers with a close
|
|
175
|
+
# frame of its own. A browser that is already dead never answers, so without this
|
|
180
176
|
# the socket stays open until GC finalizes it -- long enough to exhaust
|
|
181
177
|
# the process's file descriptors when browsers are restarted in a loop.
|
|
182
178
|
#
|
|
183
179
|
# @return [void]
|
|
184
180
|
#
|
|
185
181
|
def force_close
|
|
182
|
+
@outgoing.close
|
|
186
183
|
@sock.close unless @sock.closed?
|
|
187
184
|
rescue IOError, SystemCallError
|
|
188
185
|
# nop
|
|
@@ -190,16 +187,50 @@ module Ferrum
|
|
|
190
187
|
|
|
191
188
|
private
|
|
192
189
|
|
|
193
|
-
def
|
|
194
|
-
|
|
195
|
-
|
|
190
|
+
def drive(deadline = nil)
|
|
191
|
+
@mutex.synchronize do
|
|
192
|
+
yield
|
|
193
|
+
queue_encoded(deadline)
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def queue_encoded(deadline = nil)
|
|
198
|
+
@encoded.each { |data| @outgoing << [data, deadline] }
|
|
199
|
+
rescue ClosedQueueError
|
|
200
|
+
@messages.close
|
|
201
|
+
ensure
|
|
202
|
+
@encoded.clear
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def start_writer
|
|
206
|
+
@writer = Utils::Thread.spawn do
|
|
207
|
+
while (frame = @outgoing.pop)
|
|
208
|
+
data, deadline = frame
|
|
209
|
+
write_frame(data) unless deadline && Utils::ElapsedTime.monotonic_time > deadline
|
|
210
|
+
end
|
|
211
|
+
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EBADF, IOError # rubocop:disable Lint/ShadowedException
|
|
212
|
+
# The browser went away, nothing to report.
|
|
213
|
+
rescue StandardError => e
|
|
214
|
+
warn("Ferrum: websocket writer stopped, #{e.class}: #{e.message}\n #{e.backtrace&.first}")
|
|
215
|
+
ensure
|
|
216
|
+
@messages.close
|
|
217
|
+
force_close
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def write_frame(data)
|
|
222
|
+
0.step(data.bytesize - 1, WRITE_CHUNK_SIZE) do |offset|
|
|
223
|
+
watchdog = Concurrent::ScheduledTask.execute(@write_timeout) { abandon_stalled_write } if @write_timeout
|
|
224
|
+
@sock.write(data.byteslice(offset, WRITE_CHUNK_SIZE))
|
|
225
|
+
ensure
|
|
226
|
+
watchdog&.cancel
|
|
227
|
+
end
|
|
196
228
|
end
|
|
197
229
|
|
|
198
230
|
def abandon_stalled_write
|
|
199
|
-
|
|
231
|
+
@close_reason = "Browser accepted no data for #{@write_timeout}s and is likely stuck, connection closed"
|
|
200
232
|
@messages.close
|
|
201
|
-
|
|
202
|
-
raise DeadBrowserError, "Browser accepted no data for #{timeout}s and is likely stuck, connection closed"
|
|
233
|
+
force_close
|
|
203
234
|
end
|
|
204
235
|
|
|
205
236
|
def start
|
|
@@ -208,9 +239,9 @@ module Ferrum
|
|
|
208
239
|
data = @sock.readpartial(512)
|
|
209
240
|
break unless data
|
|
210
241
|
|
|
211
|
-
|
|
242
|
+
drive { @driver.parse(data) }
|
|
212
243
|
end
|
|
213
|
-
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
|
|
244
|
+
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EBADF, IOError # rubocop:disable Lint/ShadowedException
|
|
214
245
|
# The browser went away, nothing to report.
|
|
215
246
|
rescue StandardError => e
|
|
216
247
|
warn("Ferrum: websocket reader stopped, #{e.class}: #{e.message}\n #{e.backtrace&.first}")
|
data/lib/ferrum/client.rb
CHANGED
|
@@ -160,7 +160,7 @@ module Ferrum
|
|
|
160
160
|
@ws_url = ws_url
|
|
161
161
|
@options = options
|
|
162
162
|
@pendings = Concurrent::Hash.new
|
|
163
|
-
@ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger)
|
|
163
|
+
@ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger, options.protocol_timeout)
|
|
164
164
|
@subscriber = Subscriber.new
|
|
165
165
|
|
|
166
166
|
start
|
|
@@ -208,8 +208,8 @@ module Ferrum
|
|
|
208
208
|
# Whether to return immediately instead of waiting for a response.
|
|
209
209
|
#
|
|
210
210
|
# @param [Numeric, nil] timeout
|
|
211
|
-
# How long to wait for the
|
|
212
|
-
#
|
|
211
|
+
# How long to wait for the response, counting the time the message waits
|
|
212
|
+
# to be sent. Defaults to `protocol_timeout`.
|
|
213
213
|
#
|
|
214
214
|
# @return [Boolean, Hash]
|
|
215
215
|
# `true` when sent asynchronously, otherwise the parsed `"result"`
|
|
@@ -222,7 +222,7 @@ module Ferrum
|
|
|
222
222
|
@ws.send_message(message, timeout)
|
|
223
223
|
true
|
|
224
224
|
else
|
|
225
|
-
raise
|
|
225
|
+
raise dead_browser_error if @ws.messages.closed?
|
|
226
226
|
|
|
227
227
|
pending = Concurrent::IVar.new
|
|
228
228
|
@pendings[message[:id]] = pending
|
|
@@ -233,7 +233,7 @@ module Ferrum
|
|
|
233
233
|
@pendings.delete(message[:id])
|
|
234
234
|
end
|
|
235
235
|
|
|
236
|
-
raise
|
|
236
|
+
raise dead_browser_error if data.nil? && @ws.messages.closed?
|
|
237
237
|
raise TimeoutError.new(message[:method], session_id: message[:sessionId]) unless data
|
|
238
238
|
|
|
239
239
|
error, response = data.values_at("error", "result")
|
|
@@ -365,6 +365,10 @@ module Ferrum
|
|
|
365
365
|
@pendings.each_value { |pending| pending.try_set(nil) }
|
|
366
366
|
end
|
|
367
367
|
|
|
368
|
+
def dead_browser_error
|
|
369
|
+
@ws.close_reason ? DeadBrowserError.new(@ws.close_reason) : DeadBrowserError.new
|
|
370
|
+
end
|
|
371
|
+
|
|
368
372
|
# Locked so two concurrent commands never share an id and read each
|
|
369
373
|
# other's responses from @pendings.
|
|
370
374
|
def next_command_id
|
data/lib/ferrum/version.rb
CHANGED