ferrum 0.19.0 → 0.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/ferrum/browser/options/chrome.rb +0 -1
- data/lib/ferrum/client/web_socket.rb +95 -65
- data/lib/ferrum/client.rb +11 -6
- data/lib/ferrum/frame/dom.rb +2 -1
- data/lib/ferrum/mouse.rb +6 -0
- data/lib/ferrum/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b44af007f48e698ad75285cccfcb44da6254de2cb5e9b94965691130ae2ec8e0
|
|
4
|
+
data.tar.gz: 576166fff45f907937bcc23e28a202a101f1d0c16e55772bbf50ec239ab5f627
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2951cc2f1e098ddf0ea6df16d4abd68936548667fc191f0d828955c7bcbb719d4d00c260ba4b9941906387b66cef42d6d4cf5e625bb678d2460555c8c1c21ebe
|
|
7
|
+
data.tar.gz: 0fd32129fe61aa693163d41b3986dba56ad7a3729e852e96bf78d9b49cc0a820707cb0cd77c6c91a84faeb1c6e042dca7115144de6da72d1e7d9a52da41c6dec
|
|
@@ -12,18 +12,34 @@ module Ferrum
|
|
|
12
12
|
# framing, and exposes a queue of parsed incoming messages alongside
|
|
13
13
|
# methods to send commands and close the connection.
|
|
14
14
|
#
|
|
15
|
+
# The socket is only ever touched by two threads: the reader reads it and
|
|
16
|
+
# the writer writes it, both blocking. Callers and the driver only encode
|
|
17
|
+
# frames and queue them for the writer, so no lock is held while waiting
|
|
18
|
+
# on the socket and a browser that stops reading blocks nobody but the
|
|
19
|
+
# writer. A frame chunk the browser doesn't take within the write timeout
|
|
20
|
+
# closes the connection. JRuby can't mix a blocking read with
|
|
21
|
+
# `write_nonblock` or `IO.select` from another thread on the same socket,
|
|
22
|
+
# so neither is used.
|
|
23
|
+
#
|
|
15
24
|
class WebSocket
|
|
16
25
|
WEBSOCKET_BUG_SLEEP = 0.05
|
|
17
26
|
DEFAULT_PORTS = { "ws" => 80, "wss" => 443 }.freeze
|
|
18
27
|
SKIP_LOGGING_SCREENSHOTS = !ENV["FERRUM_LOGGING_SCREENSHOTS"]
|
|
28
|
+
WRITE_CHUNK_SIZE = 64 * 1024
|
|
29
|
+
ESCAPE_OR_LONE_SURROGATE = /\\u[dD][89abAB]\h{2}\\u[dD][c-fC-F]\h{2}|\\u([dD][89a-fA-F]\h{2})|\\./
|
|
19
30
|
|
|
20
31
|
attr_reader :url, :messages
|
|
21
32
|
|
|
22
|
-
|
|
23
|
-
|
|
33
|
+
# @return [String, nil]
|
|
34
|
+
# Why Ferrum closed the connection itself, `nil` if it didn't.
|
|
35
|
+
attr_reader :close_reason
|
|
36
|
+
|
|
37
|
+
def initialize(url, max_receive_size, logger, write_timeout = nil)
|
|
38
|
+
@url = url
|
|
24
39
|
@logger = logger
|
|
25
|
-
|
|
26
|
-
|
|
40
|
+
@write_timeout = write_timeout
|
|
41
|
+
uri = URI.parse(@url)
|
|
42
|
+
port = uri.port || DEFAULT_PORTS[uri.scheme]
|
|
27
43
|
|
|
28
44
|
if port == 443 || url.scheme == "wss"
|
|
29
45
|
tcp = TCPSocket.new(uri.host, port)
|
|
@@ -39,8 +55,10 @@ module Ferrum
|
|
|
39
55
|
@driver = ::WebSocket::Driver.client(self, max_length: max_receive_size)
|
|
40
56
|
# websocket-driver holds no locks and is called from many threads: commands from
|
|
41
57
|
# callers, pong/close replies from the reader. One lock keeps frames from interleaving.
|
|
42
|
-
@
|
|
58
|
+
@mutex = Mutex.new
|
|
59
|
+
@encoded = []
|
|
43
60
|
@messages = Queue.new
|
|
61
|
+
@outgoing = Queue.new
|
|
44
62
|
|
|
45
63
|
@screenshot_commands = Concurrent::Hash.new if SKIP_LOGGING_SCREENSHOTS
|
|
46
64
|
|
|
@@ -49,8 +67,9 @@ module Ferrum
|
|
|
49
67
|
@driver.on(:close, &method(:on_close))
|
|
50
68
|
|
|
51
69
|
start
|
|
70
|
+
start_writer
|
|
52
71
|
|
|
53
|
-
|
|
72
|
+
drive { @driver.start }
|
|
54
73
|
end
|
|
55
74
|
|
|
56
75
|
#
|
|
@@ -88,76 +107,54 @@ module Ferrum
|
|
|
88
107
|
end
|
|
89
108
|
|
|
90
109
|
#
|
|
91
|
-
# Handles the driver's `:close` event: closes the message queue
|
|
92
|
-
#
|
|
110
|
+
# Handles the driver's `:close` event: closes the message queue, lets the
|
|
111
|
+
# writer send what's queued and close the socket, then kills the reader
|
|
112
|
+
# thread.
|
|
93
113
|
#
|
|
94
114
|
# @return [void]
|
|
95
115
|
#
|
|
96
116
|
def on_close(_event)
|
|
97
117
|
@messages.close
|
|
98
|
-
|
|
118
|
+
queue_encoded
|
|
119
|
+
@outgoing.close
|
|
99
120
|
@thread.kill
|
|
100
121
|
end
|
|
101
122
|
|
|
102
123
|
#
|
|
103
|
-
# Serializes a CDP command to JSON and
|
|
104
|
-
# frame.
|
|
124
|
+
# Serializes a CDP command to JSON and queues it as a websocket text
|
|
125
|
+
# frame. Returns without waiting for the frame to be written.
|
|
105
126
|
#
|
|
106
127
|
# @param [Hash] data
|
|
107
128
|
# The message to send, must include an `:id` key.
|
|
108
129
|
#
|
|
109
130
|
# @param [Numeric, nil] timeout
|
|
110
|
-
# How long the
|
|
111
|
-
#
|
|
131
|
+
# How long the frame may wait in the queue. A frame still queued after
|
|
132
|
+
# that is dropped, nobody waits for its response anymore. `nil` keeps it
|
|
133
|
+
# until it's written.
|
|
112
134
|
#
|
|
113
135
|
# @return [void]
|
|
114
136
|
#
|
|
115
|
-
# @raise [Ferrum::DeadBrowserError]
|
|
116
|
-
# If the browser accepted no data for `timeout` seconds. The connection
|
|
117
|
-
# is closed, since the frame was cut off midway.
|
|
118
|
-
#
|
|
119
137
|
def send_message(data, timeout = nil)
|
|
120
138
|
@screenshot_commands[data[:id]] = true if SKIP_LOGGING_SCREENSHOTS
|
|
121
139
|
|
|
122
140
|
json = data.to_json
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
@driver.text(json)
|
|
126
|
-
ensure
|
|
127
|
-
@write_timeout = nil
|
|
128
|
-
end
|
|
141
|
+
deadline = Utils::ElapsedTime.monotonic_time + timeout if timeout
|
|
142
|
+
drive(deadline) { @driver.text(json) }
|
|
129
143
|
@logger&.puts("\n\n▶ #{Utils::ElapsedTime.elapsed_time} #{json}")
|
|
130
144
|
end
|
|
131
145
|
|
|
132
146
|
#
|
|
133
|
-
#
|
|
134
|
-
#
|
|
135
|
-
#
|
|
136
|
-
#
|
|
137
|
-
# A browser that stops reading its socket (hung, or stopped) would block
|
|
138
|
-
# a plain write forever, holding the lock every other command needs to be
|
|
139
|
-
# sent. So the write gives up once the browser accepts no data for the
|
|
140
|
-
# timeout of the command being sent.
|
|
147
|
+
# Collects the raw bytes of a frame. Called by `websocket-driver` to
|
|
148
|
+
# emit frames, one call per frame, always from within the driver call
|
|
149
|
+
# that queues them for the writer thread.
|
|
141
150
|
#
|
|
142
151
|
# @param [String] data
|
|
143
|
-
# The raw bytes
|
|
152
|
+
# The raw bytes of one frame.
|
|
144
153
|
#
|
|
145
154
|
# @return [void]
|
|
146
155
|
#
|
|
147
|
-
# @raise [Ferrum::DeadBrowserError]
|
|
148
|
-
# If the browser accepted no data for the command's timeout.
|
|
149
|
-
#
|
|
150
156
|
def write(data)
|
|
151
|
-
|
|
152
|
-
written = @sock.write_nonblock(data, exception: false)
|
|
153
|
-
if written.is_a?(Integer)
|
|
154
|
-
data = data.byteslice(written..)
|
|
155
|
-
elsif !wait_for_socket(written)
|
|
156
|
-
abandon_stalled_write
|
|
157
|
-
end
|
|
158
|
-
end
|
|
159
|
-
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
|
|
160
|
-
@messages.close
|
|
157
|
+
@encoded << data
|
|
161
158
|
end
|
|
162
159
|
|
|
163
160
|
#
|
|
@@ -166,22 +163,23 @@ module Ferrum
|
|
|
166
163
|
# @return [void]
|
|
167
164
|
#
|
|
168
165
|
def close
|
|
169
|
-
|
|
166
|
+
drive { @driver.close }
|
|
170
167
|
end
|
|
171
168
|
|
|
172
169
|
#
|
|
173
|
-
#
|
|
174
|
-
# completed.
|
|
170
|
+
# Stops the writer and closes the underlying socket whether or not the
|
|
171
|
+
# close handshake ever completed.
|
|
175
172
|
#
|
|
176
|
-
# {#close} only
|
|
177
|
-
# {#on_close}
|
|
178
|
-
# its own. A browser that is already dead never answers, so without this
|
|
173
|
+
# {#close} only queues a close frame; the socket itself is closed by the
|
|
174
|
+
# writer once {#on_close} runs, when the browser answers with a close
|
|
175
|
+
# frame of its own. A browser that is already dead never answers, so without this
|
|
179
176
|
# the socket stays open until GC finalizes it -- long enough to exhaust
|
|
180
177
|
# the process's file descriptors when browsers are restarted in a loop.
|
|
181
178
|
#
|
|
182
179
|
# @return [void]
|
|
183
180
|
#
|
|
184
181
|
def force_close
|
|
182
|
+
@outgoing.close
|
|
185
183
|
@sock.close unless @sock.closed?
|
|
186
184
|
rescue IOError, SystemCallError
|
|
187
185
|
# nop
|
|
@@ -189,16 +187,50 @@ module Ferrum
|
|
|
189
187
|
|
|
190
188
|
private
|
|
191
189
|
|
|
192
|
-
def
|
|
193
|
-
|
|
194
|
-
|
|
190
|
+
def drive(deadline = nil)
|
|
191
|
+
@mutex.synchronize do
|
|
192
|
+
yield
|
|
193
|
+
queue_encoded(deadline)
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def queue_encoded(deadline = nil)
|
|
198
|
+
@encoded.each { |data| @outgoing << [data, deadline] }
|
|
199
|
+
rescue ClosedQueueError
|
|
200
|
+
@messages.close
|
|
201
|
+
ensure
|
|
202
|
+
@encoded.clear
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def start_writer
|
|
206
|
+
@writer = Utils::Thread.spawn do
|
|
207
|
+
while (frame = @outgoing.pop)
|
|
208
|
+
data, deadline = frame
|
|
209
|
+
write_frame(data) unless deadline && Utils::ElapsedTime.monotonic_time > deadline
|
|
210
|
+
end
|
|
211
|
+
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EBADF, IOError # rubocop:disable Lint/ShadowedException
|
|
212
|
+
# The browser went away, nothing to report.
|
|
213
|
+
rescue StandardError => e
|
|
214
|
+
warn("Ferrum: websocket writer stopped, #{e.class}: #{e.message}\n #{e.backtrace&.first}")
|
|
215
|
+
ensure
|
|
216
|
+
@messages.close
|
|
217
|
+
force_close
|
|
218
|
+
end
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def write_frame(data)
|
|
222
|
+
0.step(data.bytesize - 1, WRITE_CHUNK_SIZE) do |offset|
|
|
223
|
+
watchdog = Concurrent::ScheduledTask.execute(@write_timeout) { abandon_stalled_write } if @write_timeout
|
|
224
|
+
@sock.write(data.byteslice(offset, WRITE_CHUNK_SIZE))
|
|
225
|
+
ensure
|
|
226
|
+
watchdog&.cancel
|
|
227
|
+
end
|
|
195
228
|
end
|
|
196
229
|
|
|
197
230
|
def abandon_stalled_write
|
|
198
|
-
|
|
231
|
+
@close_reason = "Browser accepted no data for #{@write_timeout}s and is likely stuck, connection closed"
|
|
199
232
|
@messages.close
|
|
200
|
-
|
|
201
|
-
raise DeadBrowserError, "Browser accepted no data for #{timeout}s and is likely stuck, connection closed"
|
|
233
|
+
force_close
|
|
202
234
|
end
|
|
203
235
|
|
|
204
236
|
def start
|
|
@@ -207,9 +239,9 @@ module Ferrum
|
|
|
207
239
|
data = @sock.readpartial(512)
|
|
208
240
|
break unless data
|
|
209
241
|
|
|
210
|
-
|
|
242
|
+
drive { @driver.parse(data) }
|
|
211
243
|
end
|
|
212
|
-
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError # rubocop:disable Lint/ShadowedException
|
|
244
|
+
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, Errno::EBADF, IOError # rubocop:disable Lint/ShadowedException
|
|
213
245
|
# The browser went away, nothing to report.
|
|
214
246
|
rescue StandardError => e
|
|
215
247
|
warn("Ferrum: websocket reader stopped, #{e.class}: #{e.message}\n #{e.backtrace&.first}")
|
|
@@ -223,14 +255,12 @@ module Ferrum
|
|
|
223
255
|
rescue JSON::NestingError
|
|
224
256
|
# nop
|
|
225
257
|
rescue JSON::ParserError
|
|
226
|
-
|
|
258
|
+
safely_parse_lone_surrogates(data)
|
|
227
259
|
end
|
|
228
260
|
|
|
229
|
-
def
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
escaped_data = unescaped_unicode.encode("UTF-8", "UTF-8", undef: :replace, invalid: :replace, replace: "?")
|
|
233
|
-
JSON.parse(escaped_data, max_nesting: false)
|
|
261
|
+
def safely_parse_lone_surrogates(data)
|
|
262
|
+
replaced = data.gsub(ESCAPE_OR_LONE_SURROGATE) { |escape| ::Regexp.last_match(1) ? "\\ufffd" : escape }
|
|
263
|
+
JSON.parse(replaced, max_nesting: false)
|
|
234
264
|
rescue JSON::ParserError
|
|
235
265
|
# nop
|
|
236
266
|
end
|
data/lib/ferrum/client.rb
CHANGED
|
@@ -160,7 +160,7 @@ module Ferrum
|
|
|
160
160
|
@ws_url = ws_url
|
|
161
161
|
@options = options
|
|
162
162
|
@pendings = Concurrent::Hash.new
|
|
163
|
-
@ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger)
|
|
163
|
+
@ws = WebSocket.new(ws_url, options.ws_max_receive_size, options.logger, options.protocol_timeout)
|
|
164
164
|
@subscriber = Subscriber.new
|
|
165
165
|
|
|
166
166
|
start
|
|
@@ -208,8 +208,8 @@ module Ferrum
|
|
|
208
208
|
# Whether to return immediately instead of waiting for a response.
|
|
209
209
|
#
|
|
210
210
|
# @param [Numeric, nil] timeout
|
|
211
|
-
# How long to wait for the
|
|
212
|
-
#
|
|
211
|
+
# How long to wait for the response, counting the time the message waits
|
|
212
|
+
# to be sent. Defaults to `protocol_timeout`.
|
|
213
213
|
#
|
|
214
214
|
# @return [Boolean, Hash]
|
|
215
215
|
# `true` when sent asynchronously, otherwise the parsed `"result"`
|
|
@@ -222,7 +222,7 @@ module Ferrum
|
|
|
222
222
|
@ws.send_message(message, timeout)
|
|
223
223
|
true
|
|
224
224
|
else
|
|
225
|
-
raise
|
|
225
|
+
raise dead_browser_error if @ws.messages.closed?
|
|
226
226
|
|
|
227
227
|
pending = Concurrent::IVar.new
|
|
228
228
|
@pendings[message[:id]] = pending
|
|
@@ -233,7 +233,7 @@ module Ferrum
|
|
|
233
233
|
@pendings.delete(message[:id])
|
|
234
234
|
end
|
|
235
235
|
|
|
236
|
-
raise
|
|
236
|
+
raise dead_browser_error if data.nil? && @ws.messages.closed?
|
|
237
237
|
raise TimeoutError.new(message[:method], session_id: message[:sessionId]) unless data
|
|
238
238
|
|
|
239
239
|
error, response = data.values_at("error", "result")
|
|
@@ -365,6 +365,10 @@ module Ferrum
|
|
|
365
365
|
@pendings.each_value { |pending| pending.try_set(nil) }
|
|
366
366
|
end
|
|
367
367
|
|
|
368
|
+
def dead_browser_error
|
|
369
|
+
@ws.close_reason ? DeadBrowserError.new(@ws.close_reason) : DeadBrowserError.new
|
|
370
|
+
end
|
|
371
|
+
|
|
368
372
|
# Locked so two concurrent commands never share an id and read each
|
|
369
373
|
# other's responses from @pendings.
|
|
370
374
|
def next_command_id
|
|
@@ -376,7 +380,8 @@ module Ferrum
|
|
|
376
380
|
# Node has disappeared while we were trying to get it
|
|
377
381
|
when "No node with given id found",
|
|
378
382
|
"Could not find node with given id",
|
|
379
|
-
"Inspected target navigated or closed"
|
|
383
|
+
"Inspected target navigated or closed",
|
|
384
|
+
"Argument should belong to the same JavaScript world as target object"
|
|
380
385
|
raise NodeNotFoundError, error
|
|
381
386
|
# Context is lost, page is reloading
|
|
382
387
|
when "Cannot find context with specified id"
|
data/lib/ferrum/frame/dom.rb
CHANGED
|
@@ -116,7 +116,8 @@ module Ferrum
|
|
|
116
116
|
# Returns the element in which the window is embedded.
|
|
117
117
|
#
|
|
118
118
|
# @return [Node, nil]
|
|
119
|
-
# The element in which the window is embedded
|
|
119
|
+
# The element in which the window is embedded, `nil` for the main frame
|
|
120
|
+
# and for a frame from a different origin than its parent.
|
|
120
121
|
#
|
|
121
122
|
# @example
|
|
122
123
|
# browser.go_to("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/iframe")
|
data/lib/ferrum/mouse.rb
CHANGED
|
@@ -152,6 +152,7 @@ module Ferrum
|
|
|
152
152
|
from_y = @y
|
|
153
153
|
@x = x
|
|
154
154
|
@y = y
|
|
155
|
+
button = pressed_button
|
|
155
156
|
|
|
156
157
|
steps.times do |i|
|
|
157
158
|
new_x = from_x + ((@x - from_x) * ((i + 1) / steps.to_f))
|
|
@@ -162,6 +163,7 @@ module Ferrum
|
|
|
162
163
|
type: "mouseMoved",
|
|
163
164
|
x: new_x,
|
|
164
165
|
y: new_y,
|
|
166
|
+
button: button,
|
|
165
167
|
buttons: @buttons)
|
|
166
168
|
end
|
|
167
169
|
|
|
@@ -185,6 +187,10 @@ module Ferrum
|
|
|
185
187
|
button
|
|
186
188
|
end
|
|
187
189
|
|
|
190
|
+
def pressed_button
|
|
191
|
+
BUTTON_MASKS.find { |_name, mask| mask.positive? && @buttons.anybits?(mask) }&.first || "none"
|
|
192
|
+
end
|
|
193
|
+
|
|
188
194
|
def register_event_button(type, button)
|
|
189
195
|
case type
|
|
190
196
|
when "mousePressed"
|
data/lib/ferrum/version.rb
CHANGED