ferrum 0.15 → 0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +103 -14
- data/lib/ferrum/browser/command.rb +0 -4
- data/lib/ferrum/browser/options/base.rb +9 -0
- data/lib/ferrum/browser/options/chrome.rb +6 -10
- data/lib/ferrum/browser/options.rb +2 -1
- data/lib/ferrum/browser/process.rb +9 -11
- data/lib/ferrum/browser.rb +45 -2
- data/lib/ferrum/client/subscriber.rb +9 -3
- data/lib/ferrum/client/web_socket.rb +26 -5
- data/lib/ferrum/client.rb +12 -2
- data/lib/ferrum/context.rb +34 -11
- data/lib/ferrum/contexts.rb +20 -4
- data/lib/ferrum/cookies.rb +27 -0
- data/lib/ferrum/errors.rb +8 -2
- data/lib/ferrum/frame/dom.rb +18 -2
- data/lib/ferrum/frame.rb +17 -0
- data/lib/ferrum/keyboard.rb +0 -1
- data/lib/ferrum/mouse.rb +43 -7
- data/lib/ferrum/network/exchange.rb +33 -3
- data/lib/ferrum/network/request.rb +9 -0
- data/lib/ferrum/network.rb +52 -17
- data/lib/ferrum/node.rb +12 -0
- data/lib/ferrum/page/frames.rb +8 -6
- data/lib/ferrum/page/screencast.rb +102 -0
- data/lib/ferrum/page/screenshot.rb +19 -5
- data/lib/ferrum/page.rb +33 -2
- data/lib/ferrum/target.rb +10 -1
- data/lib/ferrum/utils/elapsed_time.rb +4 -0
- data/lib/ferrum/version.rb +1 -1
- metadata +18 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f586cea5a9987e2a53bfb860fb503d36fc9e2f4e16b7288dafe68b2e62f20ede
|
4
|
+
data.tar.gz: a33d17e091b0d7b506f0934dd19f056ab15c72891659cc227f03bccad8d90cdd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 031a4dcb48ba0d22c1070ab8e905881f07369e7c19b61f03ef747d4ff3813be097a8f8bc65f692f4ed186687ba1e4e83c9de4482939a963b2541364c0d745b16
|
7
|
+
data.tar.gz: 207975dde463249331095cada046454110ee80d8a4a42fdc7f51898ad514e0aa5395946e037abd5c5599a7fa2d570c279925555822d432c4470c6e955f37b85b
|
data/README.md
CHANGED
@@ -25,9 +25,6 @@ going to crawl sites you better use Ferrum or
|
|
25
25
|
* [Vessel](https://github.com/rubycdp/vessel) high-level web crawling framework
|
26
26
|
based on Ferrum and Mechanize.
|
27
27
|
|
28
|
-
The development is done in [](https://jb.gg/ruby)
|
29
|
-
provided by [OSS license](https://jb.gg/OpenSourceSupport).
|
30
|
-
|
31
28
|
## Index
|
32
29
|
|
33
30
|
* [Install](https://github.com/rubycdp/ferrum#install)
|
@@ -37,6 +34,7 @@ provided by [OSS license](https://jb.gg/OpenSourceSupport).
|
|
37
34
|
* [Navigation](https://github.com/rubycdp/ferrum#navigation)
|
38
35
|
* [Finders](https://github.com/rubycdp/ferrum#finders)
|
39
36
|
* [Screenshots](https://github.com/rubycdp/ferrum#screenshots)
|
37
|
+
* [Screencast](https://github.com/rubycdp/ferrum#screencast)
|
40
38
|
* [Network](https://github.com/rubycdp/ferrum#network)
|
41
39
|
* [Downloads](https://github.com/rubycdp/ferrum#downloads)
|
42
40
|
* [Proxy](https://github.com/rubycdp/ferrum#proxy)
|
@@ -139,7 +137,7 @@ browser.quit
|
|
139
137
|
In docker as root you must pass the no-sandbox browser option:
|
140
138
|
|
141
139
|
```ruby
|
142
|
-
Ferrum::Browser.new(browser_options: {
|
140
|
+
Ferrum::Browser.new(browser_options: { "no-sandbox": nil })
|
143
141
|
```
|
144
142
|
|
145
143
|
It has also been reported that the Chrome process repeatedly crashes when running inside a Docker container on an M1 Mac preventing Ferrum from working. Ferrum should work as expected when deployed to a Docker container on a non-M1 Mac.
|
@@ -153,8 +151,8 @@ Ferrum::Browser.new(options)
|
|
153
151
|
```
|
154
152
|
|
155
153
|
* options `Hash`
|
156
|
-
* `:headless` (
|
157
|
-
|
154
|
+
* `:headless` (Boolean) - Set browser as headless or not, `true` by default.
|
155
|
+
* `:incognito` (Boolean) - Create an incognito profile for the browser startup window, `true` by default.
|
158
156
|
* `:xvfb` (Boolean) - Run browser in a virtual framebuffer, `false` by default.
|
159
157
|
* `:flatten` (Boolean) - Use one websocket connection to the browser and all the pages in flatten mode.
|
160
158
|
* `:window_size` (Array) - The dimensions of the browser window in which to
|
@@ -188,7 +186,7 @@ Ferrum::Browser.new(options)
|
|
188
186
|
* `:url` (String) - URL for a running instance of Chrome. If this is set, a
|
189
187
|
browser process will not be spawned.
|
190
188
|
* `:ws_url` (String) - Websocket url for a running instance of Chrome. If this is set, a
|
191
|
-
|
189
|
+
browser process will not be spawned. It's higher priority than `:url`, setting both doesn't make sense.
|
192
190
|
* `:process_timeout` (Integer) - How long to wait for the Chrome process to
|
193
191
|
respond on startup.
|
194
192
|
* `:ws_max_receive_size` (Integer) - How big messages to accept from Chrome
|
@@ -398,7 +396,7 @@ Saves screenshot on a disk or returns it as base64.
|
|
398
396
|
`:binary` automatically
|
399
397
|
* :encoding `Symbol` `:base64` | `:binary` you can set it to return image as
|
400
398
|
Base64
|
401
|
-
* :format `String` "jpeg" | "png"
|
399
|
+
* :format `String` "jpeg" ("jpg") | "png" | "webp"
|
402
400
|
* :quality `Integer` 0-100 works for jpeg only
|
403
401
|
* :full `Boolean` whether you need full page screenshot or a viewport
|
404
402
|
* :selector `String` css selector for given element, optional
|
@@ -419,7 +417,7 @@ page.screenshot(path: "google.jpg") # => 30902
|
|
419
417
|
# Save to Base64 the whole page not only viewport and reduce quality
|
420
418
|
page.screenshot(full: true, quality: 60, encoding: :base64) # "iVBORw0KGgoAAAANSUhEUgAABAAAAAMACAYAAAC6uhUNAAAAAXNSR0IArs4c6Q...
|
421
419
|
# Save on the disk with the selected element in PNG
|
422
|
-
page.screenshot(path: "google.png", selector:
|
420
|
+
page.screenshot(path: "google.png", selector: "textarea") # => 11340
|
423
421
|
# Save to Base64 with an area of the page in PNG
|
424
422
|
page.screenshot(path: "google.png", area: { x: 0, y: 0, width: 400, height: 300 }) # => 54239
|
425
423
|
# Save with specific background color
|
@@ -461,6 +459,58 @@ page.go_to("https://google.com/")
|
|
461
459
|
page.mhtml(path: "google.mhtml") # => 87742
|
462
460
|
```
|
463
461
|
|
462
|
+
## Screencast
|
463
|
+
|
464
|
+
#### start_screencast(\*\*options) { |data, metadata, session_id| ... }
|
465
|
+
|
466
|
+
Starts sending frames to record screencast to the given block.
|
467
|
+
|
468
|
+
* options `Hash`
|
469
|
+
* :format `Symbol` `:jpeg` | `:png` The format the image should be returned in.
|
470
|
+
* :quality `Integer` The image quality. **Note:** 0-100 works for JPEG only.
|
471
|
+
* :max_width `Integer` Maximum screencast frame width.
|
472
|
+
* :max_height `Integer` Maximum screencast frame height.
|
473
|
+
* :every_nth_frame `Integer` Send every n-th frame.
|
474
|
+
|
475
|
+
* Block inputs:
|
476
|
+
* data `String` Base64-encoded compressed image.
|
477
|
+
* metadata `Hash` Screencast frame metadata.
|
478
|
+
* "offsetTop" `Integer` Top offset in DIP.
|
479
|
+
* "pageScaleFactor" `Integer` Page scale factor.
|
480
|
+
* "deviceWidth" `Integer` Device screen width in DIP.
|
481
|
+
* "deviceHeight" `Integer` Device screen height in DIP.
|
482
|
+
* "scrollOffsetX" `Integer` Position of horizontal scroll in CSS pixels.
|
483
|
+
* "scrollOffsetY" `Integer` Position of vertical scroll in CSS pixels.
|
484
|
+
* "timestamp" `Float` (optional) Frame swap timestamp in seconds since Unix epoch.
|
485
|
+
* session_id `Integer` Frame number.
|
486
|
+
|
487
|
+
```ruby
|
488
|
+
require "base64"
|
489
|
+
|
490
|
+
page.go_to("https://apple.com/ipad")
|
491
|
+
|
492
|
+
page.start_screencast(format: :jpeg, quality: 75) do |data, metadata|
|
493
|
+
timestamp = (metadata["timestamp"] * 1000).to_i
|
494
|
+
File.binwrite("image_#{timestamp}.jpg", Base64.decode64(data))
|
495
|
+
end
|
496
|
+
|
497
|
+
sleep 10
|
498
|
+
|
499
|
+
page.stop_screencast
|
500
|
+
```
|
501
|
+
|
502
|
+
> ### 📝 NOTE
|
503
|
+
>
|
504
|
+
> Chrome only sends new frames while page content is changing. For example, if
|
505
|
+
> there is an animation or a video on the page, Chrome sends frames at the rate
|
506
|
+
> requested. On the other hand, if the page is nothing but a wall of static text,
|
507
|
+
> Chrome sends frames while the page renders. Once Chrome has finished rendering
|
508
|
+
> the page, it sends no more frames until something changes (e.g., navigating to
|
509
|
+
> another location).
|
510
|
+
|
511
|
+
#### stop_screencast
|
512
|
+
|
513
|
+
Stops sending frames.
|
464
514
|
|
465
515
|
## Network
|
466
516
|
|
@@ -504,9 +554,9 @@ page.go_to("https://github.com/")
|
|
504
554
|
page.network.status # => 200
|
505
555
|
```
|
506
556
|
|
507
|
-
#### wait_for_idle(\*\*options)
|
557
|
+
#### wait_for_idle(\*\*options) : `Boolean`
|
508
558
|
|
509
|
-
Waits for network idle
|
559
|
+
Waits for network idle, returns `true` in case of success and `false` if there are still connections.
|
510
560
|
|
511
561
|
* options `Hash`
|
512
562
|
* :connections `Integer` how many connections are allowed for network to be
|
@@ -519,7 +569,17 @@ Waits for network idle or raises `Ferrum::TimeoutError` error
|
|
519
569
|
```ruby
|
520
570
|
page.go_to("https://example.com/")
|
521
571
|
page.at_xpath("//a[text() = 'No UI changes button']").click
|
522
|
-
page.network.wait_for_idle
|
572
|
+
page.network.wait_for_idle # => true
|
573
|
+
```
|
574
|
+
|
575
|
+
#### wait_for_idle!(\*\*options)
|
576
|
+
|
577
|
+
Waits for network idle or raises `Ferrum::TimeoutError` error. Accepts same arguments as `wait_for_idle`.
|
578
|
+
|
579
|
+
```ruby
|
580
|
+
page.go_to("https://example.com/")
|
581
|
+
page.at_xpath("//a[text() = 'No UI changes button']").click
|
582
|
+
page.network.wait_for_idle! # might raise an error
|
523
583
|
```
|
524
584
|
|
525
585
|
#### clear(type)
|
@@ -872,6 +932,25 @@ Removes all cookies for current page
|
|
872
932
|
page.cookies.clear # => true
|
873
933
|
```
|
874
934
|
|
935
|
+
#### store(path) : `Boolean`
|
936
|
+
|
937
|
+
Stores all cookies of current page in a file.
|
938
|
+
|
939
|
+
```ruby
|
940
|
+
# Cookies are saved into cookies.yml
|
941
|
+
page.cookies.store # => 15657
|
942
|
+
```
|
943
|
+
|
944
|
+
#### load(path) : `Boolean`
|
945
|
+
|
946
|
+
Loads all cookies from the file and sets them for current page.
|
947
|
+
|
948
|
+
```ruby
|
949
|
+
# Cookies are loaded from cookies.yml
|
950
|
+
page.cookies.load # => true
|
951
|
+
```
|
952
|
+
|
953
|
+
|
875
954
|
## Headers
|
876
955
|
|
877
956
|
`page.headers`
|
@@ -1059,9 +1138,17 @@ Frame's unique id.
|
|
1059
1138
|
|
1060
1139
|
Parent frame id if this one is nested in another one.
|
1061
1140
|
|
1141
|
+
#### parent : `Frame | nil`
|
1142
|
+
|
1143
|
+
Parent frame if this one is nested in another one.
|
1144
|
+
|
1145
|
+
#### frame_element : `Node | nil`
|
1146
|
+
|
1147
|
+
Returns the element in which the window is embedded.
|
1148
|
+
|
1062
1149
|
#### execution_id : `Integer`
|
1063
1150
|
|
1064
|
-
Execution context id which is used by JS, each frame has
|
1151
|
+
Execution context id which is used by JS, each frame has its own context in
|
1065
1152
|
which JS evaluates.
|
1066
1153
|
|
1067
1154
|
#### name : `String | nil`
|
@@ -1243,6 +1330,8 @@ frame.at_css("//a[text() = 'Log in']") # => Node
|
|
1243
1330
|
#### select
|
1244
1331
|
#### scroll_into_view
|
1245
1332
|
#### in_viewport?(of: `Node | nil`) : `Boolean`
|
1333
|
+
#### remove
|
1334
|
+
#### exists?
|
1246
1335
|
|
1247
1336
|
(chainable) Selects options by passed attribute.
|
1248
1337
|
|
@@ -1294,7 +1383,7 @@ Closes browser tabs opened by the `Browser` instance.
|
|
1294
1383
|
|
1295
1384
|
```ruby
|
1296
1385
|
# connect to a long-running Chrome process
|
1297
|
-
browser = Ferrum::Browser.new(url:
|
1386
|
+
browser = Ferrum::Browser.new(url: "http://localhost:9222")
|
1298
1387
|
|
1299
1388
|
browser.go_to("https://github.com/")
|
1300
1389
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "singleton"
|
4
|
+
require "open3"
|
4
5
|
|
5
6
|
module Ferrum
|
6
7
|
class Browser
|
@@ -12,6 +13,14 @@ module Ferrum
|
|
12
13
|
instance
|
13
14
|
end
|
14
15
|
|
16
|
+
# @return [String, nil]
|
17
|
+
def self.version
|
18
|
+
out, = Open3.capture2(instance.detect_path, "--version")
|
19
|
+
out.strip
|
20
|
+
rescue Errno::ENOENT
|
21
|
+
nil
|
22
|
+
end
|
23
|
+
|
15
24
|
def to_h
|
16
25
|
self.class::DEFAULT_OPTIONS
|
17
26
|
end
|
@@ -38,8 +38,9 @@ module Ferrum
|
|
38
38
|
"safebrowsing-disable-auto-update" => nil,
|
39
39
|
"password-store" => "basic",
|
40
40
|
"no-startup-window" => nil,
|
41
|
-
"remote-allow-origins" => "*"
|
42
|
-
|
41
|
+
"remote-allow-origins" => "*",
|
42
|
+
"disable-blink-features" => "AutomationControlled"
|
43
|
+
# NOTE: --no-sandbox is not needed if you properly set up a user in the container.
|
43
44
|
# https://github.com/ebidel/lighthouse-ci/blob/master/builder/Dockerfile#L35-L40
|
44
45
|
# "no-sandbox" => nil,
|
45
46
|
}.freeze
|
@@ -75,15 +76,10 @@ module Ferrum
|
|
75
76
|
end
|
76
77
|
|
77
78
|
def merge_default(flags, options)
|
78
|
-
defaults =
|
79
|
-
when false
|
80
|
-
except("headless", "disable-gpu")
|
81
|
-
when "new"
|
82
|
-
except("headless").merge("headless" => "new")
|
83
|
-
end
|
84
|
-
|
79
|
+
defaults = except("headless", "disable-gpu") if options.headless == false
|
85
80
|
defaults ||= DEFAULT_OPTIONS
|
86
|
-
|
81
|
+
defaults.delete("no-startup-window") if options.incognito == false
|
82
|
+
# On Windows, the --disable-gpu flag is a temporary workaround for a few bugs.
|
87
83
|
# See https://bugs.chromium.org/p/chromium/issues/detail?id=737678 for more information.
|
88
84
|
defaults = defaults.merge("disable-gpu" => nil) if Utils::Platform.windows?
|
89
85
|
# Use Metal on Apple Silicon
|
@@ -14,7 +14,7 @@ module Ferrum
|
|
14
14
|
attr_reader :window_size, :logger, :ws_max_receive_size,
|
15
15
|
:js_errors, :base_url, :slowmo, :pending_connection_errors,
|
16
16
|
:url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
|
17
|
-
:save_path, :proxy, :port, :host, :headless, :browser_options,
|
17
|
+
:save_path, :proxy, :port, :host, :headless, :incognito, :browser_options,
|
18
18
|
:ignore_default_browser_options, :xvfb, :flatten
|
19
19
|
attr_accessor :timeout, :default_user_agent
|
20
20
|
|
@@ -27,6 +27,7 @@ module Ferrum
|
|
27
27
|
@window_size = @options.fetch(:window_size, WINDOW_SIZE)
|
28
28
|
@js_errors = @options.fetch(:js_errors, false)
|
29
29
|
@headless = @options.fetch(:headless, true)
|
30
|
+
@incognito = @options.fetch(:incognito, true)
|
30
31
|
@flatten = @options.fetch(:flatten, true)
|
31
32
|
@pending_connection_errors = @options.fetch(:pending_connection_errors, true)
|
32
33
|
@process_timeout = @options.fetch(:process_timeout, PROCESS_TIMEOUT)
|
@@ -9,6 +9,8 @@ require "ferrum/browser/options/base"
|
|
9
9
|
require "ferrum/browser/options/chrome"
|
10
10
|
require "ferrum/browser/options/firefox"
|
11
11
|
require "ferrum/browser/command"
|
12
|
+
require "ferrum/utils/elapsed_time"
|
13
|
+
require "ferrum/utils/platform"
|
12
14
|
|
13
15
|
module Ferrum
|
14
16
|
class Browser
|
@@ -62,15 +64,11 @@ module Ferrum
|
|
62
64
|
def initialize(options)
|
63
65
|
@pid = @xvfb = @user_data_dir = nil
|
64
66
|
|
65
|
-
if options.ws_url
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
if options.url
|
72
|
-
response = parse_json_version(options.url)
|
73
|
-
self.ws_url = response&.[]("webSocketDebuggerUrl")
|
67
|
+
if options.ws_url || options.url
|
68
|
+
# `:ws_url` option is higher priority than `:url`, parse versions
|
69
|
+
# and use it as a ws_url, otherwise use what has been parsed.
|
70
|
+
response = parse_json_version(options.ws_url || options.url)
|
71
|
+
self.ws_url = options.ws_url || response&.[]("webSocketDebuggerUrl")
|
74
72
|
return
|
75
73
|
end
|
76
74
|
|
@@ -180,7 +178,7 @@ module Ferrum
|
|
180
178
|
|
181
179
|
def close_io(*ios)
|
182
180
|
ios.each do |io|
|
183
|
-
io.close
|
181
|
+
io.close if io && !io.closed?
|
184
182
|
rescue IOError
|
185
183
|
raise unless RUBY_ENGINE == "jruby"
|
186
184
|
end
|
@@ -207,7 +205,7 @@ module Ferrum
|
|
207
205
|
@protocol_version = response["Protocol-Version"]
|
208
206
|
|
209
207
|
response
|
210
|
-
rescue
|
208
|
+
rescue JSON::ParserError
|
211
209
|
# nop
|
212
210
|
end
|
213
211
|
end
|
data/lib/ferrum/browser.rb
CHANGED
@@ -23,6 +23,7 @@ module Ferrum
|
|
23
23
|
headers cookies network downloads
|
24
24
|
mouse keyboard
|
25
25
|
screenshot pdf mhtml viewport_size device_pixel_ratio
|
26
|
+
start_screencast stop_screencast
|
26
27
|
frames frame_by main_frame
|
27
28
|
evaluate evaluate_on evaluate_async execute evaluate_func
|
28
29
|
add_script_tag add_style_tag bypass_csp
|
@@ -44,6 +45,9 @@ module Ferrum
|
|
44
45
|
# @option options [Boolean] :headless (true)
|
45
46
|
# Set browser as headless or not.
|
46
47
|
#
|
48
|
+
# @option options [Boolean] :incognito (true)
|
49
|
+
# Create an incognito profile for the browser startup window.
|
50
|
+
#
|
47
51
|
# @option options [Boolean] :xvfb (false)
|
48
52
|
# Run browser in a virtual framebuffer.
|
49
53
|
#
|
@@ -218,10 +222,22 @@ module Ferrum
|
|
218
222
|
@client = @process = @contexts = nil
|
219
223
|
end
|
220
224
|
|
225
|
+
#
|
226
|
+
# Crashes browser.
|
227
|
+
#
|
221
228
|
def crash
|
222
229
|
command("Browser.crash")
|
223
230
|
end
|
224
231
|
|
232
|
+
#
|
233
|
+
# Close browser gracefully.
|
234
|
+
#
|
235
|
+
# You should clean up resources/connections in ruby world manually, it's only a CDP command.
|
236
|
+
#
|
237
|
+
def close
|
238
|
+
command("Browser.close")
|
239
|
+
end
|
240
|
+
|
225
241
|
#
|
226
242
|
# Gets the version information from the browser.
|
227
243
|
#
|
@@ -233,8 +249,22 @@ module Ferrum
|
|
233
249
|
VersionInfo.new(command("Browser.getVersion"))
|
234
250
|
end
|
235
251
|
|
236
|
-
|
237
|
-
|
252
|
+
#
|
253
|
+
# Opens headless session in the browser devtools frontend.
|
254
|
+
#
|
255
|
+
# @return [void]
|
256
|
+
#
|
257
|
+
# @since 0.16
|
258
|
+
#
|
259
|
+
def debug(bind = nil)
|
260
|
+
::Process.spawn(process.path, debug_url)
|
261
|
+
|
262
|
+
bind ||= binding
|
263
|
+
if bind.respond_to?(:pry)
|
264
|
+
Pry.start(bind)
|
265
|
+
else
|
266
|
+
bind.irb
|
267
|
+
end
|
238
268
|
end
|
239
269
|
|
240
270
|
private
|
@@ -254,5 +284,18 @@ module Ferrum
|
|
254
284
|
raise
|
255
285
|
end
|
256
286
|
end
|
287
|
+
|
288
|
+
def debug_url
|
289
|
+
response = JSON.parse(Net::HTTP.get(URI(build_remote_debug_url(path: "/json"))))
|
290
|
+
|
291
|
+
devtools_frontend_path = response[0]&.[]("devtoolsFrontendUrl")
|
292
|
+
raise "Could not generate debug url for remote debugging session" unless devtools_frontend_path
|
293
|
+
|
294
|
+
build_remote_debug_url(path: devtools_frontend_path)
|
295
|
+
end
|
296
|
+
|
297
|
+
def build_remote_debug_url(path:)
|
298
|
+
"http://#{process.host}:#{process.port}#{path}"
|
299
|
+
end
|
257
300
|
end
|
258
301
|
end
|
@@ -8,7 +8,7 @@ module Ferrum
|
|
8
8
|
def initialize
|
9
9
|
@regular = Queue.new
|
10
10
|
@priority = Queue.new
|
11
|
-
@on = Concurrent::Hash.new
|
11
|
+
@on = Concurrent::Hash.new
|
12
12
|
|
13
13
|
start
|
14
14
|
end
|
@@ -22,7 +22,13 @@ module Ferrum
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def on(event, &block)
|
25
|
+
@on[event] ||= Concurrent::Array.new
|
25
26
|
@on[event] << block
|
27
|
+
@on[event].index(block)
|
28
|
+
end
|
29
|
+
|
30
|
+
def off(event, id)
|
31
|
+
@on[event].delete_at(id)
|
26
32
|
true
|
27
33
|
end
|
28
34
|
|
@@ -65,8 +71,8 @@ module Ferrum
|
|
65
71
|
method, session_id, params = message.values_at("method", "sessionId", "params")
|
66
72
|
event = SessionClient.event_name(method, session_id)
|
67
73
|
|
68
|
-
total = @on[event].
|
69
|
-
@on[event]
|
74
|
+
total = @on[event]&.size.to_i
|
75
|
+
@on[event]&.each_with_index do |block, index|
|
70
76
|
# In case of multiple callbacks we provide current index and total
|
71
77
|
block.call(params, index, total)
|
72
78
|
end
|
@@ -50,13 +50,17 @@ module Ferrum
|
|
50
50
|
end
|
51
51
|
|
52
52
|
def on_message(event)
|
53
|
-
data =
|
54
|
-
|
53
|
+
data = safely_parse_json(event.data)
|
54
|
+
# If we couldn't parse JSON data for some reason (parse error or deeply nested object) we
|
55
|
+
# don't push response to @messages. Worse that could happen we raise timeout error due to command didn't return
|
56
|
+
# anything or skip the background notification, but at least we don't crash the thread that crashes the main
|
57
|
+
# thread and the application.
|
58
|
+
@messages.push(data) if data
|
55
59
|
|
56
60
|
output = event.data
|
57
|
-
if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data
|
58
|
-
@screenshot_commands.delete(data
|
59
|
-
output.sub!(/{"data":"
|
61
|
+
if SKIP_LOGGING_SCREENSHOTS && @screenshot_commands[data&.dig("id")]
|
62
|
+
@screenshot_commands.delete(data&.dig("id"))
|
63
|
+
output.sub!(/{"data":"[^"]*"}/, %("Set FERRUM_LOGGING_SCREENSHOTS=true to see screenshots in Base64"))
|
60
64
|
end
|
61
65
|
|
62
66
|
@logger&.puts(" ◀ #{Utils::ElapsedTime.elapsed_time} #{output}\n")
|
@@ -100,6 +104,23 @@ module Ferrum
|
|
100
104
|
@messages.close
|
101
105
|
end
|
102
106
|
end
|
107
|
+
|
108
|
+
def safely_parse_json(data)
|
109
|
+
JSON.parse(data, max_nesting: false)
|
110
|
+
rescue JSON::NestingError
|
111
|
+
# nop
|
112
|
+
rescue JSON::ParserError
|
113
|
+
safely_parse_escaped_json(data)
|
114
|
+
end
|
115
|
+
|
116
|
+
def safely_parse_escaped_json(data)
|
117
|
+
unescaped_unicode =
|
118
|
+
data.gsub(/\\u([\da-fA-F]{4})/) { |_| [::Regexp.last_match(1)].pack("H*").unpack("n*").pack("U*") }
|
119
|
+
escaped_data = unescaped_unicode.encode("UTF-8", "UTF-8", undef: :replace, invalid: :replace, replace: "?")
|
120
|
+
JSON.parse(escaped_data, max_nesting: false)
|
121
|
+
rescue JSON::ParserError
|
122
|
+
# nop
|
123
|
+
end
|
103
124
|
end
|
104
125
|
end
|
105
126
|
end
|
data/lib/ferrum/client.rb
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "concurrent-ruby"
|
3
4
|
require "forwardable"
|
4
5
|
require "ferrum/client/subscriber"
|
5
6
|
require "ferrum/client/web_socket"
|
7
|
+
require "ferrum/utils/thread"
|
6
8
|
|
7
9
|
module Ferrum
|
8
10
|
class SessionClient
|
@@ -26,6 +28,10 @@ module Ferrum
|
|
26
28
|
@client.on(event_name(event), &block)
|
27
29
|
end
|
28
30
|
|
31
|
+
def off(event, id)
|
32
|
+
@client.off(event_name(event), id)
|
33
|
+
end
|
34
|
+
|
29
35
|
def subscribed?(event)
|
30
36
|
@client.subscribed?(event_name(event))
|
31
37
|
end
|
@@ -34,8 +40,8 @@ module Ferrum
|
|
34
40
|
@client.respond_to?(name, include_private)
|
35
41
|
end
|
36
42
|
|
37
|
-
def method_missing(name,
|
38
|
-
@client.send(name,
|
43
|
+
def method_missing(name, *args, **opts, &block)
|
44
|
+
@client.send(name, *args, **opts, &block)
|
39
45
|
end
|
40
46
|
|
41
47
|
def close
|
@@ -99,6 +105,10 @@ module Ferrum
|
|
99
105
|
@subscriber.on(event, &block)
|
100
106
|
end
|
101
107
|
|
108
|
+
def off(event, id)
|
109
|
+
@subscriber.off(event, id)
|
110
|
+
end
|
111
|
+
|
102
112
|
def subscribed?(event)
|
103
113
|
@subscriber.subscribed?(event)
|
104
114
|
end
|
data/lib/ferrum/context.rb
CHANGED
@@ -13,7 +13,7 @@ module Ferrum
|
|
13
13
|
@client = client
|
14
14
|
@contexts = contexts
|
15
15
|
@targets = Concurrent::Map.new
|
16
|
-
@pendings = Concurrent::
|
16
|
+
@pendings = Concurrent::Map.new
|
17
17
|
end
|
18
18
|
|
19
19
|
def default_target
|
@@ -25,11 +25,11 @@ module Ferrum
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def pages
|
28
|
-
@targets.values.map(&:page)
|
28
|
+
@targets.values.reject(&:iframe?).map(&:page)
|
29
29
|
end
|
30
30
|
|
31
31
|
# When we call `page` method on target it triggers ruby to connect to given
|
32
|
-
# page by WebSocket, if there are many opened windows but we need only one
|
32
|
+
# page by WebSocket, if there are many opened windows, but we need only one
|
33
33
|
# it makes more sense to get and connect to the needed one only which
|
34
34
|
# usually is the last one.
|
35
35
|
def windows(pos = nil, size = 1)
|
@@ -46,19 +46,27 @@ module Ferrum
|
|
46
46
|
end
|
47
47
|
|
48
48
|
def create_target
|
49
|
-
@client.command("Target.createTarget", browserContextId: @id, url: "about:blank")
|
50
|
-
target = @pendings.take(@client.timeout)
|
51
|
-
raise NoSuchTargetError unless target.is_a?(Target)
|
49
|
+
target_id = @client.command("Target.createTarget", browserContextId: @id, url: "about:blank")["targetId"]
|
52
50
|
|
53
|
-
|
51
|
+
new_pending = Concurrent::IVar.new
|
52
|
+
pending = @pendings.put_if_absent(target_id, new_pending) || new_pending
|
53
|
+
resolved = pending.value(@client.timeout)
|
54
|
+
raise NoSuchTargetError unless resolved
|
55
|
+
|
56
|
+
@pendings.delete(target_id)
|
57
|
+
@targets[target_id]
|
54
58
|
end
|
55
59
|
|
56
60
|
def add_target(params:, session_id: nil)
|
57
61
|
new_target = Target.new(@client, session_id, params)
|
58
|
-
|
59
|
-
target
|
60
|
-
@
|
61
|
-
|
62
|
+
# `put_if_absent` returns nil if added a new value or existing if there was one already
|
63
|
+
target = @targets.put_if_absent(new_target.id, new_target) || new_target
|
64
|
+
@default_target ||= target
|
65
|
+
|
66
|
+
new_pending = Concurrent::IVar.new
|
67
|
+
pending = @pendings.put_if_absent(target.id, new_pending) || new_pending
|
68
|
+
pending.try_set(true)
|
69
|
+
true
|
62
70
|
end
|
63
71
|
|
64
72
|
def update_target(target_id, params)
|
@@ -69,6 +77,21 @@ module Ferrum
|
|
69
77
|
@targets.delete(target_id)
|
70
78
|
end
|
71
79
|
|
80
|
+
def attach_target(target_id)
|
81
|
+
target = @targets[target_id]
|
82
|
+
raise NoSuchTargetError unless target
|
83
|
+
|
84
|
+
session = @client.command("Target.attachToTarget", targetId: target_id, flatten: true)
|
85
|
+
target.session_id = session["sessionId"]
|
86
|
+
true
|
87
|
+
end
|
88
|
+
|
89
|
+
def find_target
|
90
|
+
@targets.each_value { |t| return t if yield(t) }
|
91
|
+
|
92
|
+
nil
|
93
|
+
end
|
94
|
+
|
72
95
|
def close_targets_connection
|
73
96
|
@targets.each_value do |target|
|
74
97
|
next unless target.connected?
|