ferrum 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +63 -1
- data/lib/ferrum/browser/binary.rb +1 -3
- data/lib/ferrum/browser/command.rb +0 -2
- data/lib/ferrum/browser/options/base.rb +5 -6
- data/lib/ferrum/browser/options/chrome.rb +0 -3
- data/lib/ferrum/browser/options.rb +33 -12
- data/lib/ferrum/browser/process/killer.rb +188 -0
- data/lib/ferrum/browser/process/output.rb +143 -0
- data/lib/ferrum/browser/process.rb +122 -144
- data/lib/ferrum/browser.rb +24 -30
- data/lib/ferrum/client/subscriber.rb +7 -2
- data/lib/ferrum/client/web_socket.rb +124 -43
- data/lib/ferrum/client.rb +36 -9
- data/lib/ferrum/console/location.rb +42 -0
- data/lib/ferrum/console/message.rb +230 -0
- data/lib/ferrum/console.rb +157 -0
- data/lib/ferrum/context.rb +41 -20
- data/lib/ferrum/contexts.rb +110 -28
- data/lib/ferrum/downloads.rb +2 -1
- data/lib/ferrum/errors.rb +50 -4
- data/lib/ferrum/frame/dom.rb +100 -45
- data/lib/ferrum/frame/runtime/legacy.rb +336 -0
- data/lib/ferrum/frame/runtime.rb +417 -186
- data/lib/ferrum/frame/script.rb +289 -0
- data/lib/ferrum/frame.rb +33 -4
- data/lib/ferrum/interceptable.rb +35 -4
- data/lib/ferrum/isolated_world.rb +94 -0
- data/lib/ferrum/keyboard.rb +95 -8
- data/lib/ferrum/log/annotated.rb +77 -0
- data/lib/ferrum/log/default.rb +55 -0
- data/lib/ferrum/log/json.rb +32 -0
- data/lib/ferrum/log/paint.rb +28 -0
- data/lib/ferrum/log.rb +247 -0
- data/lib/ferrum/mouse.rb +22 -8
- data/lib/ferrum/node.rb +127 -52
- data/lib/ferrum/page/frames.rb +10 -3
- data/lib/ferrum/page/screenshot.rb +11 -10
- data/lib/ferrum/page/stream.rb +16 -1
- data/lib/ferrum/page.rb +117 -58
- data/lib/ferrum/proxy.rb +8 -7
- data/lib/ferrum/remote_object.rb +75 -0
- data/lib/ferrum/target.rb +14 -3
- data/lib/ferrum/utils/deprecate.rb +77 -0
- data/lib/ferrum/utils/thread.rb +3 -1
- data/lib/ferrum/version.rb +1 -1
- data/lib/ferrum/worker.rb +8 -0
- data/lib/ferrum.rb +4 -1
- metadata +18 -4
- data/lib/ferrum/browser/options/firefox.rb +0 -70
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 18ff9dece365985b7cb1033d7ec3c9913602d3da49d60ceaef432d1c569c1b86
|
|
4
|
+
data.tar.gz: 18b3d49c63a924a77526eff46cce0f2dea4696580ca19cc25fb9a640f9e4bf62
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: dd8a1c24c02303a636c557bc016fd4cb0fb4cd16cf4bfe0e3c3bb7f00f92adda3b1f4568b7692002e1b1e4b55a0e8d2e1772d840183ec9d0eecbad710470bce8
|
|
7
|
+
data.tar.gz: 164a77db028af9018846f9f3f5d1451755cf0d6ed495ea2c0424235b150fc388985c6b57742b9dbfbf88dd42c70823c24a18e8ca137906556fa4dda3d9beb8fe
|
data/README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
## [Documentation](https://docs.rubycdp.com/docs/ferrum/introduction)
|
|
4
4
|
|
|
5
5
|
<img align="right"
|
|
6
|
-
width="
|
|
6
|
+
width="160" height="160"
|
|
7
7
|
alt="Ferrum logo"
|
|
8
8
|
src="https://raw.githubusercontent.com/rubycdp/ferrum/main/logo.svg?sanitize=true">
|
|
9
9
|
|
|
@@ -18,6 +18,8 @@ browsers.
|
|
|
18
18
|
* [Cuprite](https://github.com/rubycdp/cuprite) is a pure Ruby driver for [Capybara](https://github.com/teamcapybara/capybara) based on Ferrum.
|
|
19
19
|
* [Vessel](https://github.com/rubycdp/vessel) high-level web crawling framework based on Ferrum and Mechanize.
|
|
20
20
|
|
|
21
|
+
<br clear="right">
|
|
22
|
+
|
|
21
23
|
## Install
|
|
22
24
|
|
|
23
25
|
There's no official Chrome or Chromium package for Linux don't install it this
|
|
@@ -33,6 +35,17 @@ Add this to your `Gemfile` and run `bundle install`.
|
|
|
33
35
|
gem "ferrum"
|
|
34
36
|
```
|
|
35
37
|
|
|
38
|
+
Or skip installing Chrome and run it in a container with our images, see
|
|
39
|
+
[Docker](https://docs.rubycdp.com/docs/ferrum/docker#our-images-recommended):
|
|
40
|
+
|
|
41
|
+
```shell
|
|
42
|
+
docker run --rm -p 3000:3000 ghcr.io/rubycdp/chromium
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
```ruby
|
|
46
|
+
browser = Ferrum::Browser.new(url: "http://localhost:3000")
|
|
47
|
+
```
|
|
48
|
+
|
|
36
49
|
## Quick Start
|
|
37
50
|
|
|
38
51
|
Navigate to a website and save a screenshot:
|
|
@@ -72,6 +85,55 @@ JS
|
|
|
72
85
|
browser.quit
|
|
73
86
|
```
|
|
74
87
|
|
|
88
|
+
Pass arguments by name, so the script can use parameters instead of digging
|
|
89
|
+
through `arguments[0]`:
|
|
90
|
+
|
|
91
|
+
```ruby
|
|
92
|
+
page.evaluate("a + b", a: 1, b: 2) # => 3
|
|
93
|
+
page.evaluate("el.getAttribute(name)", el: page.at_css("a"), name: "href")
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
A script that is a function or arrow declaration is used as-is and takes its
|
|
97
|
+
arguments positionally, in the order it declares them:
|
|
98
|
+
|
|
99
|
+
```ruby
|
|
100
|
+
page.evaluate(<<~JS, 1, 2)
|
|
101
|
+
function(a, b) {
|
|
102
|
+
const sum = a + b;
|
|
103
|
+
return sum * 3;
|
|
104
|
+
}
|
|
105
|
+
JS
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
`await` works directly, and a function declaration's returned promise is
|
|
109
|
+
awaited:
|
|
110
|
+
|
|
111
|
+
```ruby
|
|
112
|
+
page.evaluate("await fetch(url).then(r => r.text())", url: "/api")
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Pass `timeout:` (seconds, defaulting to the page timeout) to bound how long the
|
|
116
|
+
script may take before `Ferrum::ScriptTimeoutError` is raised. `timeout:` is the
|
|
117
|
+
only reserved keyword.
|
|
118
|
+
|
|
119
|
+
Use `#execute` when you only want the side effects, `#evaluate_handle` when you
|
|
120
|
+
want to keep a value in the browser and pass it back in later, and
|
|
121
|
+
`Ferrum::Node#evaluate` to run a script with `this` bound to an element:
|
|
122
|
+
|
|
123
|
+
```ruby
|
|
124
|
+
page.execute("window.scrollBy(0, 100)") # => true
|
|
125
|
+
|
|
126
|
+
list = page.evaluate_handle("document.querySelectorAll('li')")
|
|
127
|
+
page.evaluate("Array.from(nodes).map(n => n.textContent)", nodes: list)
|
|
128
|
+
|
|
129
|
+
page.at_css("input").evaluate("this.value")
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Scripts written for the old API keep working, and Ferrum warns, with a suggested
|
|
133
|
+
rewrite, about any whose result will change. The next release, 0.21, removes the
|
|
134
|
+
old behavior, so run your suite with `FERRUM_JS_API=new` to check it's ready. See
|
|
135
|
+
[upgrading to 0.21](docs/13-javascript.md#upgrading-to-021).
|
|
136
|
+
|
|
75
137
|
Do any mouse movements you like:
|
|
76
138
|
|
|
77
139
|
```ruby
|
|
@@ -68,7 +68,6 @@ module Ferrum
|
|
|
68
68
|
[paths, exts]
|
|
69
69
|
end
|
|
70
70
|
|
|
71
|
-
# rubocop:disable Style/CollectionCompact
|
|
72
71
|
def lazy_find(cmds)
|
|
73
72
|
cmds.lazy.map do |cmd, path, ext|
|
|
74
73
|
absolute_path = File.absolute_path(cmd)
|
|
@@ -79,9 +78,8 @@ module Ferrum
|
|
|
79
78
|
next if File.directory?(cmd)
|
|
80
79
|
|
|
81
80
|
cmd
|
|
82
|
-
end.
|
|
81
|
+
end.compact
|
|
83
82
|
end
|
|
84
|
-
# rubocop:enable Style/CollectionCompact
|
|
85
83
|
end
|
|
86
84
|
end
|
|
87
85
|
end
|
|
@@ -17,8 +17,6 @@ module Ferrum
|
|
|
17
17
|
# https://github.com/cyrus-and/chrome-remote-interface#implementations
|
|
18
18
|
def self.build(options, user_data_dir)
|
|
19
19
|
defaults = case options.browser_name
|
|
20
|
-
when :firefox
|
|
21
|
-
Options::Firefox.options
|
|
22
20
|
when :chrome, :opera, :edge, nil
|
|
23
21
|
Options::Chrome.options
|
|
24
22
|
else
|
|
@@ -8,10 +8,9 @@ module Ferrum
|
|
|
8
8
|
class Options
|
|
9
9
|
#
|
|
10
10
|
# Abstract base for browser-specific default option builders. Subclasses
|
|
11
|
-
# (e.g. {Options::Chrome}
|
|
12
|
-
# `
|
|
13
|
-
#
|
|
14
|
-
# by {Command}.
|
|
11
|
+
# (e.g. {Options::Chrome}) declare their own `DEFAULT_OPTIONS` and
|
|
12
|
+
# `PLATFORM_PATH` constants and implement {#merge_required}/{#merge_default}
|
|
13
|
+
# to produce the final CLI flags used by {Command}.
|
|
15
14
|
#
|
|
16
15
|
class Base
|
|
17
16
|
include Singleton
|
|
@@ -71,7 +70,7 @@ module Ferrum
|
|
|
71
70
|
#
|
|
72
71
|
# Merges the CLI flags that are always required to drive the browser
|
|
73
72
|
# over CDP (e.g. remote debugging port, user data dir) into `flags`.
|
|
74
|
-
# Abstract; overridden by {Options::Chrome}
|
|
73
|
+
# Abstract; overridden by {Options::Chrome}.
|
|
75
74
|
#
|
|
76
75
|
# @param [Hash{String => Object}] flags
|
|
77
76
|
# The flags accumulated so far.
|
|
@@ -92,7 +91,7 @@ module Ferrum
|
|
|
92
91
|
#
|
|
93
92
|
# Merges this browser's default CLI flags (see {#to_h}) into `flags`,
|
|
94
93
|
# unless `options.ignore_default_browser_options` is set. Abstract;
|
|
95
|
-
# overridden by {Options::Chrome}
|
|
94
|
+
# overridden by {Options::Chrome}.
|
|
96
95
|
#
|
|
97
96
|
# @param [Hash{String => Object}] flags
|
|
98
97
|
# The flags accumulated so far.
|
|
@@ -48,7 +48,6 @@ module Ferrum
|
|
|
48
48
|
"disable-smooth-scrolling" => nil,
|
|
49
49
|
"disable-sync" => nil,
|
|
50
50
|
"disable-translate" => nil,
|
|
51
|
-
"disable-web-security" => nil,
|
|
52
51
|
"enable-automation" => nil,
|
|
53
52
|
"enable-features" => %w[
|
|
54
53
|
NetworkService
|
|
@@ -61,7 +60,6 @@ module Ferrum
|
|
|
61
60
|
"metrics-recording-only" => nil,
|
|
62
61
|
"mute-audio" => nil,
|
|
63
62
|
"no-crash-upload" => nil,
|
|
64
|
-
"no-crashpad" => nil,
|
|
65
63
|
"no-default-browser-check" => nil,
|
|
66
64
|
"no-first-run" => nil,
|
|
67
65
|
"no-startup-window" => nil,
|
|
@@ -104,7 +102,6 @@ module Ferrum
|
|
|
104
102
|
#
|
|
105
103
|
def merge_required(flags, options, user_data_dir)
|
|
106
104
|
flags = flags.merge("remote-debugging-port" => options.port,
|
|
107
|
-
"remote-debugging-address" => options.host,
|
|
108
105
|
"window-size" => options.window_size&.join(","),
|
|
109
106
|
"user-data-dir" => user_data_dir)
|
|
110
107
|
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "ferrum/log"
|
|
4
|
+
|
|
3
5
|
module Ferrum
|
|
4
6
|
class Browser
|
|
5
7
|
#
|
|
@@ -16,13 +18,14 @@ module Ferrum
|
|
|
16
18
|
DEFAULT_TIMEOUT = ENV.fetch("FERRUM_DEFAULT_TIMEOUT", 5).to_i
|
|
17
19
|
DEFAULT_PROTOCOL_TIMEOUT = ENV.fetch("FERRUM_PROTOCOL_TIMEOUT", DEFAULT_TIMEOUT).to_i
|
|
18
20
|
PROCESS_TIMEOUT = ENV.fetch("FERRUM_PROCESS_TIMEOUT", 10).to_i
|
|
19
|
-
DEBUG_MODE = !ENV.fetch("FERRUM_DEBUG", nil).nil?
|
|
20
21
|
|
|
21
|
-
|
|
22
|
+
warn "Ferrum: FERRUM_DEBUG is deprecated, use FERRUM_LOG=* instead" if ENV.key?("FERRUM_DEBUG")
|
|
23
|
+
|
|
24
|
+
attr_reader :window_size, :logger, :log, :ws_max_receive_size,
|
|
22
25
|
:js_errors, :base_url, :slowmo, :pending_connection_errors,
|
|
23
26
|
:url, :ws_url, :env, :process_timeout, :browser_name, :browser_path,
|
|
24
27
|
:save_path, :proxy, :port, :host, :headless, :incognito, :dockerize, :browser_options,
|
|
25
|
-
:ignore_default_browser_options, :xvfb, :flatten
|
|
28
|
+
:ignore_default_browser_options, :xvfb, :flatten, :extensions
|
|
26
29
|
attr_accessor :timeout, :protocol_timeout, :default_user_agent
|
|
27
30
|
|
|
28
31
|
def initialize(options = nil)
|
|
@@ -51,10 +54,11 @@ module Ferrum
|
|
|
51
54
|
@ignore_default_browser_options = @options[:ignore_default_browser_options]
|
|
52
55
|
|
|
53
56
|
@proxy = validate_proxy(@options[:proxy])
|
|
54
|
-
@logger = parse_logger(@options[:logger])
|
|
57
|
+
@logger, @log = parse_logger(@options[:logger])
|
|
55
58
|
@base_url = parse_base_url(@options[:base_url]) if @options[:base_url]
|
|
56
59
|
@url = @options[:url].to_s if @options[:url]
|
|
57
60
|
@ws_url = @options[:ws_url].to_s if @options[:ws_url]
|
|
61
|
+
@extensions = Array(@options[:extensions]).map { |e| read_extension(e) }.freeze
|
|
58
62
|
|
|
59
63
|
@options = @options.merge(window_size: @window_size).freeze
|
|
60
64
|
@browser_options = @options.fetch(:browser_options, {}).freeze
|
|
@@ -73,15 +77,24 @@ module Ferrum
|
|
|
73
77
|
end
|
|
74
78
|
|
|
75
79
|
#
|
|
76
|
-
#
|
|
80
|
+
# Adds an extension to run in every new document, before the document's own scripts. Applies to pages and
|
|
81
|
+
# iframes attached after the call: pages already open keep the extensions they were attached with.
|
|
82
|
+
#
|
|
83
|
+
# @param [String, Hash] extension
|
|
84
|
+
# A path to a JS file, or `{ source: "..." }`.
|
|
77
85
|
#
|
|
78
86
|
# @return [Array<String>]
|
|
79
|
-
# JS source code
|
|
87
|
+
# JS source code of every extension.
|
|
80
88
|
#
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
89
|
+
# @raise [Errno::ENOENT]
|
|
90
|
+
# If the file doesn't exist.
|
|
91
|
+
#
|
|
92
|
+
# @example
|
|
93
|
+
# browser.add_extension("/path/to/stub.js")
|
|
94
|
+
# browser.add_extension(source: "window.secret = 'top'")
|
|
95
|
+
#
|
|
96
|
+
def add_extension(extension)
|
|
97
|
+
@extensions = [*@extensions, read_extension(extension)].freeze
|
|
85
98
|
end
|
|
86
99
|
|
|
87
100
|
#
|
|
@@ -119,10 +132,18 @@ module Ferrum
|
|
|
119
132
|
|
|
120
133
|
private
|
|
121
134
|
|
|
135
|
+
def read_extension(extension)
|
|
136
|
+
(extension.is_a?(Hash) && extension[:source]) || File.read(extension)
|
|
137
|
+
end
|
|
138
|
+
|
|
122
139
|
def parse_logger(logger)
|
|
123
|
-
|
|
140
|
+
value = ENV.fetch("FERRUM_LOG") { "*" if ENV.key?("FERRUM_DEBUG") }
|
|
141
|
+
settings = value ? Log.parse(value) : {}
|
|
142
|
+
logger ||= $stdout.tap { |s| s.sync = true } if value && settings
|
|
143
|
+
return [logger, nil] unless logger && settings
|
|
124
144
|
|
|
125
|
-
|
|
145
|
+
[logger, Log.new(logger, **settings, format: Log.parse_format(ENV.fetch("FERRUM_LOG_FORMAT", nil)),
|
|
146
|
+
color: Log.parse_color(ENV.fetch("FERRUM_LOG_COLOR", nil)))]
|
|
126
147
|
end
|
|
127
148
|
|
|
128
149
|
def parse_base_url(value)
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "ferrum/utils/elapsed_time"
|
|
5
|
+
require "ferrum/utils/platform"
|
|
6
|
+
|
|
7
|
+
module Ferrum
|
|
8
|
+
class Browser
|
|
9
|
+
class Process
|
|
10
|
+
#
|
|
11
|
+
# OS-level process termination and user-data-directory removal, kept
|
|
12
|
+
# separate from the rest of {Process} (spawning, CDP handshake, ...).
|
|
13
|
+
# Both the GC finalizer and the explicit {Process#stop} path need to
|
|
14
|
+
# run the exact same kill/cleanup logic on plain pid/path values --
|
|
15
|
+
# never on `self`. A finalizer proc that closes over the object it's
|
|
16
|
+
# attached to creates a reference cycle GC can never collect, so the
|
|
17
|
+
# finalizer would never run; keeping this logic as free functions
|
|
18
|
+
# taking plain arguments sidesteps that entirely.
|
|
19
|
+
#
|
|
20
|
+
module Killer
|
|
21
|
+
module_function
|
|
22
|
+
|
|
23
|
+
KILL_TIMEOUT = 2
|
|
24
|
+
WAIT_KILLED = 0.05
|
|
25
|
+
REAP_TIMEOUT = 0.5
|
|
26
|
+
REMOVE_DIR_RETRIES = 5
|
|
27
|
+
REMOVE_DIR_RETRY_DELAY = 0.1
|
|
28
|
+
|
|
29
|
+
#
|
|
30
|
+
# Kills the process at the given pid (and its process group, if
|
|
31
|
+
# any), escalating from TERM to KILL if it doesn't exit within
|
|
32
|
+
# {KILL_TIMEOUT} seconds.
|
|
33
|
+
#
|
|
34
|
+
# @param [Integer] pid
|
|
35
|
+
#
|
|
36
|
+
# @return [void]
|
|
37
|
+
#
|
|
38
|
+
def kill(pid)
|
|
39
|
+
if Utils::Platform.windows?
|
|
40
|
+
# Process.kill is unreliable on Windows
|
|
41
|
+
::Process.kill("KILL", pid) unless system("taskkill /f /t /pid #{pid} >NUL 2>NUL")
|
|
42
|
+
return
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
send_signal(pid, "TERM")
|
|
46
|
+
start = Utils::ElapsedTime.monotonic_time
|
|
47
|
+
leader_exited = false
|
|
48
|
+
loop do
|
|
49
|
+
# The leader (the pid we actually spawned and can #wait on) may
|
|
50
|
+
# exit well before the rest of its process group does -- e.g. a
|
|
51
|
+
# renderer/zygote that ignores TERM. Reaping the leader must not
|
|
52
|
+
# by itself end the loop, or that child is orphaned forever
|
|
53
|
+
# since KILL is only ever sent from the timeout branch below.
|
|
54
|
+
leader_exited ||= !::Process.wait(pid, ::Process::WNOHANG).nil?
|
|
55
|
+
break if leader_exited && !process_group_alive?(pid)
|
|
56
|
+
|
|
57
|
+
sleep(WAIT_KILLED)
|
|
58
|
+
next unless Utils::ElapsedTime.timeout?(start, KILL_TIMEOUT)
|
|
59
|
+
|
|
60
|
+
send_signal(pid, "KILL")
|
|
61
|
+
wait_reaped(pid) unless leader_exited
|
|
62
|
+
break
|
|
63
|
+
end
|
|
64
|
+
rescue Errno::ESRCH, Errno::ECHILD
|
|
65
|
+
# nop
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
#
|
|
69
|
+
# Waits for `pid` to become reapable, giving up after `timeout`.
|
|
70
|
+
#
|
|
71
|
+
# Deliberately a bounded poll rather than a blocking `Process.wait`.
|
|
72
|
+
# {#finalizer} installs {#kill} as an `ObjectSpace` finalizer, so
|
|
73
|
+
# it can run while the interpreter is shutting down, and a blocking
|
|
74
|
+
# wait there never returns.
|
|
75
|
+
#
|
|
76
|
+
# @param [Integer] pid
|
|
77
|
+
# @param [Float] timeout
|
|
78
|
+
#
|
|
79
|
+
# @return [Integer, nil]
|
|
80
|
+
# The reaped pid, or `nil` if it didn't exit in time or wasn't ours.
|
|
81
|
+
#
|
|
82
|
+
def wait_reaped(pid, timeout: REAP_TIMEOUT)
|
|
83
|
+
start = Utils::ElapsedTime.monotonic_time
|
|
84
|
+
loop do
|
|
85
|
+
reaped = ::Process.wait(pid, ::Process::WNOHANG)
|
|
86
|
+
return reaped if reaped
|
|
87
|
+
return nil if Utils::ElapsedTime.timeout?(start, timeout)
|
|
88
|
+
|
|
89
|
+
sleep(WAIT_KILLED)
|
|
90
|
+
end
|
|
91
|
+
rescue Errno::ECHILD, Errno::ESRCH
|
|
92
|
+
nil
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
#
|
|
96
|
+
# Builds the proc registered as a browser's `ObjectSpace` finalizer,
|
|
97
|
+
# cleaning up after a browser nobody quit.
|
|
98
|
+
#
|
|
99
|
+
# Kills every pid first and removes the directory afterwards. That
|
|
100
|
+
# order is the whole point of this being one finalizer instead of
|
|
101
|
+
# two: Ruby runs an object's finalizers in registration order, and
|
|
102
|
+
# removing the user data directory while the browser is still alive
|
|
103
|
+
# means Chrome recreates it on the way out, leaking it for good.
|
|
104
|
+
#
|
|
105
|
+
# @param [Array<Integer>, Integer, nil] pids
|
|
106
|
+
# Process ids to kill, e.g. the browser and Xvfb.
|
|
107
|
+
# @param [String, nil] path
|
|
108
|
+
# User data directory to remove once they are dead.
|
|
109
|
+
#
|
|
110
|
+
# @return [Proc]
|
|
111
|
+
#
|
|
112
|
+
def finalizer(pids, path)
|
|
113
|
+
pids = Array(pids).compact
|
|
114
|
+
proc do
|
|
115
|
+
pids.each { |pid| kill(pid) }
|
|
116
|
+
remove_directory(path) if path
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
#
|
|
121
|
+
# Signals the whole process group Chrome was spawned into (it's
|
|
122
|
+
# spawned with `pgroup: true`), so its child processes (renderer,
|
|
123
|
+
# GPU, zygote, ...) are cleaned up too instead of being left
|
|
124
|
+
# orphaned. Falls back to signaling just the pid directly if
|
|
125
|
+
# there's no such process group to signal (e.g. Xvfb, which isn't
|
|
126
|
+
# spawned with `pgroup: true`) or the group can't be signaled.
|
|
127
|
+
#
|
|
128
|
+
# @param [Integer] pid
|
|
129
|
+
# @param [String] name
|
|
130
|
+
#
|
|
131
|
+
# @return [void]
|
|
132
|
+
#
|
|
133
|
+
def send_signal(pid, name)
|
|
134
|
+
::Process.kill(name, -pid)
|
|
135
|
+
rescue Errno::EPERM, Errno::ESRCH
|
|
136
|
+
::Process.kill(name, pid)
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
#
|
|
140
|
+
# Checks whether any process in pid's process group is still alive.
|
|
141
|
+
#
|
|
142
|
+
# @param [Integer] pid
|
|
143
|
+
#
|
|
144
|
+
# @return [Boolean]
|
|
145
|
+
#
|
|
146
|
+
def process_group_alive?(pid)
|
|
147
|
+
::Process.kill(0, -pid)
|
|
148
|
+
true
|
|
149
|
+
rescue Errno::ESRCH
|
|
150
|
+
false
|
|
151
|
+
rescue Errno::EPERM
|
|
152
|
+
true
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
#
|
|
156
|
+
# Removes the given directory, retrying with exponential backoff on
|
|
157
|
+
# transient errors. Chrome can briefly hold file locks right after
|
|
158
|
+
# being killed, so the directory may not be removable on the first
|
|
159
|
+
# try; retrying avoids leaking temp directories in that case.
|
|
160
|
+
#
|
|
161
|
+
# @param [String] path
|
|
162
|
+
# Directory to remove.
|
|
163
|
+
# @param [Integer] retries
|
|
164
|
+
# Maximum number of removal attempts.
|
|
165
|
+
# @param [Float] delay
|
|
166
|
+
# Base delay, in seconds, before the first retry; doubles on each
|
|
167
|
+
# subsequent attempt.
|
|
168
|
+
#
|
|
169
|
+
# @return [void]
|
|
170
|
+
#
|
|
171
|
+
def remove_directory(path, retries: REMOVE_DIR_RETRIES, delay: REMOVE_DIR_RETRY_DELAY)
|
|
172
|
+
retries.times do |attempt|
|
|
173
|
+
FileUtils.remove_entry(path)
|
|
174
|
+
break
|
|
175
|
+
rescue Errno::ENOENT
|
|
176
|
+
break
|
|
177
|
+
rescue Errno::ENOTEMPTY, Errno::EBUSY, Errno::EACCES, Errno::EPERM => e
|
|
178
|
+
raise e if attempt == retries - 1
|
|
179
|
+
|
|
180
|
+
sleep(delay * (2**attempt))
|
|
181
|
+
end
|
|
182
|
+
rescue StandardError => e
|
|
183
|
+
warn("[Ferrum] Failed to remove user data dir #{path}: #{e.class}: #{e.message}")
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
end
|
|
188
|
+
end
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ferrum
|
|
4
|
+
class Browser
|
|
5
|
+
class Process
|
|
6
|
+
#
|
|
7
|
+
# What the browser process wrote to stdout and stderr: keeps the last
|
|
8
|
+
# {LIMIT} lines and writes each one to the `:browser` log channel. Owns
|
|
9
|
+
# the pipe the process writes to, which is read on its own thread for as
|
|
10
|
+
# long as the process lives, so it never fills up and blocks the browser.
|
|
11
|
+
#
|
|
12
|
+
class Output
|
|
13
|
+
LIMIT = 100
|
|
14
|
+
DRAIN_TIMEOUT = 0.5
|
|
15
|
+
|
|
16
|
+
def initialize(log = nil)
|
|
17
|
+
@log = log
|
|
18
|
+
@lines = []
|
|
19
|
+
@partial = String.new(encoding: Encoding::BINARY)
|
|
20
|
+
@mutex = Mutex.new
|
|
21
|
+
@io = @reader = nil
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
#
|
|
25
|
+
# Opens the pipe and yields its write end, to be given to the process
|
|
26
|
+
# as stdout and stderr. The write end is closed here afterwards, so
|
|
27
|
+
# reading hits the end of the pipe once the process exits.
|
|
28
|
+
#
|
|
29
|
+
# @yieldparam [IO] io
|
|
30
|
+
#
|
|
31
|
+
# @return [void]
|
|
32
|
+
#
|
|
33
|
+
def pipe
|
|
34
|
+
@io, writer = IO.pipe
|
|
35
|
+
yield writer
|
|
36
|
+
ensure
|
|
37
|
+
close_io(writer)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
#
|
|
41
|
+
# Reads the pipe until the output matches, the process exits or the
|
|
42
|
+
# time is up.
|
|
43
|
+
#
|
|
44
|
+
# @param [Regexp] regexp
|
|
45
|
+
#
|
|
46
|
+
# @param [Numeric] timeout
|
|
47
|
+
#
|
|
48
|
+
# @return [MatchData, nil]
|
|
49
|
+
#
|
|
50
|
+
def wait_for(regexp, timeout)
|
|
51
|
+
deadline = Utils::ElapsedTime.monotonic_time + timeout
|
|
52
|
+
while (now = Utils::ElapsedTime.monotonic_time) < deadline
|
|
53
|
+
begin
|
|
54
|
+
self << @io.read_nonblock(512)
|
|
55
|
+
rescue IO::WaitReadable
|
|
56
|
+
@io.wait_readable(deadline - now)
|
|
57
|
+
rescue EOFError
|
|
58
|
+
return
|
|
59
|
+
else
|
|
60
|
+
match = to_s.match(regexp)
|
|
61
|
+
return match if match
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
#
|
|
67
|
+
# Keeps reading the pipe on a new thread until it's closed.
|
|
68
|
+
#
|
|
69
|
+
# @return [Thread]
|
|
70
|
+
#
|
|
71
|
+
def follow
|
|
72
|
+
@reader = Utils::Thread.spawn do
|
|
73
|
+
loop { self << @io.readpartial(4096) }
|
|
74
|
+
rescue IOError
|
|
75
|
+
self << "\n" unless @partial.empty?
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
#
|
|
80
|
+
# Gives the reading thread {DRAIN_TIMEOUT} seconds to take what the
|
|
81
|
+
# process wrote last, then closes the pipe. The lines are kept.
|
|
82
|
+
#
|
|
83
|
+
# @return [void]
|
|
84
|
+
#
|
|
85
|
+
def close
|
|
86
|
+
@reader&.join(DRAIN_TIMEOUT)
|
|
87
|
+
close_io(@io)
|
|
88
|
+
@io = @reader = nil
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
#
|
|
92
|
+
# Adds a chunk of output, which can end in the middle of a line.
|
|
93
|
+
#
|
|
94
|
+
# @param [String] chunk
|
|
95
|
+
#
|
|
96
|
+
# @return [self]
|
|
97
|
+
#
|
|
98
|
+
def <<(chunk)
|
|
99
|
+
lines = @mutex.synchronize do
|
|
100
|
+
*complete, @partial = (@partial + chunk.b).split("\n", -1)
|
|
101
|
+
complete.map! { |line| utf8(line) }
|
|
102
|
+
@lines.concat(complete)
|
|
103
|
+
@lines.shift(@lines.size - LIMIT) if @lines.size > LIMIT
|
|
104
|
+
complete
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
lines.each { |line| @log&.browser(line) }
|
|
108
|
+
self
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
#
|
|
112
|
+
# The last lines, oldest first.
|
|
113
|
+
#
|
|
114
|
+
# @return [Array<String>]
|
|
115
|
+
#
|
|
116
|
+
def lines
|
|
117
|
+
@mutex.synchronize { @partial.empty? ? @lines.dup : [*@lines, utf8(@partial)] }
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
#
|
|
121
|
+
# The last lines joined by newlines.
|
|
122
|
+
#
|
|
123
|
+
# @return [String]
|
|
124
|
+
#
|
|
125
|
+
def to_s
|
|
126
|
+
lines.join("\n")
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
private
|
|
130
|
+
|
|
131
|
+
def utf8(line)
|
|
132
|
+
line.dup.force_encoding(Encoding::UTF_8).scrub.delete_suffix("\r")
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def close_io(io)
|
|
136
|
+
io.close if io && !io.closed?
|
|
137
|
+
rescue IOError
|
|
138
|
+
raise unless RUBY_ENGINE == "jruby"
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|