bidi2pdf 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -2
- data/README.md +145 -0
- data/docker/Dockerfile +7 -0
- data/docker/Dockerfile.chromedriver +7 -0
- data/docker/Dockerfile.slim +7 -0
- data/lib/bidi2pdf/bidi/buffered_web_socket_client.rb +9 -0
- data/lib/bidi2pdf/bidi/commands/browsing_context_get_tree.rb +16 -0
- data/lib/bidi2pdf/bidi/commands/cdp_send_command.rb +25 -0
- data/lib/bidi2pdf/bidi/commands.rb +2 -0
- data/lib/bidi2pdf/bidi/session.rb +57 -24
- data/lib/bidi2pdf/chrome_sweeper/inspector.rb +117 -0
- data/lib/bidi2pdf/chrome_sweeper/settings.rb +79 -0
- data/lib/bidi2pdf/chrome_sweeper.rb +398 -0
- data/lib/bidi2pdf/chromedriver_api.rb +72 -0
- data/lib/bidi2pdf/cli/session_commands.rb +82 -0
- data/lib/bidi2pdf/cli.rb +43 -0
- data/lib/bidi2pdf/dsl.rb +2 -1
- data/lib/bidi2pdf/launcher.rb +2 -1
- data/lib/bidi2pdf/session_registry/heartbeat.rb +83 -0
- data/lib/bidi2pdf/session_registry.rb +152 -0
- data/lib/bidi2pdf/session_sweeper.rb +66 -0
- data/lib/bidi2pdf/session_warmer.rb +112 -1
- data/lib/bidi2pdf/version.rb +1 -1
- data/lib/bidi2pdf.rb +4 -0
- data/sig/bidi2pdf/bidi/buffered_web_socket_client.rbs +7 -0
- data/sig/bidi2pdf/bidi/commands/browsing_context_get_tree.rbs +12 -0
- data/sig/bidi2pdf/bidi/commands/cdp_send_command.rbs +23 -0
- data/sig/bidi2pdf/bidi/session.rbs +25 -3
- data/sig/bidi2pdf/chrome_sweeper/inspector.rbs +61 -0
- data/sig/bidi2pdf/chrome_sweeper/settings.rbs +34 -0
- data/sig/bidi2pdf/chrome_sweeper.rbs +220 -0
- data/sig/bidi2pdf/chromedriver_api.rbs +44 -0
- data/sig/bidi2pdf/cli/session_commands.rbs +35 -0
- data/sig/bidi2pdf/cli.rbs +6 -0
- data/sig/bidi2pdf/session_registry/heartbeat.rbs +47 -0
- data/sig/bidi2pdf/session_registry.rbs +75 -0
- data/sig/bidi2pdf/session_sweeper.rbs +37 -0
- data/sig/bidi2pdf/session_warmer.rbs +66 -0
- data/sig/bidi2pdf/version.rbs +1 -1
- metadata +29 -3
data/lib/bidi2pdf/cli.rb
CHANGED
|
@@ -5,11 +5,13 @@ require "yaml"
|
|
|
5
5
|
require "tempfile"
|
|
6
6
|
|
|
7
7
|
require_relative "cli/json_output"
|
|
8
|
+
require_relative "cli/session_commands"
|
|
8
9
|
|
|
9
10
|
module Bidi2pdf
|
|
10
11
|
# rubocop:disable Metrics/AbcSize
|
|
11
12
|
class CLI < Thor
|
|
12
13
|
include JsonOutput
|
|
14
|
+
include SessionCommands
|
|
13
15
|
|
|
14
16
|
class_option :config, type: :string, desc: "Load configuration from YAML file"
|
|
15
17
|
|
|
@@ -183,6 +185,47 @@ module Bidi2pdf
|
|
|
183
185
|
end
|
|
184
186
|
map "run" => :run_recipe
|
|
185
187
|
|
|
188
|
+
desc "sessions", "List the Chrome sessions a remote chromedriver holds: id, age, tabs, responsive (no URLs)"
|
|
189
|
+
option :remote_browser_url, type: :string, required: true, desc: "The chromedriver's new-session URL (http://host:3000/session)"
|
|
190
|
+
option :scope, type: :string, default: "all", enum: %w[all recorded], desc: "all sessions, or only those bidi2pdf recorded"
|
|
191
|
+
option :json, type: :boolean, default: false, desc: "Emit a JSON array instead of a table"
|
|
192
|
+
|
|
193
|
+
def sessions
|
|
194
|
+
return print_sessions(chrome_sweeper.sessions) unless options[:json]
|
|
195
|
+
|
|
196
|
+
reserve_stdout_for_machine_output { puts JSON.generate(chrome_sweeper.sessions.map { |info| session_hash(info) }) }
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
desc "sweep", "Close leaked Chrome sessions on a remote chromedriver (too old, unresponsive, or over a limit)"
|
|
200
|
+
long_desc <<~USAGE, wrap: false
|
|
201
|
+
Closes sessions older than --older-than seconds, sessions that failed --unresponsive-checks
|
|
202
|
+
checks in a row (taken --check-interval seconds apart, so a sweep takes that long), and while
|
|
203
|
+
more than --max-sessions exist, the oldest ones. A session younger than --min-age seconds is
|
|
204
|
+
never closed, and neither is one a running bidi2pdf process holds (its lease is renewed). --scope all looks at
|
|
205
|
+
every session on that chromedriver - use it only for a chromedriver your application owns;
|
|
206
|
+
--scope recorded (default) only at sessions bidi2pdf recorded on this machine.
|
|
207
|
+
Exits 1 when a close failed or the limit is still exceeded.
|
|
208
|
+
USAGE
|
|
209
|
+
option :remote_browser_url, type: :string, required: true, desc: "The chromedriver's new-session URL (http://host:3000/session)"
|
|
210
|
+
option :scope, type: :string, default: "recorded", enum: %w[recorded all], desc: "Which sessions may be closed"
|
|
211
|
+
option :older_than, type: :numeric, desc: "Close sessions older than this many seconds (default 600)"
|
|
212
|
+
option :max_sessions, type: :numeric, desc: "Close the oldest sessions while more than this many exist"
|
|
213
|
+
option :min_age, type: :numeric, desc: "Never close a session younger than this many seconds (default 60)"
|
|
214
|
+
option :unresponsive_checks, type: :numeric, desc: "Close a session after this many failed checks in a row (default 2)"
|
|
215
|
+
option :check_interval, type: :numeric, default: 10, desc: "Seconds between those checks; 0 sweeps at once, without the unresponsive rule"
|
|
216
|
+
option :pressure, type: :boolean, default: false, desc: "Last resort: close every session no live process holds that is past --min-age"
|
|
217
|
+
option :dry_run, type: :boolean, default: false, desc: "Report what would be closed, close nothing"
|
|
218
|
+
option :json, type: :boolean, default: false, desc: "Emit the sweep result as JSON"
|
|
219
|
+
|
|
220
|
+
def sweep
|
|
221
|
+
result = if options[:json]
|
|
222
|
+
reserve_stdout_for_machine_output { run_sweep.tap { |swept| puts JSON.generate(sweep_hash(swept)) } }
|
|
223
|
+
else
|
|
224
|
+
run_sweep.tap { |swept| print_sweep(swept) }
|
|
225
|
+
end
|
|
226
|
+
exit(1) if result.errors.any? || result.limit_exceeded
|
|
227
|
+
end
|
|
228
|
+
|
|
186
229
|
desc "template", "Generate a config file template"
|
|
187
230
|
option :output, default: "bidi2pdf.yml", desc: "Output configuration filename"
|
|
188
231
|
|
data/lib/bidi2pdf/dsl.rb
CHANGED
|
@@ -47,7 +47,8 @@ module Bidi2pdf
|
|
|
47
47
|
Bidi2pdf::Bidi::Session.new(
|
|
48
48
|
session_url: remote_browser_url,
|
|
49
49
|
headless: true, # remote is always headless
|
|
50
|
-
chrome_args: chrome_args
|
|
50
|
+
chrome_args: chrome_args,
|
|
51
|
+
registry: Bidi2pdf::SessionRegistry.new(remote_browser_url)
|
|
51
52
|
)
|
|
52
53
|
else
|
|
53
54
|
manager = Bidi2pdf::ChromedriverManager.new(port: port, headless: headless, chrome_args: chrome_args)
|
data/lib/bidi2pdf/launcher.rb
CHANGED
|
@@ -109,7 +109,8 @@ module Bidi2pdf
|
|
|
109
109
|
|
|
110
110
|
def session
|
|
111
111
|
if @remote_browser_url
|
|
112
|
-
@custom_session = Bidi::Session.new(session_url: @remote_browser_url, headless: @headless, chrome_args: @chrome_args
|
|
112
|
+
@custom_session = Bidi::Session.new(session_url: @remote_browser_url, headless: @headless, chrome_args: @chrome_args,
|
|
113
|
+
registry: Bidi2pdf::SessionRegistry.new(@remote_browser_url))
|
|
113
114
|
else
|
|
114
115
|
@manager = ChromedriverManager.new(port: @port, headless: @headless, chrome_args: @chrome_args)
|
|
115
116
|
@manager.start
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bidi2pdf
|
|
4
|
+
class SessionRegistry
|
|
5
|
+
# Keeps the leases of every session this process holds fresh: one thread per process renews
|
|
6
|
+
# them in their registry files every +interval+ seconds, so a sweeper in another process (a
|
|
7
|
+
# second Puma worker, a job worker sharing the registry directory) can tell a live session
|
|
8
|
+
# from one whose process died. The thread runs only while something is held.
|
|
9
|
+
#
|
|
10
|
+
# Fork-aware: a child process (Puma forks its workers) starts with nothing held and without the
|
|
11
|
+
# parent's thread - the parent's sessions are the parent's to renew.
|
|
12
|
+
module Heartbeat
|
|
13
|
+
DEFAULT_INTERVAL = DEFAULT_LEASE_TTL / 3.0
|
|
14
|
+
|
|
15
|
+
@mutex = Mutex.new
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
# @return [Numeric] seconds between renewals - keep it well below the sweeper's lease_ttl.
|
|
19
|
+
attr_writer :interval
|
|
20
|
+
|
|
21
|
+
def interval = @interval || DEFAULT_INTERVAL
|
|
22
|
+
|
|
23
|
+
def hold(registry, session_id)
|
|
24
|
+
synchronize do
|
|
25
|
+
(held[registry.path] ||= [registry, Set.new])[1] << session_id.to_s
|
|
26
|
+
@thread = Thread.new { beat_loop } unless @thread&.alive?
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def release(registry, session_id)
|
|
31
|
+
synchronize do
|
|
32
|
+
ids = held.dig(registry.path, 1)
|
|
33
|
+
next unless ids
|
|
34
|
+
|
|
35
|
+
ids.delete(session_id.to_s)
|
|
36
|
+
held.delete(registry.path) if ids.empty?
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# @return [Array<String>] the session ids this process holds in +registry+'s file.
|
|
41
|
+
def held_ids(registry)
|
|
42
|
+
synchronize { held.dig(registry.path, 1).to_a }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Renews every held lease now.
|
|
46
|
+
def beat!(now: Time.now.to_i)
|
|
47
|
+
synchronize { held.values.map { |registry, ids| [registry, ids.to_a] } }
|
|
48
|
+
.each { |registry, ids| registry.renew(ids, at: now) }
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def beat_loop
|
|
54
|
+
loop do
|
|
55
|
+
sleep interval
|
|
56
|
+
break if synchronize { held.empty? }
|
|
57
|
+
|
|
58
|
+
beat!
|
|
59
|
+
end
|
|
60
|
+
rescue StandardError => e
|
|
61
|
+
Bidi2pdf.logger.warn "session registry heartbeat stopped: #{e.message}"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def held
|
|
65
|
+
reset_after_fork
|
|
66
|
+
@held ||= {}
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def reset_after_fork
|
|
70
|
+
return if @pid == Process.pid
|
|
71
|
+
|
|
72
|
+
@pid = Process.pid
|
|
73
|
+
@held = {}
|
|
74
|
+
@thread = nil
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def synchronize(&)
|
|
78
|
+
@mutex.synchronize(&)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "json"
|
|
5
|
+
require "socket"
|
|
6
|
+
require "tmpdir"
|
|
7
|
+
|
|
8
|
+
module Bidi2pdf
|
|
9
|
+
# Remembers which sessions bidi2pdf opened on a shared (remote) chromedriver, when, by which
|
|
10
|
+
# process, and whether that process is still alive - in a small JSON file per chromedriver URL - so
|
|
11
|
+
# a process started later, or a ChromeSweeper, can find and close the ones a process that died
|
|
12
|
+
# without a clean shutdown left behind (see SessionSweeper). chromedriver itself keeps a session
|
|
13
|
+
# until someone deletes it, and it drops any tag we could attach to one (a custom capability is
|
|
14
|
+
# accepted but echoed nowhere - checked against chromedriver 153), hence a file.
|
|
15
|
+
#
|
|
16
|
+
# Liveness is a lease: a session the process #hold s is renewed by its Heartbeat every
|
|
17
|
+
# Heartbeat.interval seconds. A session whose lease is fresh belongs to a live process - a render
|
|
18
|
+
# in flight or a warm spare - and no sweeper closes it; one whose lease ran out belongs to a dead
|
|
19
|
+
# process. #record alone writes an entry nobody renews.
|
|
20
|
+
#
|
|
21
|
+
# Everything here is best effort and fail-open: a directory that is not writable, a locked-out or
|
|
22
|
+
# corrupt file only switches orphan tracking off for this process (one warning, one
|
|
23
|
+
# +session_warmer.registry_unavailable.bidi2pdf+ notification) - rendering never depends on it.
|
|
24
|
+
# The file is created 0600 and every change happens under an exclusive lock, so processes sharing
|
|
25
|
+
# the directory (Puma workers, a job worker, a spec run) can record at the same time.
|
|
26
|
+
class SessionRegistry
|
|
27
|
+
FILE_PREFIX = "bidi2pdf-sessions-"
|
|
28
|
+
# Seconds a lease stays fresh without a renewal.
|
|
29
|
+
DEFAULT_LEASE_TTL = 60
|
|
30
|
+
|
|
31
|
+
attr_reader :path
|
|
32
|
+
|
|
33
|
+
def self.owner = "#{Socket.gethostname}:#{Process.pid}"
|
|
34
|
+
|
|
35
|
+
def initialize(session_url, dir: nil)
|
|
36
|
+
@path = File.join(dir || Dir.tmpdir, "#{FILE_PREFIX}#{Digest::SHA256.hexdigest(session_url.to_s)[0, 16]}.json")
|
|
37
|
+
@warned = false
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Records a session the caller opened and keeps its lease fresh while this process lives.
|
|
41
|
+
def hold(session_id)
|
|
42
|
+
return false if session_id.nil?
|
|
43
|
+
|
|
44
|
+
recorded = record(session_id)
|
|
45
|
+
Heartbeat.hold(self, session_id)
|
|
46
|
+
recorded
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Stops renewing the lease but keeps the entry - for a session this process could not close, so
|
|
50
|
+
# a sweeper takes it once the lease ran out.
|
|
51
|
+
def release(session_id)
|
|
52
|
+
Heartbeat.release(self, session_id) unless session_id.nil?
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Writes an entry, leased from +created_at+ on but not renewed (see #hold).
|
|
56
|
+
def record(session_id, created_at: Time.now.to_i)
|
|
57
|
+
return false if session_id.nil?
|
|
58
|
+
|
|
59
|
+
update do |entries|
|
|
60
|
+
entries[session_id.to_s] = { "created_at" => created_at.to_i, "renewed_at" => created_at.to_i, "owner" => self.class.owner }
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Forgets a session the caller closed itself (or found gone).
|
|
65
|
+
def forget(session_id)
|
|
66
|
+
return false if session_id.nil?
|
|
67
|
+
|
|
68
|
+
release(session_id)
|
|
69
|
+
update { |entries| entries.delete(session_id.to_s) }
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Renews the leases of +session_ids+ that are still recorded.
|
|
73
|
+
def renew(session_ids, at: Time.now.to_i)
|
|
74
|
+
update do |entries|
|
|
75
|
+
session_ids.each { |id| entries[id]["renewed_at"] = at.to_i if entries.key?(id) }
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Every recorded session: { id => opened at (epoch seconds) }.
|
|
80
|
+
def recorded
|
|
81
|
+
read.transform_values { |entry| entry["created_at"] }
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# The recorded session ids whose lease was renewed within +ttl+ seconds before +now+ - sessions
|
|
85
|
+
# of a live process.
|
|
86
|
+
def leased(now: Time.now.to_i, ttl: DEFAULT_LEASE_TTL)
|
|
87
|
+
read.select { |_, entry| entry["renewed_at"].to_i >= now - ttl }.keys
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# The recorded session ids opened at or before +cutoff+ (epoch seconds).
|
|
91
|
+
def recorded_before(cutoff)
|
|
92
|
+
recorded.select { |_, created_at| created_at <= cutoff }.keys
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
private
|
|
96
|
+
|
|
97
|
+
def read
|
|
98
|
+
return {} unless File.exist?(path)
|
|
99
|
+
|
|
100
|
+
File.open(path, File::RDONLY) do |file|
|
|
101
|
+
file.flock(File::LOCK_SH)
|
|
102
|
+
parse(file.read)
|
|
103
|
+
end
|
|
104
|
+
rescue SystemCallError, IOError => e
|
|
105
|
+
unavailable!(e)
|
|
106
|
+
{}
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def update
|
|
110
|
+
File.open(path, File::RDWR | File::CREAT, 0o600) do |file|
|
|
111
|
+
file.flock(File::LOCK_EX)
|
|
112
|
+
entries = parse(file.read)
|
|
113
|
+
yield entries
|
|
114
|
+
file.rewind
|
|
115
|
+
file.truncate(0)
|
|
116
|
+
file.write(JSON.generate(entries))
|
|
117
|
+
file.flush
|
|
118
|
+
end
|
|
119
|
+
true
|
|
120
|
+
rescue SystemCallError, IOError => e
|
|
121
|
+
unavailable!(e)
|
|
122
|
+
false
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# A corrupt or foreign file counts as empty rather than as an error; an entry of the first
|
|
126
|
+
# format (just the creation time) has no lease.
|
|
127
|
+
def parse(text)
|
|
128
|
+
data = text.to_s.empty? ? {} : JSON.parse(text)
|
|
129
|
+
return {} unless data.is_a?(Hash)
|
|
130
|
+
|
|
131
|
+
data.filter_map { |id, entry| [id, normalize(entry)] if id.is_a?(String) && normalize(entry) }.to_h
|
|
132
|
+
rescue JSON::ParserError
|
|
133
|
+
{}
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def normalize(entry)
|
|
137
|
+
return { "created_at" => entry } if entry.is_a?(Integer)
|
|
138
|
+
|
|
139
|
+
entry if entry.is_a?(Hash) && entry["created_at"].is_a?(Integer)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
def unavailable!(error)
|
|
143
|
+
return if @warned
|
|
144
|
+
|
|
145
|
+
@warned = true
|
|
146
|
+
Bidi2pdf.logger.warn "session_warmer: session registry #{path} unavailable, leftover sessions will not be tracked: #{error.message}"
|
|
147
|
+
Bidi2pdf.notification_service.instrument("session_warmer.registry_unavailable.bidi2pdf", { path: path, error: error.class.name })
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
require_relative "session_registry/heartbeat"
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
require "uri"
|
|
5
|
+
|
|
6
|
+
module Bidi2pdf
|
|
7
|
+
# Closes sessions a SessionWarmer recorded (SessionRegistry) on a shared chromedriver that are
|
|
8
|
+
# older than a cutoff - left behind by a process that died without a clean shutdown, and holding a
|
|
9
|
+
# whole Chrome each until someone deletes them. Run once when a warmer starts.
|
|
10
|
+
#
|
|
11
|
+
# Only recorded sessions are ever touched - another tool's sessions on the same chromedriver never
|
|
12
|
+
# are, and neither is one whose lease a live process keeps renewing (SessionRegistry#hold). A
|
|
13
|
+
# recorded session that is already gone ("invalid session id" / 404) is just forgotten.
|
|
14
|
+
# Fail-open: any error is logged and ends the sweep; it never raises.
|
|
15
|
+
class SessionSweeper
|
|
16
|
+
# @param session_url [String] the chromedriver's new-session URL (".../session").
|
|
17
|
+
# @param registry [SessionRegistry]
|
|
18
|
+
# @param http [#call, nil] (method, url) -> status code; injectable for tests.
|
|
19
|
+
def initialize(session_url, registry, http: nil)
|
|
20
|
+
@session_url = session_url.to_s.chomp("/")
|
|
21
|
+
@registry = registry
|
|
22
|
+
@http = http || method(:net_http)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Closes recorded sessions opened more than +older_than+ seconds before +now+.
|
|
26
|
+
#
|
|
27
|
+
# @return [Integer] how many sessions were actually closed.
|
|
28
|
+
def sweep(older_than:, now: Time.now.to_i)
|
|
29
|
+
leftovers = @registry.recorded_before(now - older_than) - @registry.leased(now: now)
|
|
30
|
+
closed = leftovers.count { |id| closed_now?(id) }
|
|
31
|
+
report(closed)
|
|
32
|
+
closed
|
|
33
|
+
rescue StandardError => e
|
|
34
|
+
Bidi2pdf.logger.warn "session_warmer: sweeping leftover sessions failed: #{e.message}"
|
|
35
|
+
Bidi2pdf.notification_service.instrument("session_warmer.sweep_failed.bidi2pdf", { error: e.class.name })
|
|
36
|
+
0
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
# True when this call closed the session; a session already gone is forgotten but not counted.
|
|
42
|
+
def closed_now?(id)
|
|
43
|
+
status = @http.call(:delete, "#{@session_url}/#{id}")
|
|
44
|
+
return false unless [200, 404].include?(status)
|
|
45
|
+
|
|
46
|
+
@registry.forget(id)
|
|
47
|
+
status == 200
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def report(closed)
|
|
51
|
+
return if closed.zero?
|
|
52
|
+
|
|
53
|
+
Bidi2pdf.logger.info "session_warmer: closed #{closed} leftover session(s) on #{@session_url}"
|
|
54
|
+
Bidi2pdf.notification_service.instrument("session_warmer.orphans_closed.bidi2pdf", { count: closed })
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# chromedriver answers a DELETE of a session it no longer has with 404 "invalid session id".
|
|
58
|
+
def net_http(method, url)
|
|
59
|
+
uri = URI(url)
|
|
60
|
+
request = method == :delete ? Net::HTTP::Delete.new(uri) : Net::HTTP::Get.new(uri)
|
|
61
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 5, read_timeout: 30) do |http|
|
|
62
|
+
http.request(request).code.to_i
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -50,6 +50,24 @@ module Bidi2pdf
|
|
|
50
50
|
# 1.25x this value even when no render ever comes. +nil+ disables the limit.
|
|
51
51
|
attr_accessor :max_idle_age
|
|
52
52
|
|
|
53
|
+
# @return [Numeric, Symbol, nil] Seconds after which a session recorded by any warmer on the
|
|
54
|
+
# same remote chromedriver counts as left behind - its process died without closing it - and
|
|
55
|
+
# is closed when a warmer starts (SessionSweeper). +:auto+ (the default) is twice
|
|
56
|
+
# max_idle_age: a live warmer recycles its own spares within ~1.25x of that, so no live
|
|
57
|
+
# process's slot is ever taken. +nil+ disables the sweep. Remote mode only.
|
|
58
|
+
attr_accessor :orphan_age
|
|
59
|
+
|
|
60
|
+
# @return [String, nil] Directory for the session registry file (SessionRegistry); +nil+ means
|
|
61
|
+
# Dir.tmpdir. Every process that should clean up after the others must share it.
|
|
62
|
+
attr_accessor :registry_dir
|
|
63
|
+
|
|
64
|
+
# @return [Hash, nil] Options for a ChromeSweeper on the remote chromedriver (+scope+,
|
|
65
|
+
# +orphan_age+, +min_age+, +unresponsive_checks+, +max_sessions+, +pids_limit+, +interval+,
|
|
66
|
+
# ...), or +nil+ (default) for none. This warmer's own sessions are never touched by it; with
|
|
67
|
+
# an +interval+ it sweeps in the background, and a session that fails to start triggers one
|
|
68
|
+
# sweep and one retry. Remote mode only.
|
|
69
|
+
attr_accessor :sweeper
|
|
70
|
+
|
|
53
71
|
DEFAULT_MAX_IDLE_AGE = 300
|
|
54
72
|
|
|
55
73
|
def initialize
|
|
@@ -59,12 +77,23 @@ module Bidi2pdf
|
|
|
59
77
|
@remote_browser_url = nil
|
|
60
78
|
@slot_factory = nil
|
|
61
79
|
@max_idle_age = DEFAULT_MAX_IDLE_AGE
|
|
80
|
+
@orphan_age = :auto
|
|
81
|
+
@registry_dir = nil
|
|
82
|
+
@sweeper = nil
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# The orphan age in seconds, or nil when the sweep is off (also when +:auto+ has no
|
|
86
|
+
# max_idle_age to derive it from).
|
|
87
|
+
def effective_orphan_age
|
|
88
|
+
orphan_age == :auto ? max_idle_age && (max_idle_age * 2) : orphan_age
|
|
62
89
|
end
|
|
63
90
|
|
|
64
91
|
# @raise [ArgumentError] if a setting can't be honored - checked once, at construction.
|
|
65
92
|
def validate!
|
|
66
93
|
validate_size!
|
|
67
94
|
validate_max_idle_age!
|
|
95
|
+
validate_orphan_age!
|
|
96
|
+
validate_sweeper!
|
|
68
97
|
end
|
|
69
98
|
|
|
70
99
|
private
|
|
@@ -80,6 +109,18 @@ module Bidi2pdf
|
|
|
80
109
|
|
|
81
110
|
raise ArgumentError, "max_idle_age must be nil or a positive number of seconds, got #{max_idle_age.inspect}"
|
|
82
111
|
end
|
|
112
|
+
|
|
113
|
+
def validate_orphan_age!
|
|
114
|
+
return if orphan_age.nil? || orphan_age == :auto || (orphan_age.is_a?(Numeric) && orphan_age.positive?)
|
|
115
|
+
|
|
116
|
+
raise ArgumentError, "orphan_age must be :auto, nil or a positive number of seconds, got #{orphan_age.inspect}"
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def validate_sweeper!
|
|
120
|
+
return if sweeper.nil? || sweeper.is_a?(Hash)
|
|
121
|
+
|
|
122
|
+
raise ArgumentError, "sweeper must be nil or a Hash of ChromeSweeper options, got #{sweeper.inspect}"
|
|
123
|
+
end
|
|
83
124
|
end
|
|
84
125
|
|
|
85
126
|
class << self
|
|
@@ -140,6 +181,12 @@ module Bidi2pdf
|
|
|
140
181
|
instance.with_tab(&)
|
|
141
182
|
end
|
|
142
183
|
|
|
184
|
+
# Sweeps the remote chromedriver with the configured sweeper; nil when there is none (or no
|
|
185
|
+
# warmer yet - this never creates one).
|
|
186
|
+
def sweep!(reason: :manual)
|
|
187
|
+
@instance&.sweep!(reason: reason)
|
|
188
|
+
end
|
|
189
|
+
|
|
143
190
|
# Retires every currently-warm spare and resets the singleton.
|
|
144
191
|
def shutdown
|
|
145
192
|
@instance&.shutdown
|
|
@@ -181,8 +228,19 @@ module Bidi2pdf
|
|
|
181
228
|
@warming = 0
|
|
182
229
|
@shutdown = false
|
|
183
230
|
@reaper_wakeup = Thread::Queue.new
|
|
231
|
+
@own_sessions = []
|
|
232
|
+
@registry = build_registry
|
|
233
|
+
sweep_leftovers
|
|
234
|
+
@sweeper = build_sweeper
|
|
184
235
|
prewarm
|
|
185
|
-
|
|
236
|
+
start_background_work
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Sweeps the remote chromedriver now - for an application that suspects leaked sessions.
|
|
240
|
+
#
|
|
241
|
+
# @return [ChromeSweeper::Result, nil] nil when no sweeper is configured.
|
|
242
|
+
def sweep!(reason: :manual)
|
|
243
|
+
@sweeper&.sweep!(reason: reason)
|
|
186
244
|
end
|
|
187
245
|
|
|
188
246
|
# Checks out a slot, creates an isolated UserContext/Window/Tab for one render, yields the tab,
|
|
@@ -218,6 +276,7 @@ module Bidi2pdf
|
|
|
218
276
|
|
|
219
277
|
@reaper_wakeup << :stop
|
|
220
278
|
@reaper&.join
|
|
279
|
+
@sweeper&.stop
|
|
221
280
|
threads.each(&:join)
|
|
222
281
|
spares.each { |slot| retire(slot) }
|
|
223
282
|
end
|
|
@@ -235,7 +294,54 @@ module Bidi2pdf
|
|
|
235
294
|
end
|
|
236
295
|
|
|
237
296
|
def create_slot
|
|
297
|
+
slot = new_slot
|
|
298
|
+
id = session_id_of(slot)
|
|
299
|
+
@registry&.hold(id)
|
|
300
|
+
@mutex.synchronize { @own_sessions << id } if id
|
|
301
|
+
slot
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
# A chromedriver out of room for another Chrome refuses the session; with a sweeper, close
|
|
305
|
+
# everything nobody holds (a pressure sweep) and try once more.
|
|
306
|
+
def new_slot
|
|
238
307
|
@slot_factory.call
|
|
308
|
+
rescue Bidi2pdf::SessionNotStartedError
|
|
309
|
+
raise unless @sweeper
|
|
310
|
+
|
|
311
|
+
@sweeper.sweep!(reason: :create_failed, pressure: true)
|
|
312
|
+
@slot_factory.call
|
|
313
|
+
end
|
|
314
|
+
|
|
315
|
+
def start_background_work
|
|
316
|
+
@reaper = Thread.new { reap_loop } if @config.max_idle_age
|
|
317
|
+
@sweeper.start if @sweeper&.interval
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
def build_sweeper
|
|
321
|
+
return unless @config.remote_browser_url && @config.sweeper
|
|
322
|
+
|
|
323
|
+
options = { registry: @registry }.merge(@config.sweeper)
|
|
324
|
+
Bidi2pdf::ChromeSweeper.new(@config.remote_browser_url, own_sessions: -> { @mutex.synchronize { @own_sessions.dup } }, **options)
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
# Remote mode with an orphan age or a sweeper: sessions are recorded, so a later start (or the
|
|
328
|
+
# sweeper, whose default scope is :recorded) can close the ones this process leaves behind if it
|
|
329
|
+
# dies uncleanly. The start-up sweep itself needs the orphan age - see #sweep_leftovers.
|
|
330
|
+
def build_registry
|
|
331
|
+
return unless @config.remote_browser_url && (@config.effective_orphan_age || @config.sweeper)
|
|
332
|
+
|
|
333
|
+
Bidi2pdf::SessionRegistry.new(@config.remote_browser_url, dir: @config.registry_dir)
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
def sweep_leftovers
|
|
337
|
+
return unless @registry && @config.effective_orphan_age
|
|
338
|
+
|
|
339
|
+
Bidi2pdf::SessionSweeper.new(@config.remote_browser_url, @registry).sweep(older_than: @config.effective_orphan_age)
|
|
340
|
+
end
|
|
341
|
+
|
|
342
|
+
def session_id_of(slot)
|
|
343
|
+
session = slot[:session]
|
|
344
|
+
session.session_id if session.respond_to?(:session_id)
|
|
239
345
|
end
|
|
240
346
|
|
|
241
347
|
# Session#started? is just an internal flag set once at startup - it never flips back if Chrome,
|
|
@@ -366,8 +472,13 @@ module Bidi2pdf
|
|
|
366
472
|
retire(slot) if discard
|
|
367
473
|
end
|
|
368
474
|
|
|
475
|
+
# Forgotten only after closing: a process that dies in between leaves a record whose session is
|
|
476
|
+
# already gone - the next sweep just drops it - never an open session nobody knows about.
|
|
369
477
|
def retire(slot)
|
|
370
478
|
self.class.retire_slot(session: slot[:session], manager: slot[:manager])
|
|
479
|
+
id = session_id_of(slot)
|
|
480
|
+
@registry&.forget(id)
|
|
481
|
+
@mutex.synchronize { @own_sessions.delete(id) }
|
|
371
482
|
end
|
|
372
483
|
|
|
373
484
|
def safe_close(label, &)
|
data/lib/bidi2pdf/version.rb
CHANGED
data/lib/bidi2pdf.rb
CHANGED
|
@@ -10,6 +10,10 @@ require_relative "bidi2pdf/dsl"
|
|
|
10
10
|
require_relative "bidi2pdf/notifications"
|
|
11
11
|
require_relative "bidi2pdf/notifications/logging_subscriber"
|
|
12
12
|
require_relative "bidi2pdf/notifications/json_subscriber"
|
|
13
|
+
require_relative "bidi2pdf/session_registry"
|
|
14
|
+
require_relative "bidi2pdf/session_sweeper"
|
|
15
|
+
require_relative "bidi2pdf/chromedriver_api"
|
|
16
|
+
require_relative "bidi2pdf/chrome_sweeper"
|
|
13
17
|
require_relative "bidi2pdf/session_warmer"
|
|
14
18
|
require_relative "bidi2pdf/verbose_logger"
|
|
15
19
|
|
|
@@ -59,6 +59,13 @@ module Bidi2pdf
|
|
|
59
59
|
|
|
60
60
|
def emit: (untyped event, *untyped) -> untyped
|
|
61
61
|
|
|
62
|
+
# The reader closes the socket from its own thread when the peer hangs up, and #close does not
|
|
63
|
+
# wait for a write in flight - so a write can fail with "stream closed in another thread" (or
|
|
64
|
+
# EPIPE) because *we* closed it. By then the connection is over and :close has been emitted;
|
|
65
|
+
# there is nothing left to report. Seen in the handshake write of #connect, which had no
|
|
66
|
+
# rescue: a server that answered and hung up at once made #connect raise although the
|
|
67
|
+
# handshake went through. Any other write failure still raises (#send and #say_goodbye
|
|
68
|
+
# handle it).
|
|
62
69
|
def write: (untyped bytes) -> untyped
|
|
63
70
|
|
|
64
71
|
def say_goodbye: () -> untyped
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
module Bidi
|
|
3
|
+
module Commands
|
|
4
|
+
# browsingContext.getTree - the session's top-level browsing contexts (tabs/windows).
|
|
5
|
+
class BrowsingContextGetTree
|
|
6
|
+
include Base
|
|
7
|
+
|
|
8
|
+
def method_name: () -> "browsingContext.getTree"
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
module Bidi
|
|
3
|
+
module Commands
|
|
4
|
+
# goog:cdp.sendCommand - chromedriver's BiDi extension that runs one Chrome DevTools Protocol
|
|
5
|
+
# command; without a +session+ it runs at browser level (e.g. SystemInfo.getProcessInfo).
|
|
6
|
+
class CdpSendCommand
|
|
7
|
+
@cdp_method: untyped
|
|
8
|
+
|
|
9
|
+
@cdp_params: untyped
|
|
10
|
+
|
|
11
|
+
@session: untyped
|
|
12
|
+
|
|
13
|
+
include Base
|
|
14
|
+
|
|
15
|
+
def initialize: (method: untyped, ?params: ::Hash[untyped, untyped], ?session: untyped?) -> void
|
|
16
|
+
|
|
17
|
+
def params: () -> untyped
|
|
18
|
+
|
|
19
|
+
def method_name: () -> "goog:cdp.sendCommand"
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|