bidi2pdf 0.1.17 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +32 -2
- data/README.md +111 -0
- data/lib/bidi2pdf/bidi/commands/browsing_context_get_tree.rb +16 -0
- data/lib/bidi2pdf/bidi/commands/cdp_send_command.rb +25 -0
- data/lib/bidi2pdf/bidi/commands.rb +2 -0
- data/lib/bidi2pdf/bidi/session.rb +53 -24
- data/lib/bidi2pdf/chrome_sweeper/inspector.rb +117 -0
- data/lib/bidi2pdf/chrome_sweeper/settings.rb +80 -0
- data/lib/bidi2pdf/chrome_sweeper.rb +398 -0
- data/lib/bidi2pdf/chromedriver_api.rb +72 -0
- data/lib/bidi2pdf/cli/session_commands.rb +82 -0
- data/lib/bidi2pdf/cli.rb +43 -0
- data/lib/bidi2pdf/dsl.rb +2 -1
- data/lib/bidi2pdf/launcher.rb +2 -1
- data/lib/bidi2pdf/session_registry/heartbeat.rb +86 -0
- data/lib/bidi2pdf/session_registry.rb +77 -13
- data/lib/bidi2pdf/session_sweeper.rb +5 -3
- data/lib/bidi2pdf/session_warmer.rb +74 -7
- data/lib/bidi2pdf/version.rb +1 -1
- data/lib/bidi2pdf.rb +2 -0
- data/sig/bidi2pdf/bidi/commands/browsing_context_get_tree.rbs +12 -0
- data/sig/bidi2pdf/bidi/commands/cdp_send_command.rbs +23 -0
- data/sig/bidi2pdf/bidi/session.rbs +19 -3
- data/sig/bidi2pdf/chrome_sweeper/inspector.rbs +61 -0
- data/sig/bidi2pdf/chrome_sweeper/settings.rbs +35 -0
- data/sig/bidi2pdf/chrome_sweeper.rbs +220 -0
- data/sig/bidi2pdf/chromedriver_api.rbs +44 -0
- data/sig/bidi2pdf/cli/session_commands.rbs +35 -0
- data/sig/bidi2pdf/cli.rbs +6 -0
- data/sig/bidi2pdf/session_registry/heartbeat.rbs +50 -0
- data/sig/bidi2pdf/session_registry.rbs +46 -9
- data/sig/bidi2pdf/session_sweeper.rbs +2 -1
- data/sig/bidi2pdf/session_warmer.rbs +42 -2
- data/sig/bidi2pdf/version.rbs +1 -1
- metadata +17 -1
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bidi2pdf
|
|
4
|
+
class SessionRegistry
|
|
5
|
+
# Keeps the leases of every session this process holds fresh: one thread per process renews
|
|
6
|
+
# them in their registry files every +interval+ seconds, so a sweeper in another process (a
|
|
7
|
+
# second Puma worker, a job worker sharing the registry directory) can tell a live session
|
|
8
|
+
# from one whose process died. The thread runs only while something is held.
|
|
9
|
+
#
|
|
10
|
+
# Fork-aware: a child process (Puma forks its workers) starts with nothing held and without the
|
|
11
|
+
# parent's thread - the parent's sessions are the parent's to renew.
|
|
12
|
+
module Heartbeat
|
|
13
|
+
DEFAULT_INTERVAL = DEFAULT_LEASE_TTL / 3.0
|
|
14
|
+
|
|
15
|
+
@mutex = Mutex.new
|
|
16
|
+
|
|
17
|
+
class << self
|
|
18
|
+
# @return [Numeric] seconds between renewals - keep it well below the sweeper's lease_ttl.
|
|
19
|
+
attr_writer :interval
|
|
20
|
+
|
|
21
|
+
def interval = @interval || DEFAULT_INTERVAL
|
|
22
|
+
|
|
23
|
+
# The TTL this process writes into its leases: three missed renewals and it is taken for dead.
|
|
24
|
+
def lease_ttl = interval * 3
|
|
25
|
+
|
|
26
|
+
def hold(registry, session_id)
|
|
27
|
+
synchronize do
|
|
28
|
+
(held[registry.path] ||= [registry, Set.new])[1] << session_id.to_s
|
|
29
|
+
@thread = Thread.new { beat_loop } unless @thread&.alive?
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def release(registry, session_id)
|
|
34
|
+
synchronize do
|
|
35
|
+
ids = held.dig(registry.path, 1)
|
|
36
|
+
next unless ids
|
|
37
|
+
|
|
38
|
+
ids.delete(session_id.to_s)
|
|
39
|
+
held.delete(registry.path) if ids.empty?
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# @return [Array<String>] the session ids this process holds in +registry+'s file.
|
|
44
|
+
def held_ids(registry)
|
|
45
|
+
synchronize { held.dig(registry.path, 1).to_a }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Renews every held lease now.
|
|
49
|
+
def beat!(now: Time.now.to_f)
|
|
50
|
+
synchronize { held.values.map { |registry, ids| [registry, ids.to_a] } }
|
|
51
|
+
.each { |registry, ids| registry.renew(ids, at: now, ttl: lease_ttl) }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def beat_loop
|
|
57
|
+
loop do
|
|
58
|
+
sleep interval
|
|
59
|
+
break if synchronize { held.empty? }
|
|
60
|
+
|
|
61
|
+
beat!
|
|
62
|
+
end
|
|
63
|
+
rescue StandardError => e
|
|
64
|
+
Bidi2pdf.logger.warn "session registry heartbeat stopped: #{e.message}"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def held
|
|
68
|
+
reset_after_fork
|
|
69
|
+
@held ||= {}
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def reset_after_fork
|
|
73
|
+
return if @pid == Process.pid
|
|
74
|
+
|
|
75
|
+
@pid = Process.pid
|
|
76
|
+
@held = {}
|
|
77
|
+
@thread = nil
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def synchronize(&)
|
|
81
|
+
@mutex.synchronize(&)
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
@@ -2,48 +2,103 @@
|
|
|
2
2
|
|
|
3
3
|
require "digest"
|
|
4
4
|
require "json"
|
|
5
|
+
require "socket"
|
|
5
6
|
require "tmpdir"
|
|
6
7
|
|
|
7
8
|
module Bidi2pdf
|
|
8
|
-
# Remembers which sessions
|
|
9
|
-
#
|
|
10
|
-
# process
|
|
11
|
-
#
|
|
12
|
-
#
|
|
9
|
+
# Remembers which sessions bidi2pdf opened on a shared (remote) chromedriver, when, by which
|
|
10
|
+
# process, and whether that process is still alive - in a small JSON file per chromedriver URL - so
|
|
11
|
+
# a process started later, or a ChromeSweeper, can find and close the ones a process that died
|
|
12
|
+
# without a clean shutdown left behind (see SessionSweeper). chromedriver itself keeps a session
|
|
13
|
+
# until someone deletes it, and it drops any tag we could attach to one (a custom capability is
|
|
14
|
+
# accepted but echoed nowhere - checked against chromedriver 153), hence a file.
|
|
15
|
+
#
|
|
16
|
+
# Liveness is a lease: a session the process #hold s is renewed by its Heartbeat every
|
|
17
|
+
# Heartbeat.interval seconds, and every entry carries its own TTL - the time the owner promises to
|
|
18
|
+
# renew it within (three heartbeats) - so a sweeper in another process never has to guess how
|
|
19
|
+
# often that owner renews. A session whose lease is fresh belongs to a live process - a render in
|
|
20
|
+
# flight or a warm spare - and no sweeper closes it; one whose lease ran out belongs to a dead
|
|
21
|
+
# process. #record alone writes an entry nobody renews.
|
|
13
22
|
#
|
|
14
23
|
# Everything here is best effort and fail-open: a directory that is not writable, a locked-out or
|
|
15
24
|
# corrupt file only switches orphan tracking off for this process (one warning, one
|
|
16
25
|
# +session_warmer.registry_unavailable.bidi2pdf+ notification) - rendering never depends on it.
|
|
17
26
|
# The file is created 0600 and every change happens under an exclusive lock, so processes sharing
|
|
18
|
-
# the directory (
|
|
27
|
+
# the directory (Puma workers, a job worker, a spec run) can record at the same time.
|
|
19
28
|
class SessionRegistry
|
|
20
29
|
FILE_PREFIX = "bidi2pdf-sessions-"
|
|
30
|
+
# Seconds a lease stays fresh without a renewal when its entry carries no TTL of its own (written
|
|
31
|
+
# by bidi2pdf 0.1.18) - matches that version's fixed 20 s heartbeat.
|
|
32
|
+
DEFAULT_LEASE_TTL = 60
|
|
21
33
|
|
|
22
34
|
attr_reader :path
|
|
23
35
|
|
|
36
|
+
def self.owner = "#{Socket.gethostname}:#{Process.pid}"
|
|
37
|
+
|
|
24
38
|
def initialize(session_url, dir: nil)
|
|
25
39
|
@path = File.join(dir || Dir.tmpdir, "#{FILE_PREFIX}#{Digest::SHA256.hexdigest(session_url.to_s)[0, 16]}.json")
|
|
26
40
|
@warned = false
|
|
27
41
|
end
|
|
28
42
|
|
|
29
|
-
# Records a session the caller
|
|
30
|
-
def
|
|
43
|
+
# Records a session the caller opened and keeps its lease fresh while this process lives.
|
|
44
|
+
def hold(session_id)
|
|
45
|
+
return false if session_id.nil?
|
|
46
|
+
|
|
47
|
+
recorded = record(session_id, ttl: Heartbeat.lease_ttl)
|
|
48
|
+
Heartbeat.hold(self, session_id)
|
|
49
|
+
recorded
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Stops renewing the lease but keeps the entry - for a session this process could not close, so
|
|
53
|
+
# a sweeper takes it once the lease ran out.
|
|
54
|
+
def release(session_id)
|
|
55
|
+
Heartbeat.release(self, session_id) unless session_id.nil?
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Writes an entry, leased from +created_at+ on for +ttl+ seconds (nil: the reader's default) but
|
|
59
|
+
# not renewed (see #hold).
|
|
60
|
+
# Times are stored as fractions of a second: a whole-second timestamp could cost a short lease
|
|
61
|
+
# (0.5 s heartbeats: 1.5 s) up to a second of its life. Entries of whole seconds still read.
|
|
62
|
+
def record(session_id, created_at: Time.now.to_f, ttl: nil)
|
|
31
63
|
return false if session_id.nil?
|
|
32
64
|
|
|
33
|
-
update
|
|
65
|
+
update do |entries|
|
|
66
|
+
entries[session_id.to_s] = { "created_at" => created_at.to_f, "renewed_at" => created_at.to_f, "ttl" => ttl, "owner" => self.class.owner }.compact
|
|
67
|
+
end
|
|
34
68
|
end
|
|
35
69
|
|
|
36
70
|
# Forgets a session the caller closed itself (or found gone).
|
|
37
71
|
def forget(session_id)
|
|
38
72
|
return false if session_id.nil?
|
|
39
73
|
|
|
74
|
+
release(session_id)
|
|
40
75
|
update { |entries| entries.delete(session_id.to_s) }
|
|
41
76
|
end
|
|
42
77
|
|
|
78
|
+
# Renews the leases of +session_ids+ that are still recorded, for +ttl+ seconds when given.
|
|
79
|
+
def renew(session_ids, at: Time.now.to_f, ttl: nil)
|
|
80
|
+
update do |entries|
|
|
81
|
+
session_ids.select { |id| entries.key?(id) }.each do |id|
|
|
82
|
+
entries[id]["renewed_at"] = at.to_f
|
|
83
|
+
entries[id]["ttl"] = ttl if ttl
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Every recorded session: { id => opened at (epoch seconds) }.
|
|
89
|
+
def recorded
|
|
90
|
+
read.transform_values { |entry| entry["created_at"] }
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# The recorded session ids whose lease is still fresh at +now+ - sessions of a live process. An
|
|
94
|
+
# entry's own TTL wins; +ttl+ is only for entries without one.
|
|
95
|
+
def leased(now: Time.now.to_f, ttl: DEFAULT_LEASE_TTL)
|
|
96
|
+
read.select { |_, entry| entry["renewed_at"].to_f + (entry["ttl"] || ttl) >= now }.keys
|
|
97
|
+
end
|
|
98
|
+
|
|
43
99
|
# The recorded session ids opened at or before +cutoff+ (epoch seconds).
|
|
44
100
|
def recorded_before(cutoff)
|
|
45
|
-
|
|
46
|
-
entries.select { |_, created_at| created_at <= cutoff }.keys
|
|
101
|
+
recorded.select { |_, created_at| created_at <= cutoff }.keys
|
|
47
102
|
end
|
|
48
103
|
|
|
49
104
|
private
|
|
@@ -76,16 +131,23 @@ module Bidi2pdf
|
|
|
76
131
|
false
|
|
77
132
|
end
|
|
78
133
|
|
|
79
|
-
# A corrupt or foreign file counts as empty rather than as an error
|
|
134
|
+
# A corrupt or foreign file counts as empty rather than as an error; an entry of the first
|
|
135
|
+
# format (just the creation time) has no lease.
|
|
80
136
|
def parse(text)
|
|
81
137
|
data = text.to_s.empty? ? {} : JSON.parse(text)
|
|
82
138
|
return {} unless data.is_a?(Hash)
|
|
83
139
|
|
|
84
|
-
data.
|
|
140
|
+
data.filter_map { |id, entry| [id, normalize(entry)] if id.is_a?(String) && normalize(entry) }.to_h
|
|
85
141
|
rescue JSON::ParserError
|
|
86
142
|
{}
|
|
87
143
|
end
|
|
88
144
|
|
|
145
|
+
def normalize(entry)
|
|
146
|
+
return { "created_at" => entry } if entry.is_a?(Numeric)
|
|
147
|
+
|
|
148
|
+
entry if entry.is_a?(Hash) && entry["created_at"].is_a?(Numeric)
|
|
149
|
+
end
|
|
150
|
+
|
|
89
151
|
def unavailable!(error)
|
|
90
152
|
return if @warned
|
|
91
153
|
|
|
@@ -95,3 +157,5 @@ module Bidi2pdf
|
|
|
95
157
|
end
|
|
96
158
|
end
|
|
97
159
|
end
|
|
160
|
+
|
|
161
|
+
require_relative "session_registry/heartbeat"
|
|
@@ -9,7 +9,8 @@ module Bidi2pdf
|
|
|
9
9
|
# whole Chrome each until someone deletes them. Run once when a warmer starts.
|
|
10
10
|
#
|
|
11
11
|
# Only recorded sessions are ever touched - another tool's sessions on the same chromedriver never
|
|
12
|
-
# are
|
|
12
|
+
# are, and neither is one whose lease a live process keeps renewing (SessionRegistry#hold). A
|
|
13
|
+
# recorded session that is already gone ("invalid session id" / 404) is just forgotten.
|
|
13
14
|
# Fail-open: any error is logged and ends the sweep; it never raises.
|
|
14
15
|
class SessionSweeper
|
|
15
16
|
# @param session_url [String] the chromedriver's new-session URL (".../session").
|
|
@@ -24,8 +25,9 @@ module Bidi2pdf
|
|
|
24
25
|
# Closes recorded sessions opened more than +older_than+ seconds before +now+.
|
|
25
26
|
#
|
|
26
27
|
# @return [Integer] how many sessions were actually closed.
|
|
27
|
-
def sweep(older_than:, now: Time.now.
|
|
28
|
-
|
|
28
|
+
def sweep(older_than:, now: Time.now.to_f)
|
|
29
|
+
leftovers = @registry.recorded_before(now - older_than) - @registry.leased(now: now)
|
|
30
|
+
closed = leftovers.count { |id| closed_now?(id) }
|
|
29
31
|
report(closed)
|
|
30
32
|
closed
|
|
31
33
|
rescue StandardError => e
|
|
@@ -61,6 +61,17 @@ module Bidi2pdf
|
|
|
61
61
|
# Dir.tmpdir. Every process that should clean up after the others must share it.
|
|
62
62
|
attr_accessor :registry_dir
|
|
63
63
|
|
|
64
|
+
# @return [Hash, nil] Options for a ChromeSweeper on the remote chromedriver (+scope+,
|
|
65
|
+
# +orphan_age+, +min_age+, +unresponsive_checks+, +max_sessions+, +pids_limit+, +interval+,
|
|
66
|
+
# ...), or +nil+ (default) for none. This warmer's own sessions are never touched by it; with
|
|
67
|
+
# an +interval+ it sweeps in the background, and a session that fails to start triggers one
|
|
68
|
+
# pressure sweep and one retry (see +retry_refused_sessions+). Remote mode only.
|
|
69
|
+
attr_accessor :sweeper
|
|
70
|
+
|
|
71
|
+
# @return [Boolean] With a +sweeper+: when chromedriver refuses a new session, sweep under
|
|
72
|
+
# pressure and try once more. Default true; false leaves a refused session to the caller.
|
|
73
|
+
attr_accessor :retry_refused_sessions
|
|
74
|
+
|
|
64
75
|
DEFAULT_MAX_IDLE_AGE = 300
|
|
65
76
|
|
|
66
77
|
def initialize
|
|
@@ -72,6 +83,8 @@ module Bidi2pdf
|
|
|
72
83
|
@max_idle_age = DEFAULT_MAX_IDLE_AGE
|
|
73
84
|
@orphan_age = :auto
|
|
74
85
|
@registry_dir = nil
|
|
86
|
+
@sweeper = nil
|
|
87
|
+
@retry_refused_sessions = true
|
|
75
88
|
end
|
|
76
89
|
|
|
77
90
|
# The orphan age in seconds, or nil when the sweep is off (also when +:auto+ has no
|
|
@@ -85,6 +98,7 @@ module Bidi2pdf
|
|
|
85
98
|
validate_size!
|
|
86
99
|
validate_max_idle_age!
|
|
87
100
|
validate_orphan_age!
|
|
101
|
+
validate_sweeper!
|
|
88
102
|
end
|
|
89
103
|
|
|
90
104
|
private
|
|
@@ -106,6 +120,12 @@ module Bidi2pdf
|
|
|
106
120
|
|
|
107
121
|
raise ArgumentError, "orphan_age must be :auto, nil or a positive number of seconds, got #{orphan_age.inspect}"
|
|
108
122
|
end
|
|
123
|
+
|
|
124
|
+
def validate_sweeper!
|
|
125
|
+
return if sweeper.nil? || sweeper.is_a?(Hash)
|
|
126
|
+
|
|
127
|
+
raise ArgumentError, "sweeper must be nil or a Hash of ChromeSweeper options, got #{sweeper.inspect}"
|
|
128
|
+
end
|
|
109
129
|
end
|
|
110
130
|
|
|
111
131
|
class << self
|
|
@@ -166,6 +186,12 @@ module Bidi2pdf
|
|
|
166
186
|
instance.with_tab(&)
|
|
167
187
|
end
|
|
168
188
|
|
|
189
|
+
# Sweeps the remote chromedriver with the configured sweeper; nil when there is none (or no
|
|
190
|
+
# warmer yet - this never creates one).
|
|
191
|
+
def sweep!(reason: :manual)
|
|
192
|
+
@instance&.sweep!(reason: reason)
|
|
193
|
+
end
|
|
194
|
+
|
|
169
195
|
# Retires every currently-warm spare and resets the singleton.
|
|
170
196
|
def shutdown
|
|
171
197
|
@instance&.shutdown
|
|
@@ -207,10 +233,19 @@ module Bidi2pdf
|
|
|
207
233
|
@warming = 0
|
|
208
234
|
@shutdown = false
|
|
209
235
|
@reaper_wakeup = Thread::Queue.new
|
|
236
|
+
@own_sessions = []
|
|
210
237
|
@registry = build_registry
|
|
211
238
|
sweep_leftovers
|
|
239
|
+
@sweeper = build_sweeper
|
|
212
240
|
prewarm
|
|
213
|
-
|
|
241
|
+
start_background_work
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Sweeps the remote chromedriver now - for an application that suspects leaked sessions.
|
|
245
|
+
#
|
|
246
|
+
# @return [ChromeSweeper::Result, nil] nil when no sweeper is configured.
|
|
247
|
+
def sweep!(reason: :manual)
|
|
248
|
+
@sweeper&.sweep!(reason: reason)
|
|
214
249
|
end
|
|
215
250
|
|
|
216
251
|
# Checks out a slot, creates an isolated UserContext/Window/Tab for one render, yields the tab,
|
|
@@ -246,6 +281,7 @@ module Bidi2pdf
|
|
|
246
281
|
|
|
247
282
|
@reaper_wakeup << :stop
|
|
248
283
|
@reaper&.join
|
|
284
|
+
@sweeper&.stop
|
|
249
285
|
threads.each(&:join)
|
|
250
286
|
spares.each { |slot| retire(slot) }
|
|
251
287
|
end
|
|
@@ -263,19 +299,48 @@ module Bidi2pdf
|
|
|
263
299
|
end
|
|
264
300
|
|
|
265
301
|
def create_slot
|
|
266
|
-
|
|
302
|
+
slot = new_slot
|
|
303
|
+
id = session_id_of(slot)
|
|
304
|
+
@registry&.hold(id)
|
|
305
|
+
@mutex.synchronize { @own_sessions << id } if id
|
|
306
|
+
slot
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# A chromedriver out of room for another Chrome refuses the session; with a sweeper, close
|
|
310
|
+
# everything nobody holds (a pressure sweep) and try once more.
|
|
311
|
+
def new_slot
|
|
312
|
+
@slot_factory.call
|
|
313
|
+
rescue Bidi2pdf::SessionNotStartedError
|
|
314
|
+
raise unless @sweeper && @config.retry_refused_sessions
|
|
315
|
+
|
|
316
|
+
@sweeper.sweep!(reason: :create_failed, pressure: true)
|
|
317
|
+
@slot_factory.call
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
def start_background_work
|
|
321
|
+
@reaper = Thread.new { reap_loop } if @config.max_idle_age
|
|
322
|
+
@sweeper.start if @sweeper&.interval
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
def build_sweeper
|
|
326
|
+
return unless @config.remote_browser_url && @config.sweeper
|
|
327
|
+
|
|
328
|
+
options = { registry: @registry }.merge(@config.sweeper)
|
|
329
|
+
Bidi2pdf::ChromeSweeper.new(@config.remote_browser_url, own_sessions: -> { @mutex.synchronize { @own_sessions.dup } }, **options)
|
|
267
330
|
end
|
|
268
331
|
|
|
269
|
-
# Remote mode
|
|
270
|
-
#
|
|
332
|
+
# Remote mode: every session is leased, so no sweeper in any process takes a warm spare or a
|
|
333
|
+
# render in flight, and a later start (or a sweeper) can close the ones this process leaves
|
|
334
|
+
# behind if it dies uncleanly. The start-up sweep itself needs the orphan age - see
|
|
335
|
+
# #sweep_leftovers.
|
|
271
336
|
def build_registry
|
|
272
|
-
return unless @config.remote_browser_url
|
|
337
|
+
return unless @config.remote_browser_url
|
|
273
338
|
|
|
274
339
|
Bidi2pdf::SessionRegistry.new(@config.remote_browser_url, dir: @config.registry_dir)
|
|
275
340
|
end
|
|
276
341
|
|
|
277
342
|
def sweep_leftovers
|
|
278
|
-
return unless @registry
|
|
343
|
+
return unless @registry && @config.effective_orphan_age
|
|
279
344
|
|
|
280
345
|
Bidi2pdf::SessionSweeper.new(@config.remote_browser_url, @registry).sweep(older_than: @config.effective_orphan_age)
|
|
281
346
|
end
|
|
@@ -417,7 +482,9 @@ module Bidi2pdf
|
|
|
417
482
|
# already gone - the next sweep just drops it - never an open session nobody knows about.
|
|
418
483
|
def retire(slot)
|
|
419
484
|
self.class.retire_slot(session: slot[:session], manager: slot[:manager])
|
|
420
|
-
|
|
485
|
+
id = session_id_of(slot)
|
|
486
|
+
@registry&.forget(id)
|
|
487
|
+
@mutex.synchronize { @own_sessions.delete(id) }
|
|
421
488
|
end
|
|
422
489
|
|
|
423
490
|
def safe_close(label, &)
|
data/lib/bidi2pdf/version.rb
CHANGED
data/lib/bidi2pdf.rb
CHANGED
|
@@ -12,6 +12,8 @@ require_relative "bidi2pdf/notifications/logging_subscriber"
|
|
|
12
12
|
require_relative "bidi2pdf/notifications/json_subscriber"
|
|
13
13
|
require_relative "bidi2pdf/session_registry"
|
|
14
14
|
require_relative "bidi2pdf/session_sweeper"
|
|
15
|
+
require_relative "bidi2pdf/chromedriver_api"
|
|
16
|
+
require_relative "bidi2pdf/chrome_sweeper"
|
|
15
17
|
require_relative "bidi2pdf/session_warmer"
|
|
16
18
|
require_relative "bidi2pdf/verbose_logger"
|
|
17
19
|
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
module Bidi
|
|
3
|
+
module Commands
|
|
4
|
+
# browsingContext.getTree - the session's top-level browsing contexts (tabs/windows).
|
|
5
|
+
class BrowsingContextGetTree
|
|
6
|
+
include Base
|
|
7
|
+
|
|
8
|
+
def method_name: () -> "browsingContext.getTree"
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
module Bidi
|
|
3
|
+
module Commands
|
|
4
|
+
# goog:cdp.sendCommand - chromedriver's BiDi extension that runs one Chrome DevTools Protocol
|
|
5
|
+
# command; without a +session+ it runs at browser level (e.g. SystemInfo.getProcessInfo).
|
|
6
|
+
class CdpSendCommand
|
|
7
|
+
@cdp_method: untyped
|
|
8
|
+
|
|
9
|
+
@cdp_params: untyped
|
|
10
|
+
|
|
11
|
+
@session: untyped
|
|
12
|
+
|
|
13
|
+
include Base
|
|
14
|
+
|
|
15
|
+
def initialize: (method: untyped, ?params: ::Hash[untyped, untyped], ?session: untyped?) -> void
|
|
16
|
+
|
|
17
|
+
def params: () -> untyped
|
|
18
|
+
|
|
19
|
+
def method_name: () -> "goog:cdp.sendCommand"
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -27,6 +27,8 @@ module Bidi2pdf
|
|
|
27
27
|
|
|
28
28
|
@chrome_args: untyped
|
|
29
29
|
|
|
30
|
+
@registry: untyped
|
|
31
|
+
|
|
30
32
|
@client: untyped
|
|
31
33
|
|
|
32
34
|
@browser: untyped
|
|
@@ -59,7 +61,10 @@ module Bidi2pdf
|
|
|
59
61
|
# @param [String] session_url The URL for the session.
|
|
60
62
|
# @param [Boolean] headless Whether to run the browser in headless mode. Defaults to true.
|
|
61
63
|
# @param [Array<String>] chrome_args Additional Chrome arguments. Defaults to predefined arguments.
|
|
62
|
-
|
|
64
|
+
# @param [Bidi2pdf::SessionRegistry, nil] registry Records the session on a shared (remote)
|
|
65
|
+
# chromedriver while it is open, so a sweep can close it if this process dies without
|
|
66
|
+
# closing it (ChromeSweeper, SessionSweeper). nil records nothing.
|
|
67
|
+
def initialize: (session_url: untyped, ?headless: bool, ?chrome_args: untyped, ?registry: untyped?) -> void
|
|
63
68
|
|
|
64
69
|
# Starts the session and initializes the client.
|
|
65
70
|
#
|
|
@@ -76,8 +81,8 @@ module Bidi2pdf
|
|
|
76
81
|
# @return [Bidi2pdf::Bidi::Browser] The browser instance.
|
|
77
82
|
def browser: () -> untyped
|
|
78
83
|
|
|
79
|
-
# Closes the session and cleans up resources.
|
|
80
|
-
#
|
|
84
|
+
# Closes the session and cleans up resources: BiDi +session.end+ first, and when Chrome does not
|
|
85
|
+
# answer that (a tab stuck in a loop), chromedriver's +DELETE /session/{id}+.
|
|
81
86
|
def close: () -> untyped
|
|
82
87
|
|
|
83
88
|
# Retrieves user contexts for the session.
|
|
@@ -164,6 +169,17 @@ module Bidi2pdf
|
|
|
164
169
|
# @raise [SessionNotStartedError] If the session could not be started.
|
|
165
170
|
def handle_error: (untyped value) -> (nil | untyped)
|
|
166
171
|
|
|
172
|
+
def end_via_bidi: () -> untyped
|
|
173
|
+
|
|
174
|
+
def record_session: (untyped id) -> untyped
|
|
175
|
+
|
|
176
|
+
# The BiDi session.end got no answer - typically a Chrome that hangs or crashed. chromedriver
|
|
177
|
+
# still ends the session (and kills its Chrome) on an HTTP DELETE, so a hung Chrome does not
|
|
178
|
+
# outlive this process. Only for a session this object created over HTTP (it knows the id).
|
|
179
|
+
#
|
|
180
|
+
# @return [Boolean] true when the session is gone.
|
|
181
|
+
def delete_via_chromedriver: () -> untyped
|
|
182
|
+
|
|
167
183
|
# Cleans up resources associated with the session.
|
|
168
184
|
def cleanup: () -> untyped
|
|
169
185
|
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
class ChromeSweeper
|
|
3
|
+
# Looks at one session on a chromedriver from the outside: attaches a second BiDi connection to
|
|
4
|
+
# its +webSocketUrl+ (chromedriver allows that for any session), asks for the tab tree, the first
|
|
5
|
+
# tab's +performance.timeOrigin+ (Chrome keeps no session start time anywhere - the first tab is
|
|
6
|
+
# created with the session, so its time origin is the session's age to within a few hundred ms)
|
|
7
|
+
# and the renderer CPU times (CDP +SystemInfo.getProcessInfo+ through +goog:cdp.sendCommand+).
|
|
8
|
+
#
|
|
9
|
+
# Read-only, and never reads page content: tab URLs are counted, not returned or logged (a
|
|
10
|
+
# +data:+ URL carries the whole rendered document). Closes its own connection every time.
|
|
11
|
+
class Inspector
|
|
12
|
+
@timeout: untyped
|
|
13
|
+
|
|
14
|
+
@clock: untyped
|
|
15
|
+
|
|
16
|
+
@client_factory: untyped
|
|
17
|
+
|
|
18
|
+
# @!attribute age [Float, nil] seconds since the session started, nil when nothing tells.
|
|
19
|
+
# @!attribute source [Symbol] :registry, :tab or :unknown - where +age+ comes from.
|
|
20
|
+
# @!attribute responsive [Boolean] whether the session answered the checks.
|
|
21
|
+
# @!attribute cpu_times [Hash{Integer => Float}] renderer pid => CPU seconds.
|
|
22
|
+
# @!attribute live [Boolean] a live process holds its lease (never inspected, never closed).
|
|
23
|
+
SessionInfo: untyped
|
|
24
|
+
|
|
25
|
+
DEFAULT_TIMEOUT: 5
|
|
26
|
+
|
|
27
|
+
# @param clock [#call] epoch seconds as a Float.
|
|
28
|
+
# @param client_factory [#call] ws_url -> a started, open Bidi::Client; injectable for tests.
|
|
29
|
+
def initialize: (?timeout: untyped, ?clock: untyped, ?client_factory: untyped?) -> void
|
|
30
|
+
|
|
31
|
+
# @param entry [ChromedriverApi::Entry]
|
|
32
|
+
# @param recorded_at [Numeric, nil] epoch seconds the registry has for it; wins over the tab.
|
|
33
|
+
# @return [SessionInfo]
|
|
34
|
+
def examine: (untyped entry, ?recorded_at: untyped?) -> untyped
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def probe: (untyped client, untyped entry, untyped recorded_at) -> untyped
|
|
39
|
+
|
|
40
|
+
def tree: (untyped client) -> untyped
|
|
41
|
+
|
|
42
|
+
# A tab stuck in an endless loop never evaluates anything - nil then.
|
|
43
|
+
def time_origin: (untyped client, untyped context) -> untyped
|
|
44
|
+
|
|
45
|
+
# On a freshly attached connection the first command to a session - and, seen once in the
|
|
46
|
+
# acceptance run, to a tab - can time out although the session is healthy (chromedriver
|
|
47
|
+
# 153/154); the second one answers. Only a second timeout counts.
|
|
48
|
+
def once_more_on_timeout: () { () -> untyped } -> untyped
|
|
49
|
+
|
|
50
|
+
def cpu_times: (untyped client) -> untyped
|
|
51
|
+
|
|
52
|
+
def age_of: (untyped recorded_at, untyped origin) -> (::Array[untyped | :registry] | ::Array[untyped | :tab] | ::Array[nil | :unknown])
|
|
53
|
+
|
|
54
|
+
def unreachable: (untyped entry, untyped recorded_at) -> untyped
|
|
55
|
+
|
|
56
|
+
def command: (untyped client, untyped cmd) -> untyped
|
|
57
|
+
|
|
58
|
+
def connect: (untyped ws_url) -> untyped
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
class ChromeSweeper
|
|
3
|
+
SCOPES: ::Array[:recorded | :all]
|
|
4
|
+
|
|
5
|
+
DEFAULT_ORPHAN_AGE: 600
|
|
6
|
+
|
|
7
|
+
DEFAULT_MIN_AGE: 60
|
|
8
|
+
|
|
9
|
+
DEFAULT_UNRESPONSIVE_CHECKS: 2
|
|
10
|
+
|
|
11
|
+
DEFAULT_PIDS_BUDGET: ::Float
|
|
12
|
+
|
|
13
|
+
DEFAULT_THREADS_PER_SESSION: 110
|
|
14
|
+
|
|
15
|
+
DEFAULT_LEASE_TTL: untyped
|
|
16
|
+
|
|
17
|
+
# What a ChromeSweeper closes and when - validated on construction; an unknown setting raises
|
|
18
|
+
# ArgumentError.
|
|
19
|
+
#
|
|
20
|
+
# @!attribute scope [Symbol] :recorded (sessions a SessionRegistry knows) or :all.
|
|
21
|
+
# @!attribute orphan_age [Numeric, nil] close sessions older than this; nil turns the rule off.
|
|
22
|
+
# @!attribute min_age [Numeric] never close a session younger than this.
|
|
23
|
+
# @!attribute unresponsive_checks [Integer, nil] close after this many failed checks in a row.
|
|
24
|
+
# @!attribute max_sessions [Integer, Symbol, nil] the limit; +:auto+ derives it from pids_limit.
|
|
25
|
+
# @!attribute pids_limit [Integer, nil] the chromedriver container's pids limit, for +:auto+.
|
|
26
|
+
# @!attribute pids_budget [Float] share of +pids_limit+ Chrome sessions may use.
|
|
27
|
+
# @!attribute threads_per_session [Integer] threads (Docker counts them as pids) of one session.
|
|
28
|
+
# @!attribute interval [Numeric, nil] seconds between sweeps of ChromeSweeper#start's thread.
|
|
29
|
+
# @!attribute dry_run [Boolean] report what would be closed, close nothing.
|
|
30
|
+
# @!attribute lease_ttl [Numeric] for registry entries without a TTL of their own (written by
|
|
31
|
+
# bidi2pdf 0.1.18): renewed within this many seconds means a live process holds it. Entries
|
|
32
|
+
# written since carry the TTL their owner's heartbeat promises (SessionRegistry#hold).
|
|
33
|
+
Settings: untyped
|
|
34
|
+
end
|
|
35
|
+
end
|