bidi2pdf 0.1.17 → 0.1.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +32 -2
- data/README.md +111 -0
- data/lib/bidi2pdf/bidi/commands/browsing_context_get_tree.rb +16 -0
- data/lib/bidi2pdf/bidi/commands/cdp_send_command.rb +25 -0
- data/lib/bidi2pdf/bidi/commands.rb +2 -0
- data/lib/bidi2pdf/bidi/session.rb +53 -24
- data/lib/bidi2pdf/chrome_sweeper/inspector.rb +117 -0
- data/lib/bidi2pdf/chrome_sweeper/settings.rb +80 -0
- data/lib/bidi2pdf/chrome_sweeper.rb +398 -0
- data/lib/bidi2pdf/chromedriver_api.rb +72 -0
- data/lib/bidi2pdf/cli/session_commands.rb +82 -0
- data/lib/bidi2pdf/cli.rb +43 -0
- data/lib/bidi2pdf/dsl.rb +2 -1
- data/lib/bidi2pdf/launcher.rb +2 -1
- data/lib/bidi2pdf/session_registry/heartbeat.rb +86 -0
- data/lib/bidi2pdf/session_registry.rb +77 -13
- data/lib/bidi2pdf/session_sweeper.rb +5 -3
- data/lib/bidi2pdf/session_warmer.rb +74 -7
- data/lib/bidi2pdf/version.rb +1 -1
- data/lib/bidi2pdf.rb +2 -0
- data/sig/bidi2pdf/bidi/commands/browsing_context_get_tree.rbs +12 -0
- data/sig/bidi2pdf/bidi/commands/cdp_send_command.rbs +23 -0
- data/sig/bidi2pdf/bidi/session.rbs +19 -3
- data/sig/bidi2pdf/chrome_sweeper/inspector.rbs +61 -0
- data/sig/bidi2pdf/chrome_sweeper/settings.rbs +35 -0
- data/sig/bidi2pdf/chrome_sweeper.rbs +220 -0
- data/sig/bidi2pdf/chromedriver_api.rbs +44 -0
- data/sig/bidi2pdf/cli/session_commands.rbs +35 -0
- data/sig/bidi2pdf/cli.rbs +6 -0
- data/sig/bidi2pdf/session_registry/heartbeat.rbs +50 -0
- data/sig/bidi2pdf/session_registry.rbs +46 -9
- data/sig/bidi2pdf/session_sweeper.rbs +2 -1
- data/sig/bidi2pdf/session_warmer.rbs +42 -2
- data/sig/bidi2pdf/version.rbs +1 -1
- metadata +17 -1
|
@@ -0,0 +1,398 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "forwardable"
|
|
4
|
+
require_relative "chromedriver_api"
|
|
5
|
+
require_relative "session_registry"
|
|
6
|
+
require_relative "chrome_sweeper/settings"
|
|
7
|
+
require_relative "chrome_sweeper/inspector"
|
|
8
|
+
|
|
9
|
+
module Bidi2pdf
|
|
10
|
+
# Closes Chrome sessions a shared chromedriver holds that nobody will close any more - left behind
|
|
11
|
+
# by a process that died, or stuck in a tab that never returns - and keeps the number of sessions
|
|
12
|
+
# below a limit. chromedriver keeps every session (a whole Chrome) until someone deletes it.
|
|
13
|
+
#
|
|
14
|
+
# Which sessions it looks at (+scope+):
|
|
15
|
+
# - +:recorded+ (default) - only sessions a SessionRegistry proves were opened through bidi2pdf.
|
|
16
|
+
# - +:all+ - every session on that chromedriver (+GET /sessions+), aged by its first tab. Only for
|
|
17
|
+
# a chromedriver the application owns: on a shared one it closes other clients' old sessions.
|
|
18
|
+
#
|
|
19
|
+
# What it closes, in this order - never a session of +own_sessions+, never one whose lease a live
|
|
20
|
+
# process keeps renewing (SessionRegistry#hold - a render in flight or a warm spare of another
|
|
21
|
+
# worker), and never one younger than +min_age+, not even over the limit:
|
|
22
|
+
# 1. sessions older than +orphan_age+;
|
|
23
|
+
# 2. sessions that failed +unresponsive_checks+ checks in a row (no answer, or a renderer burning
|
|
24
|
+
# CPU at full speed between two sweeps - an endless loop);
|
|
25
|
+
# 3. while more sessions exist than +max_sessions+, the oldest remaining ones. When that is not
|
|
26
|
+
# enough, the sweep reports +limit_exceeded+ and closes nothing more.
|
|
27
|
+
#
|
|
28
|
+
# Run it once (#sweep!, ChromeSweeper.sweep!) - e.g. when the application suspects a leak - or
|
|
29
|
+
# periodically (+interval+, #start/#stop). As a last resort when a render fails for lack of
|
|
30
|
+
# resources, #with_retry sweeps under +pressure+ - every session nobody holds and older than
|
|
31
|
+
# +min_age+ goes - and tries once more. One sweep at a time, across processes too (a lock file
|
|
32
|
+
# next to the registry). Fail-open: a sweep never raises; errors land in the Result. See Settings
|
|
33
|
+
# for every setting and its default.
|
|
34
|
+
#
|
|
35
|
+
# @example A periodic sweeper for an application's own chromedriver
|
|
36
|
+
# sweeper = Bidi2pdf::ChromeSweeper.new("http://remote-chrome:3000/session", scope: :all,
|
|
37
|
+
# max_sessions: :auto, pids_limit: 1024, interval: 60)
|
|
38
|
+
# sweeper.start
|
|
39
|
+
class ChromeSweeper
|
|
40
|
+
extend Forwardable
|
|
41
|
+
|
|
42
|
+
# One closed (or, in a dry run, would-be closed) session. +why+: :orphan, :unresponsive,
|
|
43
|
+
# :over_limit or :pressure.
|
|
44
|
+
Closed = Data.define(:id, :age, :why)
|
|
45
|
+
|
|
46
|
+
# What one sweep saw and did.
|
|
47
|
+
Result = Data.define(:reason, :sessions, :inspected, :closed, :unresponsive, :limit, :limit_exceeded,
|
|
48
|
+
:dry_run, :skipped, :errors, :duration) do
|
|
49
|
+
def closed_count = closed.size
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# A renderer that used at least this share of the wall-clock time between two sweeps counts as
|
|
53
|
+
# stuck in a loop.
|
|
54
|
+
BUSY_CPU_SHARE = 0.9
|
|
55
|
+
|
|
56
|
+
# Errors #with_retry sweeps and retries on by default: a session chromedriver refused, and a
|
|
57
|
+
# connection or command that died or timed out - what a Chrome out of memory or pids looks like.
|
|
58
|
+
# A command Chrome answered with an error (CmdError) is the page's problem, not a resource one.
|
|
59
|
+
RETRYABLE = ->(error) { error.is_a?(Bidi2pdf::SessionNotStartedError) || (error.is_a?(Bidi2pdf::WebsocketError) && !error.is_a?(Bidi2pdf::CmdError)) }
|
|
60
|
+
|
|
61
|
+
# The bookkeeping of one sweep.
|
|
62
|
+
Sweep = Struct.new(:total, :closed, :unresponsive, :errors, :limit_exceeded, :pressure) do
|
|
63
|
+
def initialize(total = 0, pressure: false) = super(total, [], [], [], false, pressure)
|
|
64
|
+
|
|
65
|
+
def remaining = total - closed.size
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
attr_reader :session_url, :settings, :registry
|
|
69
|
+
|
|
70
|
+
def_delegators :@settings, :scope, :orphan_age, :min_age, :unresponsive_checks, :interval, :dry_run, :limit, :lease_ttl
|
|
71
|
+
|
|
72
|
+
# One-shot sweep with a throw-away sweeper. Give it a +check_interval+ for the unresponsive rule
|
|
73
|
+
# to apply - see #sweep!.
|
|
74
|
+
#
|
|
75
|
+
# @return [Result]
|
|
76
|
+
def self.sweep!(session_url, reason: :manual, check_interval: nil, pressure: false, **)
|
|
77
|
+
new(session_url, **).sweep!(reason: reason, check_interval: check_interval, pressure: pressure)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# #with_retry with a throw-away sweeper.
|
|
81
|
+
def self.with_retry(session_url, retry_on: RETRYABLE, **, &)
|
|
82
|
+
new(session_url, **).with_retry(retry_on: retry_on, &)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# @param session_url [String] chromedriver's new-session URL (".../session").
|
|
86
|
+
# @param registry [SessionRegistry, nil] defaults to the one for +session_url+ in +registry_dir+.
|
|
87
|
+
# @param own_sessions [#call] returns the ids of the caller's live sessions - never touched.
|
|
88
|
+
# @param api [ChromedriverApi, nil], inspector [Inspector, nil], clock [#call] - for tests.
|
|
89
|
+
# @param settings see Settings.
|
|
90
|
+
# @raise [InvalidConfigError] for a setting out of range.
|
|
91
|
+
def initialize(session_url, registry: nil, registry_dir: nil, own_sessions: -> { [] },
|
|
92
|
+
api: nil, inspector: nil, clock: -> { Time.now.to_f }, **settings)
|
|
93
|
+
@session_url = session_url.to_s
|
|
94
|
+
@settings = Settings.new(**settings)
|
|
95
|
+
@own_sessions = own_sessions
|
|
96
|
+
@clock = clock
|
|
97
|
+
@registry = registry || Bidi2pdf::SessionRegistry.new(@session_url, dir: registry_dir)
|
|
98
|
+
@api = api || Bidi2pdf::ChromedriverApi.new(@session_url)
|
|
99
|
+
@inspector = inspector || Inspector.new(clock: clock)
|
|
100
|
+
@mutex = Mutex.new
|
|
101
|
+
@wakeup = Thread::Queue.new
|
|
102
|
+
@failures = Hash.new(0)
|
|
103
|
+
@first_seen = {}
|
|
104
|
+
@cpu_samples = {}
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Every session in scope, inspected (age, tabs, responsive) - for diagnostics; closes nothing.
|
|
108
|
+
#
|
|
109
|
+
# @return [Array<Inspector::SessionInfo>]
|
|
110
|
+
def sessions
|
|
111
|
+
recorded = @registry.recorded
|
|
112
|
+
live = live_sessions
|
|
113
|
+
in_scope(@api.sessions, recorded).map do |entry|
|
|
114
|
+
next live_info(entry.id, recorded[entry.id]) if live.include?(entry.id)
|
|
115
|
+
|
|
116
|
+
@inspector.examine(entry, recorded_at: recorded[entry.id])
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Runs the block; when it fails with a resource error (+retry_on+: a callable or a list of error
|
|
121
|
+
# classes, RETRYABLE by default), sweeps under pressure and runs it once more. The block must
|
|
122
|
+
# be safe to run twice - it gets a new session the second time. Other errors, and a second
|
|
123
|
+
# failure, propagate unchanged.
|
|
124
|
+
def with_retry(retry_on: RETRYABLE)
|
|
125
|
+
yield
|
|
126
|
+
rescue StandardError => e
|
|
127
|
+
raise unless retryable?(retry_on, e)
|
|
128
|
+
|
|
129
|
+
Bidi2pdf.logger.warn "chrome_sweeper: #{e.class}: #{e.message} - sweeping #{@session_url} and trying once more"
|
|
130
|
+
Bidi2pdf.notification_service.instrument("chrome_sweeper.retry.bidi2pdf", { error: e.class.name })
|
|
131
|
+
sweep!(reason: :render_failed, pressure: true)
|
|
132
|
+
yield
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Inspects every session in scope and counts its checks, like a sweep, but closes nothing.
|
|
136
|
+
#
|
|
137
|
+
# @return [Array<Inspector::SessionInfo>]
|
|
138
|
+
def observe
|
|
139
|
+
@mutex.synchronize { observe_entries(@api.sessions) }
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Sweeps once. Skipped (Result#skipped) when another sweep - in this or another process - is
|
|
143
|
+
# still running.
|
|
144
|
+
#
|
|
145
|
+
# A session counts as hung only after +unresponsive_checks+ failed checks, one per sweep - a
|
|
146
|
+
# one-shot sweep never gets there. With +check_interval+ it first checks
|
|
147
|
+
# +unresponsive_checks - 1+ times, +check_interval+ seconds apart (#observe), so a single call
|
|
148
|
+
# applies every rule. The periodic thread doesn't need that: its sweeps are the checks.
|
|
149
|
+
#
|
|
150
|
+
# @param reason [Symbol] why - reported in the notification (:manual, :periodic, :create_failed…).
|
|
151
|
+
# Under +pressure+ - the last resort when a render failed - every eligible session is closed:
|
|
152
|
+
# nobody holds it, and it is older than +min_age+; +orphan_age+, the unresponsive checks and the
|
|
153
|
+
# limit no longer matter.
|
|
154
|
+
#
|
|
155
|
+
# @param check_interval [Numeric, nil] seconds between those checks; nil sweeps right away.
|
|
156
|
+
# @param pressure [Boolean] close everything nobody holds that is past +min_age+.
|
|
157
|
+
# @return [Result]
|
|
158
|
+
def sweep!(reason: :manual, check_interval: nil, pressure: false)
|
|
159
|
+
check_first(check_interval) if check_interval
|
|
160
|
+
started = monotonic
|
|
161
|
+
return result(reason, started, skipped: true) unless @mutex.try_lock
|
|
162
|
+
|
|
163
|
+
begin
|
|
164
|
+
with_lock_file { |locked| locked ? run(reason, started, pressure) : result(reason, started, skipped: true) }
|
|
165
|
+
ensure
|
|
166
|
+
@mutex.unlock
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Starts the periodic sweep thread (needs an +interval+). Idempotent.
|
|
171
|
+
def start
|
|
172
|
+
raise Bidi2pdf::InvalidConfigError, "chrome_sweeper: start needs an interval" unless interval
|
|
173
|
+
|
|
174
|
+
@thread ||= Thread.new { sweep!(reason: :periodic) until @wakeup.pop(timeout: interval) == :stop }
|
|
175
|
+
self
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Stops the periodic sweep thread and waits for a sweep in progress to finish.
|
|
179
|
+
def stop
|
|
180
|
+
return unless @thread
|
|
181
|
+
|
|
182
|
+
@wakeup << :stop
|
|
183
|
+
@thread.join
|
|
184
|
+
@thread = nil
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
private
|
|
188
|
+
|
|
189
|
+
def run(reason, started, pressure)
|
|
190
|
+
sweep, infos = sweep_sessions(pressure)
|
|
191
|
+
report(result(reason, started, sweep: sweep, inspected: infos))
|
|
192
|
+
rescue StandardError => e
|
|
193
|
+
Bidi2pdf.logger.warn "chrome_sweeper: sweep of #{@session_url} failed: #{e.message}"
|
|
194
|
+
Bidi2pdf.notification_service.instrument("chrome_sweeper.failed.bidi2pdf", { reason: reason, error: e.class.name })
|
|
195
|
+
result(reason, started, sweep: Sweep.new.tap { |failed| failed.errors << e.message })
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# A config error raises; a failed check only means the sweep has fewer checks to go on.
|
|
199
|
+
def check_first(check_interval)
|
|
200
|
+
raise Bidi2pdf::InvalidConfigError, "chrome_sweeper: check_interval must be a positive number, got #{check_interval.inspect}" unless check_interval.is_a?(Numeric) && check_interval.positive?
|
|
201
|
+
|
|
202
|
+
(unresponsive_checks.to_i - 1).times { observe_then_wait(check_interval) }
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def observe_then_wait(check_interval)
|
|
206
|
+
observe
|
|
207
|
+
rescue StandardError => e
|
|
208
|
+
Bidi2pdf.logger.warn "chrome_sweeper: checking #{@session_url} before the sweep failed: #{e.message}"
|
|
209
|
+
ensure
|
|
210
|
+
sleep check_interval
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def sweep_sessions(pressure)
|
|
214
|
+
entries = @api.sessions
|
|
215
|
+
infos = observe_entries(entries)
|
|
216
|
+
sweep = Sweep.new(entries.size, pressure: pressure)
|
|
217
|
+
decide(infos.select { |info| info.age >= min_age }, sweep)
|
|
218
|
+
[sweep, infos]
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# In-scope sessions not owned by the caller, not held by a live process and not known to be
|
|
222
|
+
# younger than min_age, with the registry time when there is one. None of the others is even
|
|
223
|
+
# attached to - a render in flight is left alone entirely.
|
|
224
|
+
def candidates(entries)
|
|
225
|
+
recorded = @registry.recorded
|
|
226
|
+
untouchable = Array(@own_sessions.call).map(&:to_s) + live_sessions
|
|
227
|
+
in_scope(entries, recorded).filter_map do |entry|
|
|
228
|
+
recorded_at = recorded[entry.id]
|
|
229
|
+
next if untouchable.include?(entry.id) || (recorded_at && now - recorded_at < min_age)
|
|
230
|
+
|
|
231
|
+
[entry, recorded_at]
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def live_sessions
|
|
236
|
+
@registry.leased(now: now, ttl: lease_ttl)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def live_info(id, recorded_at)
|
|
240
|
+
Inspector::SessionInfo.new(id: id, age: recorded_at && (now - recorded_at), source: :registry, tabs: nil,
|
|
241
|
+
responsive: nil, cpu_times: {}, live: true)
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def retryable?(retry_on, error)
|
|
245
|
+
retry_on.respond_to?(:call) ? retry_on.call(error) : Array(retry_on).any? { |klass| error.is_a?(klass) }
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def in_scope(entries, recorded)
|
|
249
|
+
scope == :all ? entries : entries.select { |entry| recorded.key?(entry.id) }
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def observe_entries(entries)
|
|
253
|
+
forget_gone(entries)
|
|
254
|
+
candidates(entries).map { |entry, recorded_at| track(@inspector.examine(entry, recorded_at: recorded_at)) }
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# Sessions chromedriver no longer has are dropped from the registry and from this sweeper's own
|
|
258
|
+
# tracking - most sessions end normally, so a long-running sweeper would otherwise keep every id
|
|
259
|
+
# it ever saw.
|
|
260
|
+
def forget_gone(entries)
|
|
261
|
+
ids = entries.map(&:id)
|
|
262
|
+
(@registry.recorded.keys - ids).each { |id| @registry.forget(id) }
|
|
263
|
+
(@first_seen.keys - ids).each { |id| forget_tracking(id) }
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def forget_tracking(id)
|
|
267
|
+
[@failures, @first_seen, @cpu_samples].each { |tracked| tracked.delete(id) }
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# Counts failed checks per session across sweeps, and ages a session by when this sweeper first
|
|
271
|
+
# saw it when nothing better is known.
|
|
272
|
+
def track(info)
|
|
273
|
+
count_check(info)
|
|
274
|
+
@first_seen[info.id] ||= now
|
|
275
|
+
info.with(age: [info.age || 0, now - @first_seen[info.id]].max)
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
def count_check(info)
|
|
279
|
+
failed = !info.responsive || busy?(info)
|
|
280
|
+
@failures[info.id] = failed ? @failures[info.id] + 1 : 0
|
|
281
|
+
@cpu_samples[info.id] = [now, info.cpu_times]
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def busy?(info)
|
|
285
|
+
sampled_at, times = @cpu_samples[info.id]
|
|
286
|
+
return false unless sampled_at && now > sampled_at
|
|
287
|
+
|
|
288
|
+
info.cpu_times.any? { |pid, cpu| times.key?(pid) && (cpu - times[pid]) / (now - sampled_at) >= BUSY_CPU_SHARE }
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def decide(eligible, sweep)
|
|
292
|
+
eligible.each do |info|
|
|
293
|
+
why = close_reason(info, sweep) || (:pressure if sweep.pressure)
|
|
294
|
+
close_session(info, why, sweep) if why
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
enforce_limit(eligible, sweep)
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def close_reason(info, sweep)
|
|
301
|
+
failed = @failures[info.id]
|
|
302
|
+
note_unresponsive(info, failed, sweep) if unresponsive_checks && failed.positive?
|
|
303
|
+
return :orphan if orphan_age && info.age > orphan_age
|
|
304
|
+
|
|
305
|
+
:unresponsive if unresponsive_checks && failed >= unresponsive_checks
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
def note_unresponsive(info, failed, sweep)
|
|
309
|
+
sweep.unresponsive << info.id
|
|
310
|
+
Bidi2pdf.notification_service.instrument("chrome_sweeper.unresponsive.bidi2pdf", { id: info.id, checks: failed })
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def enforce_limit(eligible, sweep)
|
|
314
|
+
return unless over_limit?(sweep)
|
|
315
|
+
|
|
316
|
+
oldest_open(eligible, sweep).each do |info|
|
|
317
|
+
break unless over_limit?(sweep)
|
|
318
|
+
|
|
319
|
+
close_session(info, :over_limit, sweep)
|
|
320
|
+
end
|
|
321
|
+
limit_exceeded(sweep) if over_limit?(sweep)
|
|
322
|
+
end
|
|
323
|
+
|
|
324
|
+
def over_limit?(sweep) = !limit.nil? && sweep.remaining > limit
|
|
325
|
+
|
|
326
|
+
def oldest_open(eligible, sweep)
|
|
327
|
+
closed_ids = sweep.closed.map(&:id)
|
|
328
|
+
eligible.reject { |info| closed_ids.include?(info.id) }.sort_by { |info| -info.age }
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
def limit_exceeded(sweep)
|
|
332
|
+
sweep.limit_exceeded = true
|
|
333
|
+
Bidi2pdf.logger.warn "chrome_sweeper: #{sweep.remaining} sessions on #{@session_url}, limit #{limit} - none left old enough to close"
|
|
334
|
+
Bidi2pdf.notification_service.instrument("chrome_sweeper.limit_exceeded.bidi2pdf", { sessions: sweep.remaining, limit: limit })
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
# Adds the session to the sweep's closed list unless chromedriver refused to close it.
|
|
338
|
+
def close_session(info, why, sweep)
|
|
339
|
+
return sweep.errors << "closing #{info.id} failed" unless dry_run || gone_after_delete?(info.id)
|
|
340
|
+
|
|
341
|
+
sweep.closed << announce(Closed.new(id: info.id, age: info.age.round, why: why))
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
def announce(closed)
|
|
345
|
+
Bidi2pdf.notification_service.instrument("chrome_sweeper.closed.bidi2pdf", closed.to_h.merge(dry_run: dry_run))
|
|
346
|
+
closed
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def gone_after_delete?(id)
|
|
350
|
+
return false if @api.delete_session(id) == :failed
|
|
351
|
+
|
|
352
|
+
@registry.forget(id)
|
|
353
|
+
forget_tracking(id)
|
|
354
|
+
true
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
def report(result)
|
|
358
|
+
verb = dry_run ? "would close" : "closed"
|
|
359
|
+
Bidi2pdf.logger.info "chrome_sweeper: #{verb} #{result.closed_count} of #{result.sessions} session(s) on #{@session_url}" if result.closed.any?
|
|
360
|
+
Bidi2pdf.notification_service.instrument("chrome_sweeper.sweep.bidi2pdf", sweep_payload(result))
|
|
361
|
+
result
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
def sweep_payload(result)
|
|
365
|
+
result.to_h.slice(:reason, :sessions, :limit, :limit_exceeded, :dry_run, :duration)
|
|
366
|
+
.merge(inspected: result.inspected.size, closed: result.closed_count)
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
def result(reason, started, sweep: Sweep.new, inspected: [], skipped: false)
|
|
370
|
+
Result.new(reason: reason, sessions: sweep.total, inspected: inspected, closed: sweep.closed,
|
|
371
|
+
unresponsive: sweep.unresponsive, limit: limit, limit_exceeded: sweep.limit_exceeded,
|
|
372
|
+
dry_run: dry_run, skipped: skipped, errors: sweep.errors, duration: monotonic - started)
|
|
373
|
+
end
|
|
374
|
+
|
|
375
|
+
# A lock file next to the registry, so processes sharing it never sweep at the same time. When
|
|
376
|
+
# the file cannot be created at all the sweep still runs (fail-open, like the registry itself).
|
|
377
|
+
def with_lock_file
|
|
378
|
+
file = open_lock_file
|
|
379
|
+
return yield(true) unless file
|
|
380
|
+
|
|
381
|
+
begin
|
|
382
|
+
yield file.flock(File::LOCK_EX | File::LOCK_NB)
|
|
383
|
+
ensure
|
|
384
|
+
file.close
|
|
385
|
+
end
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def open_lock_file
|
|
389
|
+
File.open("#{@registry.path}.lock", File::RDWR | File::CREAT, 0o600)
|
|
390
|
+
rescue SystemCallError
|
|
391
|
+
nil
|
|
392
|
+
end
|
|
393
|
+
|
|
394
|
+
def now = @clock.call
|
|
395
|
+
|
|
396
|
+
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
397
|
+
end
|
|
398
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "net/http"
|
|
5
|
+
require "uri"
|
|
6
|
+
|
|
7
|
+
module Bidi2pdf
|
|
8
|
+
# The two chromedriver HTTP endpoints session cleanup needs, next to the BiDi protocol: the
|
|
9
|
+
# non-standard +GET /sessions+ (every session on that chromedriver - WebDriver itself has no way to
|
|
10
|
+
# list them, BiDi neither) and +DELETE /session/{id}+ (ends a session and its Chrome even when that
|
|
11
|
+
# Chrome no longer answers BiDi).
|
|
12
|
+
#
|
|
13
|
+
# Built from the new-session URL every other part of bidi2pdf is configured with
|
|
14
|
+
# ("http://host:3000/session").
|
|
15
|
+
class ChromedriverApi
|
|
16
|
+
# One entry of +GET /sessions+: chromedriver gives no creation time, no owner, and no hint
|
|
17
|
+
# whether a client is still connected (checked against chromedriver 153/154).
|
|
18
|
+
Entry = Data.define(:id, :websocket_url, :process_id)
|
|
19
|
+
|
|
20
|
+
DEFAULT_TIMEOUT = 30
|
|
21
|
+
|
|
22
|
+
attr_reader :session_url
|
|
23
|
+
|
|
24
|
+
# @param session_url [String] chromedriver's new-session URL (".../session").
|
|
25
|
+
# @param http [#call, nil] (method, url) -> [status, body]; injectable for tests.
|
|
26
|
+
def initialize(session_url, http: nil, timeout: DEFAULT_TIMEOUT)
|
|
27
|
+
@session_url = session_url.to_s.chomp("/")
|
|
28
|
+
@timeout = timeout
|
|
29
|
+
@http = http || method(:net_http)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @return [Array<Entry>] every session chromedriver holds right now.
|
|
33
|
+
def sessions
|
|
34
|
+
status, body = @http.call(:get, sessions_url)
|
|
35
|
+
raise Bidi2pdf::Error, "chromedriver answered #{status} to GET /sessions" unless status == 200
|
|
36
|
+
|
|
37
|
+
Array(JSON.parse(body)["value"]).map do |session|
|
|
38
|
+
capabilities = session["capabilities"] || {}
|
|
39
|
+
Entry.new(id: session["id"], websocket_url: capabilities["webSocketUrl"], process_id: capabilities["goog:processID"])
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Ends a session. chromedriver answers 404 ("invalid session id") for one it no longer has.
|
|
44
|
+
#
|
|
45
|
+
# @return [Symbol] :closed, :gone or :failed
|
|
46
|
+
def delete_session(id)
|
|
47
|
+
status, = @http.call(:delete, "#{session_url}/#{id}")
|
|
48
|
+
case status
|
|
49
|
+
when 200 then :closed
|
|
50
|
+
when 404 then :gone
|
|
51
|
+
else :failed
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
# "http://host:3000/session" -> "http://host:3000/sessions"; a chromedriver started with a
|
|
58
|
+
# --url-base keeps its prefix.
|
|
59
|
+
def sessions_url
|
|
60
|
+
"#{session_url.delete_suffix("/session")}/sessions"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def net_http(method, url)
|
|
64
|
+
uri = URI(url)
|
|
65
|
+
request = method == :delete ? Net::HTTP::Delete.new(uri) : Net::HTTP::Get.new(uri)
|
|
66
|
+
Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 5, read_timeout: @timeout) do |http|
|
|
67
|
+
response = http.request(request)
|
|
68
|
+
[response.code.to_i, response.body]
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bidi2pdf
|
|
4
|
+
class CLI < Thor
|
|
5
|
+
# Output of `bidi2pdf sessions` and `bidi2pdf sweep` (ChromeSweeper). Never prints a tab URL or
|
|
6
|
+
# any page content - only ids, ages, tab counts and what was done.
|
|
7
|
+
module SessionCommands
|
|
8
|
+
# CLI flag => ChromeSweeper setting, for the ones that are only passed when given.
|
|
9
|
+
SWEEP_FLAGS = { older_than: :orphan_age, max_sessions: :max_sessions, min_age: :min_age,
|
|
10
|
+
unresponsive_checks: :unresponsive_checks }.freeze
|
|
11
|
+
# Settings that are counts - Thor hands every numeric over as a Float.
|
|
12
|
+
COUNT_SETTINGS = %i[max_sessions unresponsive_checks].freeze
|
|
13
|
+
|
|
14
|
+
private
|
|
15
|
+
|
|
16
|
+
def chrome_sweeper
|
|
17
|
+
url = options[:remote_browser_url]
|
|
18
|
+
raise Thor::Error, "--remote-browser-url is required" if url.to_s.empty?
|
|
19
|
+
|
|
20
|
+
Bidi2pdf::ChromeSweeper.new(url, scope: options[:scope].to_sym, dry_run: options[:dry_run], **sweep_flags)
|
|
21
|
+
rescue Bidi2pdf::InvalidConfigError => e
|
|
22
|
+
raise Thor::Error, e.message
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def sweep_flags
|
|
26
|
+
SWEEP_FLAGS.filter_map do |flag, setting|
|
|
27
|
+
value = options[flag]
|
|
28
|
+
[setting, COUNT_SETTINGS.include?(setting) ? value.to_i : value] unless value.nil?
|
|
29
|
+
end.to_h
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# --check-interval 0 means no checks before the sweep.
|
|
33
|
+
def run_sweep
|
|
34
|
+
interval = options[:check_interval].to_f
|
|
35
|
+
chrome_sweeper.sweep!(reason: :cli, check_interval: interval.zero? ? nil : interval, pressure: options[:pressure])
|
|
36
|
+
rescue Bidi2pdf::InvalidConfigError => e
|
|
37
|
+
raise Thor::Error, e.message
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def session_hash(info)
|
|
41
|
+
{ id: info.id, age: info.age&.round, source: info.source, tabs: info.tabs, responsive: info.responsive, live: info.live }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def print_sessions(infos)
|
|
45
|
+
return puts "no sessions" if infos.empty?
|
|
46
|
+
|
|
47
|
+
infos.each do |info|
|
|
48
|
+
age = info.age ? "#{info.age.round}s" : "?"
|
|
49
|
+
puts format("%-34<id>s %8<age>s %-8<source>s %9<tabs>s %<state>s",
|
|
50
|
+
id: info.id, age: age, source: info.source, tabs: info.tabs ? "#{info.tabs} tab(s)" : "", state: session_state(info))
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# A live session is not attached to, so there is nothing to say about its tabs.
|
|
55
|
+
def session_state(info)
|
|
56
|
+
return "live (a running process holds it)" if info.live
|
|
57
|
+
|
|
58
|
+
info.responsive ? "responsive" : "unresponsive"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def sweep_hash(result)
|
|
62
|
+
{ reason: result.reason, sessions: result.sessions, dry_run: result.dry_run, skipped: result.skipped,
|
|
63
|
+
closed: result.closed.map(&:to_h), unresponsive: result.unresponsive, limit: result.limit,
|
|
64
|
+
limit_exceeded: result.limit_exceeded, errors: result.errors }
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def print_sweep(result)
|
|
68
|
+
return puts "skipped: another sweep is running" if result.skipped
|
|
69
|
+
|
|
70
|
+
verb = result.dry_run ? "would close" : "closed"
|
|
71
|
+
result.closed.each { |closed| puts "#{verb} #{closed.id} (#{closed.why}, #{closed.age}s old)" }
|
|
72
|
+
puts "#{verb} #{result.closed_count} of #{result.sessions} session(s)"
|
|
73
|
+
print_sweep_problems(result)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def print_sweep_problems(result)
|
|
77
|
+
puts "limit #{result.limit} still exceeded - no session old enough to close" if result.limit_exceeded
|
|
78
|
+
result.errors.each { |error| warn "error: #{error}" }
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
data/lib/bidi2pdf/cli.rb
CHANGED
|
@@ -5,11 +5,13 @@ require "yaml"
|
|
|
5
5
|
require "tempfile"
|
|
6
6
|
|
|
7
7
|
require_relative "cli/json_output"
|
|
8
|
+
require_relative "cli/session_commands"
|
|
8
9
|
|
|
9
10
|
module Bidi2pdf
|
|
10
11
|
# rubocop:disable Metrics/AbcSize
|
|
11
12
|
class CLI < Thor
|
|
12
13
|
include JsonOutput
|
|
14
|
+
include SessionCommands
|
|
13
15
|
|
|
14
16
|
class_option :config, type: :string, desc: "Load configuration from YAML file"
|
|
15
17
|
|
|
@@ -183,6 +185,47 @@ module Bidi2pdf
|
|
|
183
185
|
end
|
|
184
186
|
map "run" => :run_recipe
|
|
185
187
|
|
|
188
|
+
desc "sessions", "List the Chrome sessions a remote chromedriver holds: id, age, tabs, responsive (no URLs)"
|
|
189
|
+
option :remote_browser_url, type: :string, required: true, desc: "The chromedriver's new-session URL (http://host:3000/session)"
|
|
190
|
+
option :scope, type: :string, default: "all", enum: %w[all recorded], desc: "all sessions, or only those bidi2pdf recorded"
|
|
191
|
+
option :json, type: :boolean, default: false, desc: "Emit a JSON array instead of a table"
|
|
192
|
+
|
|
193
|
+
def sessions
|
|
194
|
+
return print_sessions(chrome_sweeper.sessions) unless options[:json]
|
|
195
|
+
|
|
196
|
+
reserve_stdout_for_machine_output { puts JSON.generate(chrome_sweeper.sessions.map { |info| session_hash(info) }) }
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
desc "sweep", "Close leaked Chrome sessions on a remote chromedriver (too old, unresponsive, or over a limit)"
|
|
200
|
+
long_desc <<~USAGE, wrap: false
|
|
201
|
+
Closes sessions older than --older-than seconds, sessions that failed --unresponsive-checks
|
|
202
|
+
checks in a row (taken --check-interval seconds apart, so a sweep takes that long), and while
|
|
203
|
+
more than --max-sessions exist, the oldest ones. A session younger than --min-age seconds is
|
|
204
|
+
never closed, and neither is one a running bidi2pdf process holds (its lease is renewed). --scope all looks at
|
|
205
|
+
every session on that chromedriver - use it only for a chromedriver your application owns;
|
|
206
|
+
--scope recorded (default) only at sessions bidi2pdf recorded on this machine.
|
|
207
|
+
Exits 1 when a close failed or the limit is still exceeded.
|
|
208
|
+
USAGE
|
|
209
|
+
option :remote_browser_url, type: :string, required: true, desc: "The chromedriver's new-session URL (http://host:3000/session)"
|
|
210
|
+
option :scope, type: :string, default: "recorded", enum: %w[recorded all], desc: "Which sessions may be closed"
|
|
211
|
+
option :older_than, type: :numeric, desc: "Close sessions older than this many seconds (default 600)"
|
|
212
|
+
option :max_sessions, type: :numeric, desc: "Close the oldest sessions while more than this many exist"
|
|
213
|
+
option :min_age, type: :numeric, desc: "Never close a session younger than this many seconds (default 60)"
|
|
214
|
+
option :unresponsive_checks, type: :numeric, desc: "Close a session after this many failed checks in a row (default 2)"
|
|
215
|
+
option :check_interval, type: :numeric, default: 10, desc: "Seconds between those checks; 0 sweeps at once, without the unresponsive rule"
|
|
216
|
+
option :pressure, type: :boolean, default: false, desc: "Last resort: close every session no live process holds that is past --min-age"
|
|
217
|
+
option :dry_run, type: :boolean, default: false, desc: "Report what would be closed, close nothing"
|
|
218
|
+
option :json, type: :boolean, default: false, desc: "Emit the sweep result as JSON"
|
|
219
|
+
|
|
220
|
+
def sweep
|
|
221
|
+
result = if options[:json]
|
|
222
|
+
reserve_stdout_for_machine_output { run_sweep.tap { |swept| puts JSON.generate(sweep_hash(swept)) } }
|
|
223
|
+
else
|
|
224
|
+
run_sweep.tap { |swept| print_sweep(swept) }
|
|
225
|
+
end
|
|
226
|
+
exit(1) if result.errors.any? || result.limit_exceeded
|
|
227
|
+
end
|
|
228
|
+
|
|
186
229
|
desc "template", "Generate a config file template"
|
|
187
230
|
option :output, default: "bidi2pdf.yml", desc: "Output configuration filename"
|
|
188
231
|
|
data/lib/bidi2pdf/dsl.rb
CHANGED
|
@@ -47,7 +47,8 @@ module Bidi2pdf
|
|
|
47
47
|
Bidi2pdf::Bidi::Session.new(
|
|
48
48
|
session_url: remote_browser_url,
|
|
49
49
|
headless: true, # remote is always headless
|
|
50
|
-
chrome_args: chrome_args
|
|
50
|
+
chrome_args: chrome_args,
|
|
51
|
+
registry: Bidi2pdf::SessionRegistry.new(remote_browser_url)
|
|
51
52
|
)
|
|
52
53
|
else
|
|
53
54
|
manager = Bidi2pdf::ChromedriverManager.new(port: port, headless: headless, chrome_args: chrome_args)
|
data/lib/bidi2pdf/launcher.rb
CHANGED
|
@@ -109,7 +109,8 @@ module Bidi2pdf
|
|
|
109
109
|
|
|
110
110
|
def session
|
|
111
111
|
if @remote_browser_url
|
|
112
|
-
@custom_session = Bidi::Session.new(session_url: @remote_browser_url, headless: @headless, chrome_args: @chrome_args
|
|
112
|
+
@custom_session = Bidi::Session.new(session_url: @remote_browser_url, headless: @headless, chrome_args: @chrome_args,
|
|
113
|
+
registry: Bidi2pdf::SessionRegistry.new(@remote_browser_url))
|
|
113
114
|
else
|
|
114
115
|
@manager = ChromedriverManager.new(port: @port, headless: @headless, chrome_args: @chrome_args)
|
|
115
116
|
@manager.start
|