bidi2pdf 0.1.17 → 0.1.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +32 -2
  3. data/README.md +111 -0
  4. data/lib/bidi2pdf/bidi/commands/browsing_context_get_tree.rb +16 -0
  5. data/lib/bidi2pdf/bidi/commands/cdp_send_command.rb +25 -0
  6. data/lib/bidi2pdf/bidi/commands.rb +2 -0
  7. data/lib/bidi2pdf/bidi/session.rb +53 -24
  8. data/lib/bidi2pdf/chrome_sweeper/inspector.rb +117 -0
  9. data/lib/bidi2pdf/chrome_sweeper/settings.rb +80 -0
  10. data/lib/bidi2pdf/chrome_sweeper.rb +398 -0
  11. data/lib/bidi2pdf/chromedriver_api.rb +72 -0
  12. data/lib/bidi2pdf/cli/session_commands.rb +82 -0
  13. data/lib/bidi2pdf/cli.rb +43 -0
  14. data/lib/bidi2pdf/dsl.rb +2 -1
  15. data/lib/bidi2pdf/launcher.rb +2 -1
  16. data/lib/bidi2pdf/session_registry/heartbeat.rb +86 -0
  17. data/lib/bidi2pdf/session_registry.rb +77 -13
  18. data/lib/bidi2pdf/session_sweeper.rb +5 -3
  19. data/lib/bidi2pdf/session_warmer.rb +74 -7
  20. data/lib/bidi2pdf/version.rb +1 -1
  21. data/lib/bidi2pdf.rb +2 -0
  22. data/sig/bidi2pdf/bidi/commands/browsing_context_get_tree.rbs +12 -0
  23. data/sig/bidi2pdf/bidi/commands/cdp_send_command.rbs +23 -0
  24. data/sig/bidi2pdf/bidi/session.rbs +19 -3
  25. data/sig/bidi2pdf/chrome_sweeper/inspector.rbs +61 -0
  26. data/sig/bidi2pdf/chrome_sweeper/settings.rbs +35 -0
  27. data/sig/bidi2pdf/chrome_sweeper.rbs +220 -0
  28. data/sig/bidi2pdf/chromedriver_api.rbs +44 -0
  29. data/sig/bidi2pdf/cli/session_commands.rbs +35 -0
  30. data/sig/bidi2pdf/cli.rbs +6 -0
  31. data/sig/bidi2pdf/session_registry/heartbeat.rbs +50 -0
  32. data/sig/bidi2pdf/session_registry.rbs +46 -9
  33. data/sig/bidi2pdf/session_sweeper.rbs +2 -1
  34. data/sig/bidi2pdf/session_warmer.rbs +42 -2
  35. data/sig/bidi2pdf/version.rbs +1 -1
  36. metadata +17 -1
@@ -0,0 +1,398 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "forwardable"
4
+ require_relative "chromedriver_api"
5
+ require_relative "session_registry"
6
+ require_relative "chrome_sweeper/settings"
7
+ require_relative "chrome_sweeper/inspector"
8
+
9
+ module Bidi2pdf
10
+ # Closes Chrome sessions a shared chromedriver holds that nobody will close any more - left behind
11
+ # by a process that died, or stuck in a tab that never returns - and keeps the number of sessions
12
+ # below a limit. chromedriver keeps every session (a whole Chrome) until someone deletes it.
13
+ #
14
+ # Which sessions it looks at (+scope+):
15
+ # - +:recorded+ (default) - only sessions a SessionRegistry proves were opened through bidi2pdf.
16
+ # - +:all+ - every session on that chromedriver (+GET /sessions+), aged by its first tab. Only for
17
+ # a chromedriver the application owns: on a shared one it closes other clients' old sessions.
18
+ #
19
+ # What it closes, in this order - never a session of +own_sessions+, never one whose lease a live
20
+ # process keeps renewing (SessionRegistry#hold - a render in flight or a warm spare of another
21
+ # worker), and never one younger than +min_age+, not even over the limit:
22
+ # 1. sessions older than +orphan_age+;
23
+ # 2. sessions that failed +unresponsive_checks+ checks in a row (no answer, or a renderer burning
24
+ # CPU at full speed between two sweeps - an endless loop);
25
+ # 3. while more sessions exist than +max_sessions+, the oldest remaining ones. When that is not
26
+ # enough, the sweep reports +limit_exceeded+ and closes nothing more.
27
+ #
28
+ # Run it once (#sweep!, ChromeSweeper.sweep!) - e.g. when the application suspects a leak - or
29
+ # periodically (+interval+, #start/#stop). As a last resort when a render fails for lack of
30
+ # resources, #with_retry sweeps under +pressure+ - every session nobody holds and older than
31
+ # +min_age+ goes - and tries once more. One sweep at a time, across processes too (a lock file
32
+ # next to the registry). Fail-open: a sweep never raises; errors land in the Result. See Settings
33
+ # for every setting and its default.
34
+ #
35
+ # @example A periodic sweeper for an application's own chromedriver
36
+ # sweeper = Bidi2pdf::ChromeSweeper.new("http://remote-chrome:3000/session", scope: :all,
37
+ # max_sessions: :auto, pids_limit: 1024, interval: 60)
38
+ # sweeper.start
39
+ class ChromeSweeper
40
+ extend Forwardable
41
+
42
+ # One closed (or, in a dry run, would-be closed) session. +why+: :orphan, :unresponsive,
43
+ # :over_limit or :pressure.
44
+ Closed = Data.define(:id, :age, :why)
45
+
46
+ # What one sweep saw and did.
47
+ Result = Data.define(:reason, :sessions, :inspected, :closed, :unresponsive, :limit, :limit_exceeded,
48
+ :dry_run, :skipped, :errors, :duration) do
49
+ def closed_count = closed.size
50
+ end
51
+
52
+ # A renderer that used at least this share of the wall-clock time between two sweeps counts as
53
+ # stuck in a loop.
54
+ BUSY_CPU_SHARE = 0.9
55
+
56
+ # Errors #with_retry sweeps and retries on by default: a session chromedriver refused, and a
57
+ # connection or command that died or timed out - what a Chrome out of memory or pids looks like.
58
+ # A command Chrome answered with an error (CmdError) is the page's problem, not a resource one.
59
+ RETRYABLE = ->(error) { error.is_a?(Bidi2pdf::SessionNotStartedError) || (error.is_a?(Bidi2pdf::WebsocketError) && !error.is_a?(Bidi2pdf::CmdError)) }
60
+
61
+ # The bookkeeping of one sweep.
62
+ Sweep = Struct.new(:total, :closed, :unresponsive, :errors, :limit_exceeded, :pressure) do
63
+ def initialize(total = 0, pressure: false) = super(total, [], [], [], false, pressure)
64
+
65
+ def remaining = total - closed.size
66
+ end
67
+
68
+ attr_reader :session_url, :settings, :registry
69
+
70
+ def_delegators :@settings, :scope, :orphan_age, :min_age, :unresponsive_checks, :interval, :dry_run, :limit, :lease_ttl
71
+
72
+ # One-shot sweep with a throw-away sweeper. Give it a +check_interval+ for the unresponsive rule
73
+ # to apply - see #sweep!.
74
+ #
75
+ # @return [Result]
76
+ def self.sweep!(session_url, reason: :manual, check_interval: nil, pressure: false, **)
77
+ new(session_url, **).sweep!(reason: reason, check_interval: check_interval, pressure: pressure)
78
+ end
79
+
80
+ # #with_retry with a throw-away sweeper.
81
+ def self.with_retry(session_url, retry_on: RETRYABLE, **, &)
82
+ new(session_url, **).with_retry(retry_on: retry_on, &)
83
+ end
84
+
85
+ # @param session_url [String] chromedriver's new-session URL (".../session").
86
+ # @param registry [SessionRegistry, nil] defaults to the one for +session_url+ in +registry_dir+.
87
+ # @param own_sessions [#call] returns the ids of the caller's live sessions - never touched.
88
+ # @param api [ChromedriverApi, nil], inspector [Inspector, nil], clock [#call] - for tests.
89
+ # @param settings see Settings.
90
+ # @raise [InvalidConfigError] for a setting out of range.
91
+ def initialize(session_url, registry: nil, registry_dir: nil, own_sessions: -> { [] },
92
+ api: nil, inspector: nil, clock: -> { Time.now.to_f }, **settings)
93
+ @session_url = session_url.to_s
94
+ @settings = Settings.new(**settings)
95
+ @own_sessions = own_sessions
96
+ @clock = clock
97
+ @registry = registry || Bidi2pdf::SessionRegistry.new(@session_url, dir: registry_dir)
98
+ @api = api || Bidi2pdf::ChromedriverApi.new(@session_url)
99
+ @inspector = inspector || Inspector.new(clock: clock)
100
+ @mutex = Mutex.new
101
+ @wakeup = Thread::Queue.new
102
+ @failures = Hash.new(0)
103
+ @first_seen = {}
104
+ @cpu_samples = {}
105
+ end
106
+
107
+ # Every session in scope, inspected (age, tabs, responsive) - for diagnostics; closes nothing.
108
+ #
109
+ # @return [Array<Inspector::SessionInfo>]
110
+ def sessions
111
+ recorded = @registry.recorded
112
+ live = live_sessions
113
+ in_scope(@api.sessions, recorded).map do |entry|
114
+ next live_info(entry.id, recorded[entry.id]) if live.include?(entry.id)
115
+
116
+ @inspector.examine(entry, recorded_at: recorded[entry.id])
117
+ end
118
+ end
119
+
120
+ # Runs the block; when it fails with a resource error (+retry_on+: a callable or a list of error
121
+ # classes, RETRYABLE by default), sweeps under pressure and runs it once more. The block must
122
+ # be safe to run twice - it gets a new session the second time. Other errors, and a second
123
+ # failure, propagate unchanged.
124
+ def with_retry(retry_on: RETRYABLE)
125
+ yield
126
+ rescue StandardError => e
127
+ raise unless retryable?(retry_on, e)
128
+
129
+ Bidi2pdf.logger.warn "chrome_sweeper: #{e.class}: #{e.message} - sweeping #{@session_url} and trying once more"
130
+ Bidi2pdf.notification_service.instrument("chrome_sweeper.retry.bidi2pdf", { error: e.class.name })
131
+ sweep!(reason: :render_failed, pressure: true)
132
+ yield
133
+ end
134
+
135
+ # Inspects every session in scope and counts its checks, like a sweep, but closes nothing.
136
+ #
137
+ # @return [Array<Inspector::SessionInfo>]
138
+ def observe
139
+ @mutex.synchronize { observe_entries(@api.sessions) }
140
+ end
141
+
142
+ # Sweeps once. Skipped (Result#skipped) when another sweep - in this or another process - is
143
+ # still running.
144
+ #
145
+ # A session counts as hung only after +unresponsive_checks+ failed checks, one per sweep - a
146
+ # one-shot sweep never gets there. With +check_interval+ it first checks
147
+ # +unresponsive_checks - 1+ times, +check_interval+ seconds apart (#observe), so a single call
148
+ # applies every rule. The periodic thread doesn't need that: its sweeps are the checks.
149
+ #
150
+ # @param reason [Symbol] why - reported in the notification (:manual, :periodic, :create_failed…).
151
+ # Under +pressure+ - the last resort when a render failed - every eligible session is closed:
152
+ # nobody holds it, and it is older than +min_age+; +orphan_age+, the unresponsive checks and the
153
+ # limit no longer matter.
154
+ #
155
+ # @param check_interval [Numeric, nil] seconds between those checks; nil sweeps right away.
156
+ # @param pressure [Boolean] close everything nobody holds that is past +min_age+.
157
+ # @return [Result]
158
+ def sweep!(reason: :manual, check_interval: nil, pressure: false)
159
+ check_first(check_interval) if check_interval
160
+ started = monotonic
161
+ return result(reason, started, skipped: true) unless @mutex.try_lock
162
+
163
+ begin
164
+ with_lock_file { |locked| locked ? run(reason, started, pressure) : result(reason, started, skipped: true) }
165
+ ensure
166
+ @mutex.unlock
167
+ end
168
+ end
169
+
170
+ # Starts the periodic sweep thread (needs an +interval+). Idempotent.
171
+ def start
172
+ raise Bidi2pdf::InvalidConfigError, "chrome_sweeper: start needs an interval" unless interval
173
+
174
+ @thread ||= Thread.new { sweep!(reason: :periodic) until @wakeup.pop(timeout: interval) == :stop }
175
+ self
176
+ end
177
+
178
+ # Stops the periodic sweep thread and waits for a sweep in progress to finish.
179
+ def stop
180
+ return unless @thread
181
+
182
+ @wakeup << :stop
183
+ @thread.join
184
+ @thread = nil
185
+ end
186
+
187
+ private
188
+
189
+ def run(reason, started, pressure)
190
+ sweep, infos = sweep_sessions(pressure)
191
+ report(result(reason, started, sweep: sweep, inspected: infos))
192
+ rescue StandardError => e
193
+ Bidi2pdf.logger.warn "chrome_sweeper: sweep of #{@session_url} failed: #{e.message}"
194
+ Bidi2pdf.notification_service.instrument("chrome_sweeper.failed.bidi2pdf", { reason: reason, error: e.class.name })
195
+ result(reason, started, sweep: Sweep.new.tap { |failed| failed.errors << e.message })
196
+ end
197
+
198
+ # A config error raises; a failed check only means the sweep has fewer checks to go on.
199
+ def check_first(check_interval)
200
+ raise Bidi2pdf::InvalidConfigError, "chrome_sweeper: check_interval must be a positive number, got #{check_interval.inspect}" unless check_interval.is_a?(Numeric) && check_interval.positive?
201
+
202
+ (unresponsive_checks.to_i - 1).times { observe_then_wait(check_interval) }
203
+ end
204
+
205
+ def observe_then_wait(check_interval)
206
+ observe
207
+ rescue StandardError => e
208
+ Bidi2pdf.logger.warn "chrome_sweeper: checking #{@session_url} before the sweep failed: #{e.message}"
209
+ ensure
210
+ sleep check_interval
211
+ end
212
+
213
+ def sweep_sessions(pressure)
214
+ entries = @api.sessions
215
+ infos = observe_entries(entries)
216
+ sweep = Sweep.new(entries.size, pressure: pressure)
217
+ decide(infos.select { |info| info.age >= min_age }, sweep)
218
+ [sweep, infos]
219
+ end
220
+
221
+ # In-scope sessions not owned by the caller, not held by a live process and not known to be
222
+ # younger than min_age, with the registry time when there is one. None of the others is even
223
+ # attached to - a render in flight is left alone entirely.
224
+ def candidates(entries)
225
+ recorded = @registry.recorded
226
+ untouchable = Array(@own_sessions.call).map(&:to_s) + live_sessions
227
+ in_scope(entries, recorded).filter_map do |entry|
228
+ recorded_at = recorded[entry.id]
229
+ next if untouchable.include?(entry.id) || (recorded_at && now - recorded_at < min_age)
230
+
231
+ [entry, recorded_at]
232
+ end
233
+ end
234
+
235
+ def live_sessions
236
+ @registry.leased(now: now, ttl: lease_ttl)
237
+ end
238
+
239
+ def live_info(id, recorded_at)
240
+ Inspector::SessionInfo.new(id: id, age: recorded_at && (now - recorded_at), source: :registry, tabs: nil,
241
+ responsive: nil, cpu_times: {}, live: true)
242
+ end
243
+
244
+ def retryable?(retry_on, error)
245
+ retry_on.respond_to?(:call) ? retry_on.call(error) : Array(retry_on).any? { |klass| error.is_a?(klass) }
246
+ end
247
+
248
+ def in_scope(entries, recorded)
249
+ scope == :all ? entries : entries.select { |entry| recorded.key?(entry.id) }
250
+ end
251
+
252
+ def observe_entries(entries)
253
+ forget_gone(entries)
254
+ candidates(entries).map { |entry, recorded_at| track(@inspector.examine(entry, recorded_at: recorded_at)) }
255
+ end
256
+
257
+ # Sessions chromedriver no longer has are dropped from the registry and from this sweeper's own
258
+ # tracking - most sessions end normally, so a long-running sweeper would otherwise keep every id
259
+ # it ever saw.
260
+ def forget_gone(entries)
261
+ ids = entries.map(&:id)
262
+ (@registry.recorded.keys - ids).each { |id| @registry.forget(id) }
263
+ (@first_seen.keys - ids).each { |id| forget_tracking(id) }
264
+ end
265
+
266
+ def forget_tracking(id)
267
+ [@failures, @first_seen, @cpu_samples].each { |tracked| tracked.delete(id) }
268
+ end
269
+
270
+ # Counts failed checks per session across sweeps, and ages a session by when this sweeper first
271
+ # saw it when nothing better is known.
272
+ def track(info)
273
+ count_check(info)
274
+ @first_seen[info.id] ||= now
275
+ info.with(age: [info.age || 0, now - @first_seen[info.id]].max)
276
+ end
277
+
278
+ def count_check(info)
279
+ failed = !info.responsive || busy?(info)
280
+ @failures[info.id] = failed ? @failures[info.id] + 1 : 0
281
+ @cpu_samples[info.id] = [now, info.cpu_times]
282
+ end
283
+
284
+ def busy?(info)
285
+ sampled_at, times = @cpu_samples[info.id]
286
+ return false unless sampled_at && now > sampled_at
287
+
288
+ info.cpu_times.any? { |pid, cpu| times.key?(pid) && (cpu - times[pid]) / (now - sampled_at) >= BUSY_CPU_SHARE }
289
+ end
290
+
291
+ def decide(eligible, sweep)
292
+ eligible.each do |info|
293
+ why = close_reason(info, sweep) || (:pressure if sweep.pressure)
294
+ close_session(info, why, sweep) if why
295
+ end
296
+
297
+ enforce_limit(eligible, sweep)
298
+ end
299
+
300
+ def close_reason(info, sweep)
301
+ failed = @failures[info.id]
302
+ note_unresponsive(info, failed, sweep) if unresponsive_checks && failed.positive?
303
+ return :orphan if orphan_age && info.age > orphan_age
304
+
305
+ :unresponsive if unresponsive_checks && failed >= unresponsive_checks
306
+ end
307
+
308
+ def note_unresponsive(info, failed, sweep)
309
+ sweep.unresponsive << info.id
310
+ Bidi2pdf.notification_service.instrument("chrome_sweeper.unresponsive.bidi2pdf", { id: info.id, checks: failed })
311
+ end
312
+
313
+ def enforce_limit(eligible, sweep)
314
+ return unless over_limit?(sweep)
315
+
316
+ oldest_open(eligible, sweep).each do |info|
317
+ break unless over_limit?(sweep)
318
+
319
+ close_session(info, :over_limit, sweep)
320
+ end
321
+ limit_exceeded(sweep) if over_limit?(sweep)
322
+ end
323
+
324
+ def over_limit?(sweep) = !limit.nil? && sweep.remaining > limit
325
+
326
+ def oldest_open(eligible, sweep)
327
+ closed_ids = sweep.closed.map(&:id)
328
+ eligible.reject { |info| closed_ids.include?(info.id) }.sort_by { |info| -info.age }
329
+ end
330
+
331
+ def limit_exceeded(sweep)
332
+ sweep.limit_exceeded = true
333
+ Bidi2pdf.logger.warn "chrome_sweeper: #{sweep.remaining} sessions on #{@session_url}, limit #{limit} - none left old enough to close"
334
+ Bidi2pdf.notification_service.instrument("chrome_sweeper.limit_exceeded.bidi2pdf", { sessions: sweep.remaining, limit: limit })
335
+ end
336
+
337
+ # Adds the session to the sweep's closed list unless chromedriver refused to close it.
338
+ def close_session(info, why, sweep)
339
+ return sweep.errors << "closing #{info.id} failed" unless dry_run || gone_after_delete?(info.id)
340
+
341
+ sweep.closed << announce(Closed.new(id: info.id, age: info.age.round, why: why))
342
+ end
343
+
344
+ def announce(closed)
345
+ Bidi2pdf.notification_service.instrument("chrome_sweeper.closed.bidi2pdf", closed.to_h.merge(dry_run: dry_run))
346
+ closed
347
+ end
348
+
349
+ def gone_after_delete?(id)
350
+ return false if @api.delete_session(id) == :failed
351
+
352
+ @registry.forget(id)
353
+ forget_tracking(id)
354
+ true
355
+ end
356
+
357
+ def report(result)
358
+ verb = dry_run ? "would close" : "closed"
359
+ Bidi2pdf.logger.info "chrome_sweeper: #{verb} #{result.closed_count} of #{result.sessions} session(s) on #{@session_url}" if result.closed.any?
360
+ Bidi2pdf.notification_service.instrument("chrome_sweeper.sweep.bidi2pdf", sweep_payload(result))
361
+ result
362
+ end
363
+
364
+ def sweep_payload(result)
365
+ result.to_h.slice(:reason, :sessions, :limit, :limit_exceeded, :dry_run, :duration)
366
+ .merge(inspected: result.inspected.size, closed: result.closed_count)
367
+ end
368
+
369
+ def result(reason, started, sweep: Sweep.new, inspected: [], skipped: false)
370
+ Result.new(reason: reason, sessions: sweep.total, inspected: inspected, closed: sweep.closed,
371
+ unresponsive: sweep.unresponsive, limit: limit, limit_exceeded: sweep.limit_exceeded,
372
+ dry_run: dry_run, skipped: skipped, errors: sweep.errors, duration: monotonic - started)
373
+ end
374
+
375
+ # A lock file next to the registry, so processes sharing it never sweep at the same time. When
376
+ # the file cannot be created at all the sweep still runs (fail-open, like the registry itself).
377
+ def with_lock_file
378
+ file = open_lock_file
379
+ return yield(true) unless file
380
+
381
+ begin
382
+ yield file.flock(File::LOCK_EX | File::LOCK_NB)
383
+ ensure
384
+ file.close
385
+ end
386
+ end
387
+
388
+ def open_lock_file
389
+ File.open("#{@registry.path}.lock", File::RDWR | File::CREAT, 0o600)
390
+ rescue SystemCallError
391
+ nil
392
+ end
393
+
394
+ def now = @clock.call
395
+
396
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
397
+ end
398
+ end
@@ -0,0 +1,72 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "net/http"
5
+ require "uri"
6
+
7
+ module Bidi2pdf
8
+ # The two chromedriver HTTP endpoints session cleanup needs, next to the BiDi protocol: the
9
+ # non-standard +GET /sessions+ (every session on that chromedriver - WebDriver itself has no way to
10
+ # list them, BiDi neither) and +DELETE /session/{id}+ (ends a session and its Chrome even when that
11
+ # Chrome no longer answers BiDi).
12
+ #
13
+ # Built from the new-session URL every other part of bidi2pdf is configured with
14
+ # ("http://host:3000/session").
15
+ class ChromedriverApi
16
+ # One entry of +GET /sessions+: chromedriver gives no creation time, no owner, and no hint
17
+ # whether a client is still connected (checked against chromedriver 153/154).
18
+ Entry = Data.define(:id, :websocket_url, :process_id)
19
+
20
+ DEFAULT_TIMEOUT = 30
21
+
22
+ attr_reader :session_url
23
+
24
+ # @param session_url [String] chromedriver's new-session URL (".../session").
25
+ # @param http [#call, nil] (method, url) -> [status, body]; injectable for tests.
26
+ def initialize(session_url, http: nil, timeout: DEFAULT_TIMEOUT)
27
+ @session_url = session_url.to_s.chomp("/")
28
+ @timeout = timeout
29
+ @http = http || method(:net_http)
30
+ end
31
+
32
+ # @return [Array<Entry>] every session chromedriver holds right now.
33
+ def sessions
34
+ status, body = @http.call(:get, sessions_url)
35
+ raise Bidi2pdf::Error, "chromedriver answered #{status} to GET /sessions" unless status == 200
36
+
37
+ Array(JSON.parse(body)["value"]).map do |session|
38
+ capabilities = session["capabilities"] || {}
39
+ Entry.new(id: session["id"], websocket_url: capabilities["webSocketUrl"], process_id: capabilities["goog:processID"])
40
+ end
41
+ end
42
+
43
+ # Ends a session. chromedriver answers 404 ("invalid session id") for one it no longer has.
44
+ #
45
+ # @return [Symbol] :closed, :gone or :failed
46
+ def delete_session(id)
47
+ status, = @http.call(:delete, "#{session_url}/#{id}")
48
+ case status
49
+ when 200 then :closed
50
+ when 404 then :gone
51
+ else :failed
52
+ end
53
+ end
54
+
55
+ private
56
+
57
+ # "http://host:3000/session" -> "http://host:3000/sessions"; a chromedriver started with a
58
+ # --url-base keeps its prefix.
59
+ def sessions_url
60
+ "#{session_url.delete_suffix("/session")}/sessions"
61
+ end
62
+
63
+ def net_http(method, url)
64
+ uri = URI(url)
65
+ request = method == :delete ? Net::HTTP::Delete.new(uri) : Net::HTTP::Get.new(uri)
66
+ Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: 5, read_timeout: @timeout) do |http|
67
+ response = http.request(request)
68
+ [response.code.to_i, response.body]
69
+ end
70
+ end
71
+ end
72
+ end
@@ -0,0 +1,82 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bidi2pdf
4
+ class CLI < Thor
5
+ # Output of `bidi2pdf sessions` and `bidi2pdf sweep` (ChromeSweeper). Never prints a tab URL or
6
+ # any page content - only ids, ages, tab counts and what was done.
7
+ module SessionCommands
8
+ # CLI flag => ChromeSweeper setting, for the ones that are only passed when given.
9
+ SWEEP_FLAGS = { older_than: :orphan_age, max_sessions: :max_sessions, min_age: :min_age,
10
+ unresponsive_checks: :unresponsive_checks }.freeze
11
+ # Settings that are counts - Thor hands every numeric over as a Float.
12
+ COUNT_SETTINGS = %i[max_sessions unresponsive_checks].freeze
13
+
14
+ private
15
+
16
+ def chrome_sweeper
17
+ url = options[:remote_browser_url]
18
+ raise Thor::Error, "--remote-browser-url is required" if url.to_s.empty?
19
+
20
+ Bidi2pdf::ChromeSweeper.new(url, scope: options[:scope].to_sym, dry_run: options[:dry_run], **sweep_flags)
21
+ rescue Bidi2pdf::InvalidConfigError => e
22
+ raise Thor::Error, e.message
23
+ end
24
+
25
+ def sweep_flags
26
+ SWEEP_FLAGS.filter_map do |flag, setting|
27
+ value = options[flag]
28
+ [setting, COUNT_SETTINGS.include?(setting) ? value.to_i : value] unless value.nil?
29
+ end.to_h
30
+ end
31
+
32
+ # --check-interval 0 means no checks before the sweep.
33
+ def run_sweep
34
+ interval = options[:check_interval].to_f
35
+ chrome_sweeper.sweep!(reason: :cli, check_interval: interval.zero? ? nil : interval, pressure: options[:pressure])
36
+ rescue Bidi2pdf::InvalidConfigError => e
37
+ raise Thor::Error, e.message
38
+ end
39
+
40
+ def session_hash(info)
41
+ { id: info.id, age: info.age&.round, source: info.source, tabs: info.tabs, responsive: info.responsive, live: info.live }
42
+ end
43
+
44
+ def print_sessions(infos)
45
+ return puts "no sessions" if infos.empty?
46
+
47
+ infos.each do |info|
48
+ age = info.age ? "#{info.age.round}s" : "?"
49
+ puts format("%-34<id>s %8<age>s %-8<source>s %9<tabs>s %<state>s",
50
+ id: info.id, age: age, source: info.source, tabs: info.tabs ? "#{info.tabs} tab(s)" : "", state: session_state(info))
51
+ end
52
+ end
53
+
54
+ # A live session is not attached to, so there is nothing to say about its tabs.
55
+ def session_state(info)
56
+ return "live (a running process holds it)" if info.live
57
+
58
+ info.responsive ? "responsive" : "unresponsive"
59
+ end
60
+
61
+ def sweep_hash(result)
62
+ { reason: result.reason, sessions: result.sessions, dry_run: result.dry_run, skipped: result.skipped,
63
+ closed: result.closed.map(&:to_h), unresponsive: result.unresponsive, limit: result.limit,
64
+ limit_exceeded: result.limit_exceeded, errors: result.errors }
65
+ end
66
+
67
+ def print_sweep(result)
68
+ return puts "skipped: another sweep is running" if result.skipped
69
+
70
+ verb = result.dry_run ? "would close" : "closed"
71
+ result.closed.each { |closed| puts "#{verb} #{closed.id} (#{closed.why}, #{closed.age}s old)" }
72
+ puts "#{verb} #{result.closed_count} of #{result.sessions} session(s)"
73
+ print_sweep_problems(result)
74
+ end
75
+
76
+ def print_sweep_problems(result)
77
+ puts "limit #{result.limit} still exceeded - no session old enough to close" if result.limit_exceeded
78
+ result.errors.each { |error| warn "error: #{error}" }
79
+ end
80
+ end
81
+ end
82
+ end
data/lib/bidi2pdf/cli.rb CHANGED
@@ -5,11 +5,13 @@ require "yaml"
5
5
  require "tempfile"
6
6
 
7
7
  require_relative "cli/json_output"
8
+ require_relative "cli/session_commands"
8
9
 
9
10
  module Bidi2pdf
10
11
  # rubocop:disable Metrics/AbcSize
11
12
  class CLI < Thor
12
13
  include JsonOutput
14
+ include SessionCommands
13
15
 
14
16
  class_option :config, type: :string, desc: "Load configuration from YAML file"
15
17
 
@@ -183,6 +185,47 @@ module Bidi2pdf
183
185
  end
184
186
  map "run" => :run_recipe
185
187
 
188
+ desc "sessions", "List the Chrome sessions a remote chromedriver holds: id, age, tabs, responsive (no URLs)"
189
+ option :remote_browser_url, type: :string, required: true, desc: "The chromedriver's new-session URL (http://host:3000/session)"
190
+ option :scope, type: :string, default: "all", enum: %w[all recorded], desc: "all sessions, or only those bidi2pdf recorded"
191
+ option :json, type: :boolean, default: false, desc: "Emit a JSON array instead of a table"
192
+
193
+ def sessions
194
+ return print_sessions(chrome_sweeper.sessions) unless options[:json]
195
+
196
+ reserve_stdout_for_machine_output { puts JSON.generate(chrome_sweeper.sessions.map { |info| session_hash(info) }) }
197
+ end
198
+
199
+ desc "sweep", "Close leaked Chrome sessions on a remote chromedriver (too old, unresponsive, or over a limit)"
200
+ long_desc <<~USAGE, wrap: false
201
+ Closes sessions older than --older-than seconds, sessions that failed --unresponsive-checks
202
+ checks in a row (taken --check-interval seconds apart, so a sweep takes that long), and while
203
+ more than --max-sessions exist, the oldest ones. A session younger than --min-age seconds is
204
+ never closed, and neither is one a running bidi2pdf process holds (its lease is renewed). --scope all looks at
205
+ every session on that chromedriver - use it only for a chromedriver your application owns;
206
+ --scope recorded (default) only at sessions bidi2pdf recorded on this machine.
207
+ Exits 1 when a close failed or the limit is still exceeded.
208
+ USAGE
209
+ option :remote_browser_url, type: :string, required: true, desc: "The chromedriver's new-session URL (http://host:3000/session)"
210
+ option :scope, type: :string, default: "recorded", enum: %w[recorded all], desc: "Which sessions may be closed"
211
+ option :older_than, type: :numeric, desc: "Close sessions older than this many seconds (default 600)"
212
+ option :max_sessions, type: :numeric, desc: "Close the oldest sessions while more than this many exist"
213
+ option :min_age, type: :numeric, desc: "Never close a session younger than this many seconds (default 60)"
214
+ option :unresponsive_checks, type: :numeric, desc: "Close a session after this many failed checks in a row (default 2)"
215
+ option :check_interval, type: :numeric, default: 10, desc: "Seconds between those checks; 0 sweeps at once, without the unresponsive rule"
216
+ option :pressure, type: :boolean, default: false, desc: "Last resort: close every session no live process holds that is past --min-age"
217
+ option :dry_run, type: :boolean, default: false, desc: "Report what would be closed, close nothing"
218
+ option :json, type: :boolean, default: false, desc: "Emit the sweep result as JSON"
219
+
220
+ def sweep
221
+ result = if options[:json]
222
+ reserve_stdout_for_machine_output { run_sweep.tap { |swept| puts JSON.generate(sweep_hash(swept)) } }
223
+ else
224
+ run_sweep.tap { |swept| print_sweep(swept) }
225
+ end
226
+ exit(1) if result.errors.any? || result.limit_exceeded
227
+ end
228
+
186
229
  desc "template", "Generate a config file template"
187
230
  option :output, default: "bidi2pdf.yml", desc: "Output configuration filename"
188
231
 
data/lib/bidi2pdf/dsl.rb CHANGED
@@ -47,7 +47,8 @@ module Bidi2pdf
47
47
  Bidi2pdf::Bidi::Session.new(
48
48
  session_url: remote_browser_url,
49
49
  headless: true, # remote is always headless
50
- chrome_args: chrome_args
50
+ chrome_args: chrome_args,
51
+ registry: Bidi2pdf::SessionRegistry.new(remote_browser_url)
51
52
  )
52
53
  else
53
54
  manager = Bidi2pdf::ChromedriverManager.new(port: port, headless: headless, chrome_args: chrome_args)
@@ -109,7 +109,8 @@ module Bidi2pdf
109
109
 
110
110
  def session
111
111
  if @remote_browser_url
112
- @custom_session = Bidi::Session.new(session_url: @remote_browser_url, headless: @headless, chrome_args: @chrome_args)
112
+ @custom_session = Bidi::Session.new(session_url: @remote_browser_url, headless: @headless, chrome_args: @chrome_args,
113
+ registry: Bidi2pdf::SessionRegistry.new(@remote_browser_url))
113
114
  else
114
115
  @manager = ChromedriverManager.new(port: @port, headless: @headless, chrome_args: @chrome_args)
115
116
  @manager.start