bidi2pdf 0.1.16 → 0.1.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -2
- data/README.md +145 -0
- data/docker/Dockerfile +7 -0
- data/docker/Dockerfile.chromedriver +7 -0
- data/docker/Dockerfile.slim +7 -0
- data/lib/bidi2pdf/bidi/buffered_web_socket_client.rb +9 -0
- data/lib/bidi2pdf/bidi/commands/browsing_context_get_tree.rb +16 -0
- data/lib/bidi2pdf/bidi/commands/cdp_send_command.rb +25 -0
- data/lib/bidi2pdf/bidi/commands.rb +2 -0
- data/lib/bidi2pdf/bidi/session.rb +57 -24
- data/lib/bidi2pdf/chrome_sweeper/inspector.rb +117 -0
- data/lib/bidi2pdf/chrome_sweeper/settings.rb +79 -0
- data/lib/bidi2pdf/chrome_sweeper.rb +398 -0
- data/lib/bidi2pdf/chromedriver_api.rb +72 -0
- data/lib/bidi2pdf/cli/session_commands.rb +82 -0
- data/lib/bidi2pdf/cli.rb +43 -0
- data/lib/bidi2pdf/dsl.rb +2 -1
- data/lib/bidi2pdf/launcher.rb +2 -1
- data/lib/bidi2pdf/session_registry/heartbeat.rb +83 -0
- data/lib/bidi2pdf/session_registry.rb +152 -0
- data/lib/bidi2pdf/session_sweeper.rb +66 -0
- data/lib/bidi2pdf/session_warmer.rb +112 -1
- data/lib/bidi2pdf/version.rb +1 -1
- data/lib/bidi2pdf.rb +4 -0
- data/sig/bidi2pdf/bidi/buffered_web_socket_client.rbs +7 -0
- data/sig/bidi2pdf/bidi/commands/browsing_context_get_tree.rbs +12 -0
- data/sig/bidi2pdf/bidi/commands/cdp_send_command.rbs +23 -0
- data/sig/bidi2pdf/bidi/session.rbs +25 -3
- data/sig/bidi2pdf/chrome_sweeper/inspector.rbs +61 -0
- data/sig/bidi2pdf/chrome_sweeper/settings.rbs +34 -0
- data/sig/bidi2pdf/chrome_sweeper.rbs +220 -0
- data/sig/bidi2pdf/chromedriver_api.rbs +44 -0
- data/sig/bidi2pdf/cli/session_commands.rbs +35 -0
- data/sig/bidi2pdf/cli.rbs +6 -0
- data/sig/bidi2pdf/session_registry/heartbeat.rbs +47 -0
- data/sig/bidi2pdf/session_registry.rbs +75 -0
- data/sig/bidi2pdf/session_sweeper.rbs +37 -0
- data/sig/bidi2pdf/session_warmer.rbs +66 -0
- data/sig/bidi2pdf/version.rbs +1 -1
- metadata +29 -3
|
@@ -27,12 +27,16 @@ module Bidi2pdf
|
|
|
27
27
|
|
|
28
28
|
@chrome_args: untyped
|
|
29
29
|
|
|
30
|
+
@registry: untyped
|
|
31
|
+
|
|
30
32
|
@client: untyped
|
|
31
33
|
|
|
32
34
|
@browser: untyped
|
|
33
35
|
|
|
34
36
|
@websocket_url: untyped
|
|
35
37
|
|
|
38
|
+
@session_id: untyped
|
|
39
|
+
|
|
36
40
|
# Events to subscribe to during the session.
|
|
37
41
|
SUBSCRIBE_EVENTS: ::Array["script"]
|
|
38
42
|
|
|
@@ -48,12 +52,19 @@ module Bidi2pdf
|
|
|
48
52
|
# @return [Array<String>] The Chrome arguments for the session.
|
|
49
53
|
attr_reader chrome_args: untyped
|
|
50
54
|
|
|
55
|
+
# @return [String, nil] chromedriver's id for this session, once it was created - what
|
|
56
|
+
# SessionWarmer records so a later process can close it if this one dies uncleanly.
|
|
57
|
+
attr_reader session_id: untyped
|
|
58
|
+
|
|
51
59
|
# Initializes a new session.
|
|
52
60
|
#
|
|
53
61
|
# @param [String] session_url The URL for the session.
|
|
54
62
|
# @param [Boolean] headless Whether to run the browser in headless mode. Defaults to true.
|
|
55
63
|
# @param [Array<String>] chrome_args Additional Chrome arguments. Defaults to predefined arguments.
|
|
56
|
-
|
|
64
|
+
# @param [Bidi2pdf::SessionRegistry, nil] registry Records the session on a shared (remote)
|
|
65
|
+
# chromedriver while it is open, so a sweep can close it if this process dies without
|
|
66
|
+
# closing it (ChromeSweeper, SessionSweeper). nil records nothing.
|
|
67
|
+
def initialize: (session_url: untyped, ?headless: bool, ?chrome_args: untyped, ?registry: untyped?) -> void
|
|
57
68
|
|
|
58
69
|
# Starts the session and initializes the client.
|
|
59
70
|
#
|
|
@@ -70,8 +81,8 @@ module Bidi2pdf
|
|
|
70
81
|
# @return [Bidi2pdf::Bidi::Browser] The browser instance.
|
|
71
82
|
def browser: () -> untyped
|
|
72
83
|
|
|
73
|
-
# Closes the session and cleans up resources.
|
|
74
|
-
#
|
|
84
|
+
# Closes the session and cleans up resources: BiDi +session.end+ first, and when Chrome does not
|
|
85
|
+
# answer that (a tab stuck in a loop), chromedriver's +DELETE /session/{id}+.
|
|
75
86
|
def close: () -> untyped
|
|
76
87
|
|
|
77
88
|
# Retrieves user contexts for the session.
|
|
@@ -158,6 +169,17 @@ module Bidi2pdf
|
|
|
158
169
|
# @raise [SessionNotStartedError] If the session could not be started.
|
|
159
170
|
def handle_error: (untyped value) -> (nil | untyped)
|
|
160
171
|
|
|
172
|
+
def end_via_bidi: () -> untyped
|
|
173
|
+
|
|
174
|
+
def record_session: (untyped id) -> untyped
|
|
175
|
+
|
|
176
|
+
# The BiDi session.end got no answer - typically a Chrome that hangs or crashed. chromedriver
|
|
177
|
+
# still ends the session (and kills its Chrome) on an HTTP DELETE, so a hung Chrome does not
|
|
178
|
+
# outlive this process. Only for a session this object created over HTTP (it knows the id).
|
|
179
|
+
#
|
|
180
|
+
# @return [Boolean] true when the session is gone.
|
|
181
|
+
def delete_via_chromedriver: () -> untyped
|
|
182
|
+
|
|
161
183
|
# Cleans up resources associated with the session.
|
|
162
184
|
def cleanup: () -> untyped
|
|
163
185
|
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
class ChromeSweeper
|
|
3
|
+
# Looks at one session on a chromedriver from the outside: attaches a second BiDi connection to
|
|
4
|
+
# its +webSocketUrl+ (chromedriver allows that for any session), asks for the tab tree, the first
|
|
5
|
+
# tab's +performance.timeOrigin+ (Chrome keeps no session start time anywhere - the first tab is
|
|
6
|
+
# created with the session, so its time origin is the session's age to within a few hundred ms)
|
|
7
|
+
# and the renderer CPU times (CDP +SystemInfo.getProcessInfo+ through +goog:cdp.sendCommand+).
|
|
8
|
+
#
|
|
9
|
+
# Read-only, and never reads page content: tab URLs are counted, not returned or logged (a
|
|
10
|
+
# +data:+ URL carries the whole rendered document). Closes its own connection every time.
|
|
11
|
+
class Inspector
|
|
12
|
+
@timeout: untyped
|
|
13
|
+
|
|
14
|
+
@clock: untyped
|
|
15
|
+
|
|
16
|
+
@client_factory: untyped
|
|
17
|
+
|
|
18
|
+
# @!attribute age [Float, nil] seconds since the session started, nil when nothing tells.
|
|
19
|
+
# @!attribute source [Symbol] :registry, :tab or :unknown - where +age+ comes from.
|
|
20
|
+
# @!attribute responsive [Boolean] whether the session answered the checks.
|
|
21
|
+
# @!attribute cpu_times [Hash{Integer => Float}] renderer pid => CPU seconds.
|
|
22
|
+
# @!attribute live [Boolean] a live process holds its lease (never inspected, never closed).
|
|
23
|
+
SessionInfo: untyped
|
|
24
|
+
|
|
25
|
+
DEFAULT_TIMEOUT: 5
|
|
26
|
+
|
|
27
|
+
# @param clock [#call] epoch seconds as a Float.
|
|
28
|
+
# @param client_factory [#call] ws_url -> a started, open Bidi::Client; injectable for tests.
|
|
29
|
+
def initialize: (?timeout: untyped, ?clock: untyped, ?client_factory: untyped?) -> void
|
|
30
|
+
|
|
31
|
+
# @param entry [ChromedriverApi::Entry]
|
|
32
|
+
# @param recorded_at [Numeric, nil] epoch seconds the registry has for it; wins over the tab.
|
|
33
|
+
# @return [SessionInfo]
|
|
34
|
+
def examine: (untyped entry, ?recorded_at: untyped?) -> untyped
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def probe: (untyped client, untyped entry, untyped recorded_at) -> untyped
|
|
39
|
+
|
|
40
|
+
def tree: (untyped client) -> untyped
|
|
41
|
+
|
|
42
|
+
# A tab stuck in an endless loop never evaluates anything - nil then.
|
|
43
|
+
def time_origin: (untyped client, untyped context) -> untyped
|
|
44
|
+
|
|
45
|
+
# On a freshly attached connection the first command to a session - and, seen once in the
|
|
46
|
+
# acceptance run, to a tab - can time out although the session is healthy (chromedriver
|
|
47
|
+
# 153/154); the second one answers. Only a second timeout counts.
|
|
48
|
+
def once_more_on_timeout: () { () -> untyped } -> untyped
|
|
49
|
+
|
|
50
|
+
def cpu_times: (untyped client) -> untyped
|
|
51
|
+
|
|
52
|
+
def age_of: (untyped recorded_at, untyped origin) -> (::Array[untyped | :registry] | ::Array[untyped | :tab] | ::Array[nil | :unknown])
|
|
53
|
+
|
|
54
|
+
def unreachable: (untyped entry, untyped recorded_at) -> untyped
|
|
55
|
+
|
|
56
|
+
def command: (untyped client, untyped cmd) -> untyped
|
|
57
|
+
|
|
58
|
+
def connect: (untyped ws_url) -> untyped
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
class ChromeSweeper
|
|
3
|
+
SCOPES: ::Array[:recorded | :all]
|
|
4
|
+
|
|
5
|
+
DEFAULT_ORPHAN_AGE: 600
|
|
6
|
+
|
|
7
|
+
DEFAULT_MIN_AGE: 60
|
|
8
|
+
|
|
9
|
+
DEFAULT_UNRESPONSIVE_CHECKS: 2
|
|
10
|
+
|
|
11
|
+
DEFAULT_PIDS_BUDGET: ::Float
|
|
12
|
+
|
|
13
|
+
DEFAULT_THREADS_PER_SESSION: 110
|
|
14
|
+
|
|
15
|
+
DEFAULT_LEASE_TTL: untyped
|
|
16
|
+
|
|
17
|
+
# What a ChromeSweeper closes and when - validated on construction; an unknown setting raises
|
|
18
|
+
# ArgumentError.
|
|
19
|
+
#
|
|
20
|
+
# @!attribute scope [Symbol] :recorded (sessions a SessionRegistry knows) or :all.
|
|
21
|
+
# @!attribute orphan_age [Numeric, nil] close sessions older than this; nil turns the rule off.
|
|
22
|
+
# @!attribute min_age [Numeric] never close a session younger than this.
|
|
23
|
+
# @!attribute unresponsive_checks [Integer, nil] close after this many failed checks in a row.
|
|
24
|
+
# @!attribute max_sessions [Integer, Symbol, nil] the limit; +:auto+ derives it from pids_limit.
|
|
25
|
+
# @!attribute pids_limit [Integer, nil] the chromedriver container's pids limit, for +:auto+.
|
|
26
|
+
# @!attribute pids_budget [Float] share of +pids_limit+ Chrome sessions may use.
|
|
27
|
+
# @!attribute threads_per_session [Integer] threads (Docker counts them as pids) of one session.
|
|
28
|
+
# @!attribute interval [Numeric, nil] seconds between sweeps of ChromeSweeper#start's thread.
|
|
29
|
+
# @!attribute dry_run [Boolean] report what would be closed, close nothing.
|
|
30
|
+
# @!attribute lease_ttl [Numeric] a recorded session renewed within this many seconds belongs to
|
|
31
|
+
# a live process and is never closed (SessionRegistry#hold).
|
|
32
|
+
Settings: untyped
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
# Closes Chrome sessions a shared chromedriver holds that nobody will close any more - left behind
|
|
3
|
+
# by a process that died, or stuck in a tab that never returns - and keeps the number of sessions
|
|
4
|
+
# below a limit. chromedriver keeps every session (a whole Chrome) until someone deletes it.
|
|
5
|
+
#
|
|
6
|
+
# Which sessions it looks at (+scope+):
|
|
7
|
+
# - +:recorded+ (default) - only sessions a SessionRegistry proves were opened through bidi2pdf.
|
|
8
|
+
# - +:all+ - every session on that chromedriver (+GET /sessions+), aged by its first tab. Only for
|
|
9
|
+
# a chromedriver the application owns: on a shared one it closes other clients' old sessions.
|
|
10
|
+
#
|
|
11
|
+
# What it closes, in this order - never a session of +own_sessions+, never one whose lease a live
|
|
12
|
+
# process keeps renewing (SessionRegistry#hold - a render in flight or a warm spare of another
|
|
13
|
+
# worker), and never one younger than +min_age+, not even over the limit:
|
|
14
|
+
# 1. sessions older than +orphan_age+;
|
|
15
|
+
# 2. sessions that failed +unresponsive_checks+ checks in a row (no answer, or a renderer burning
|
|
16
|
+
# CPU at full speed between two sweeps - an endless loop);
|
|
17
|
+
# 3. while more sessions exist than +max_sessions+, the oldest remaining ones. When that is not
|
|
18
|
+
# enough, the sweep reports +limit_exceeded+ and closes nothing more.
|
|
19
|
+
#
|
|
20
|
+
# Run it once (#sweep!, ChromeSweeper.sweep!) - e.g. when the application suspects a leak - or
|
|
21
|
+
# periodically (+interval+, #start/#stop). As a last resort when a render fails for lack of
|
|
22
|
+
# resources, #with_retry sweeps under +pressure+ - every session nobody holds and older than
|
|
23
|
+
# +min_age+ goes - and tries once more. One sweep at a time, across processes too (a lock file
|
|
24
|
+
# next to the registry). Fail-open: a sweep never raises; errors land in the Result. See Settings
|
|
25
|
+
# for every setting and its default.
|
|
26
|
+
#
|
|
27
|
+
# @example A periodic sweeper for an application's own chromedriver
|
|
28
|
+
# sweeper = Bidi2pdf::ChromeSweeper.new("http://remote-chrome:3000/session", scope: :all,
|
|
29
|
+
# max_sessions: :auto, pids_limit: 1024, interval: 60)
|
|
30
|
+
# sweeper.start
|
|
31
|
+
class ChromeSweeper
|
|
32
|
+
@session_url: untyped
|
|
33
|
+
|
|
34
|
+
@settings: untyped
|
|
35
|
+
|
|
36
|
+
@own_sessions: untyped
|
|
37
|
+
|
|
38
|
+
@clock: untyped
|
|
39
|
+
|
|
40
|
+
@registry: untyped
|
|
41
|
+
|
|
42
|
+
@api: untyped
|
|
43
|
+
|
|
44
|
+
@inspector: untyped
|
|
45
|
+
|
|
46
|
+
@mutex: untyped
|
|
47
|
+
|
|
48
|
+
@wakeup: untyped
|
|
49
|
+
|
|
50
|
+
@failures: untyped
|
|
51
|
+
|
|
52
|
+
@first_seen: untyped
|
|
53
|
+
|
|
54
|
+
@cpu_samples: untyped
|
|
55
|
+
|
|
56
|
+
@thread: untyped
|
|
57
|
+
|
|
58
|
+
extend Forwardable
|
|
59
|
+
|
|
60
|
+
# One closed (or, in a dry run, would-be closed) session. +why+: :orphan, :unresponsive,
|
|
61
|
+
# :over_limit or :pressure.
|
|
62
|
+
Closed: untyped
|
|
63
|
+
|
|
64
|
+
# What one sweep saw and did.
|
|
65
|
+
Result: untyped
|
|
66
|
+
|
|
67
|
+
# A renderer that used at least this share of the wall-clock time between two sweeps counts as
|
|
68
|
+
# stuck in a loop.
|
|
69
|
+
BUSY_CPU_SHARE: ::Float
|
|
70
|
+
|
|
71
|
+
# Errors #with_retry sweeps and retries on by default: a session chromedriver refused, and a
|
|
72
|
+
# connection or command that died or timed out - what a Chrome out of memory or pids looks like.
|
|
73
|
+
# A command Chrome answered with an error (CmdError) is the page's problem, not a resource one.
|
|
74
|
+
RETRYABLE: untyped
|
|
75
|
+
|
|
76
|
+
# The bookkeeping of one sweep.
|
|
77
|
+
Sweep: untyped
|
|
78
|
+
|
|
79
|
+
attr_reader session_url: untyped
|
|
80
|
+
|
|
81
|
+
attr_reader settings: untyped
|
|
82
|
+
|
|
83
|
+
attr_reader registry: untyped
|
|
84
|
+
|
|
85
|
+
# One-shot sweep with a throw-away sweeper. Give it a +check_interval+ for the unresponsive rule
|
|
86
|
+
# to apply - see #sweep!.
|
|
87
|
+
#
|
|
88
|
+
# @return [Result]
|
|
89
|
+
def self.sweep!: (untyped session_url, ?reason: ::Symbol, ?check_interval: untyped?, ?pressure: bool, **untyped) -> untyped
|
|
90
|
+
|
|
91
|
+
# #with_retry with a throw-away sweeper.
|
|
92
|
+
def self.with_retry: (untyped session_url, ?retry_on: untyped, **untyped) { (?) -> untyped } -> untyped
|
|
93
|
+
|
|
94
|
+
# @param session_url [String] chromedriver's new-session URL (".../session").
|
|
95
|
+
# @param registry [SessionRegistry, nil] defaults to the one for +session_url+ in +registry_dir+.
|
|
96
|
+
# @param own_sessions [#call] returns the ids of the caller's live sessions - never touched.
|
|
97
|
+
# @param api [ChromedriverApi, nil], inspector [Inspector, nil], clock [#call] - for tests.
|
|
98
|
+
# @param settings see Settings.
|
|
99
|
+
# @raise [InvalidConfigError] for a setting out of range.
|
|
100
|
+
def initialize: (untyped session_url, ?registry: untyped?, ?registry_dir: untyped?, ?own_sessions: untyped, ?api: untyped?, ?inspector: untyped?, ?clock: untyped, **untyped settings) -> void
|
|
101
|
+
|
|
102
|
+
# Every session in scope, inspected (age, tabs, responsive) - for diagnostics; closes nothing.
|
|
103
|
+
#
|
|
104
|
+
# @return [Array<Inspector::SessionInfo>]
|
|
105
|
+
def sessions: () -> untyped
|
|
106
|
+
|
|
107
|
+
# Runs the block; when it fails with a resource error (+retry_on+: a callable or a list of error
|
|
108
|
+
# classes, RETRYABLE by default), sweeps under pressure and runs it once more. The block must
|
|
109
|
+
# be safe to run twice - it gets a new session the second time. Other errors, and a second
|
|
110
|
+
# failure, propagate unchanged.
|
|
111
|
+
def with_retry: (?retry_on: untyped) { () -> untyped } -> untyped
|
|
112
|
+
|
|
113
|
+
# Inspects every session in scope and counts its checks, like a sweep, but closes nothing.
|
|
114
|
+
#
|
|
115
|
+
# @return [Array<Inspector::SessionInfo>]
|
|
116
|
+
def observe: () -> untyped
|
|
117
|
+
|
|
118
|
+
# Sweeps once. Skipped (Result#skipped) when another sweep - in this or another process - is
|
|
119
|
+
# still running.
|
|
120
|
+
#
|
|
121
|
+
# A session counts as hung only after +unresponsive_checks+ failed checks, one per sweep - a
|
|
122
|
+
# one-shot sweep never gets there. With +check_interval+ it first checks
|
|
123
|
+
# +unresponsive_checks - 1+ times, +check_interval+ seconds apart (#observe), so a single call
|
|
124
|
+
# applies every rule. The periodic thread doesn't need that: its sweeps are the checks.
|
|
125
|
+
#
|
|
126
|
+
# @param reason [Symbol] why - reported in the notification (:manual, :periodic, :create_failed…).
|
|
127
|
+
# Under +pressure+ - the last resort when a render failed - every eligible session is closed:
|
|
128
|
+
# nobody holds it, and it is older than +min_age+; +orphan_age+, the unresponsive checks and the
|
|
129
|
+
# limit no longer matter.
|
|
130
|
+
#
|
|
131
|
+
# @param check_interval [Numeric, nil] seconds between those checks; nil sweeps right away.
|
|
132
|
+
# @param pressure [Boolean] close everything nobody holds that is past +min_age+.
|
|
133
|
+
# @return [Result]
|
|
134
|
+
def sweep!: (?reason: ::Symbol, ?check_interval: untyped?, ?pressure: bool) -> untyped
|
|
135
|
+
|
|
136
|
+
# Starts the periodic sweep thread (needs an +interval+). Idempotent.
|
|
137
|
+
def start: () -> self
|
|
138
|
+
|
|
139
|
+
# Stops the periodic sweep thread and waits for a sweep in progress to finish.
|
|
140
|
+
def stop: () -> (nil | untyped)
|
|
141
|
+
|
|
142
|
+
private
|
|
143
|
+
|
|
144
|
+
def run: (untyped reason, untyped started, untyped pressure) -> untyped
|
|
145
|
+
|
|
146
|
+
# A config error raises; a failed check only means the sweep has fewer checks to go on.
|
|
147
|
+
def check_first: (untyped check_interval) -> untyped
|
|
148
|
+
|
|
149
|
+
def observe_then_wait: (untyped check_interval) -> untyped
|
|
150
|
+
|
|
151
|
+
def sweep_sessions: (untyped pressure) -> ::Array[untyped]
|
|
152
|
+
|
|
153
|
+
# In-scope sessions not owned by the caller, not held by a live process and not known to be
|
|
154
|
+
# younger than min_age, with the registry time when there is one. None of the others is even
|
|
155
|
+
# attached to - a render in flight is left alone entirely.
|
|
156
|
+
def candidates: (untyped entries) -> untyped
|
|
157
|
+
|
|
158
|
+
def live_sessions: () -> untyped
|
|
159
|
+
|
|
160
|
+
def live_info: (untyped id, untyped recorded_at) -> untyped
|
|
161
|
+
|
|
162
|
+
def retryable?: (untyped retry_on, untyped error) -> untyped
|
|
163
|
+
|
|
164
|
+
def in_scope: (untyped entries, untyped recorded) -> untyped
|
|
165
|
+
|
|
166
|
+
def observe_entries: (untyped entries) -> untyped
|
|
167
|
+
|
|
168
|
+
# Sessions chromedriver no longer has are dropped from the registry and from this sweeper's own
|
|
169
|
+
# tracking - most sessions end normally, so a long-running sweeper would otherwise keep every id
|
|
170
|
+
# it ever saw.
|
|
171
|
+
def forget_gone: (untyped entries) -> untyped
|
|
172
|
+
|
|
173
|
+
def forget_tracking: (untyped id) -> untyped
|
|
174
|
+
|
|
175
|
+
# Counts failed checks per session across sweeps, and ages a session by when this sweeper first
|
|
176
|
+
# saw it when nothing better is known.
|
|
177
|
+
def track: (untyped info) -> untyped
|
|
178
|
+
|
|
179
|
+
def count_check: (untyped info) -> untyped
|
|
180
|
+
|
|
181
|
+
def busy?: (untyped info) -> (false | untyped)
|
|
182
|
+
|
|
183
|
+
def decide: (untyped eligible, untyped sweep) -> untyped
|
|
184
|
+
|
|
185
|
+
def close_reason: (untyped info, untyped sweep) -> (:orphan | untyped)
|
|
186
|
+
|
|
187
|
+
def note_unresponsive: (untyped info, untyped failed, untyped sweep) -> untyped
|
|
188
|
+
|
|
189
|
+
def enforce_limit: (untyped eligible, untyped sweep) -> (nil | untyped)
|
|
190
|
+
|
|
191
|
+
def over_limit?: (untyped sweep) -> untyped
|
|
192
|
+
|
|
193
|
+
def oldest_open: (untyped eligible, untyped sweep) -> untyped
|
|
194
|
+
|
|
195
|
+
def limit_exceeded: (untyped sweep) -> untyped
|
|
196
|
+
|
|
197
|
+
# Adds the session to the sweep's closed list unless chromedriver refused to close it.
|
|
198
|
+
def close_session: (untyped info, untyped why, untyped sweep) -> untyped
|
|
199
|
+
|
|
200
|
+
def announce: (untyped closed) -> untyped
|
|
201
|
+
|
|
202
|
+
def gone_after_delete?: (untyped id) -> (false | true)
|
|
203
|
+
|
|
204
|
+
def report: (untyped result) -> untyped
|
|
205
|
+
|
|
206
|
+
def sweep_payload: (untyped result) -> untyped
|
|
207
|
+
|
|
208
|
+
def result: (untyped reason, untyped started, ?sweep: untyped, ?inspected: untyped, ?skipped: bool) -> untyped
|
|
209
|
+
|
|
210
|
+
# A lock file next to the registry, so processes sharing it never sweep at the same time. When
|
|
211
|
+
# the file cannot be created at all the sweep still runs (fail-open, like the registry itself).
|
|
212
|
+
def with_lock_file: () { (untyped) -> untyped } -> untyped
|
|
213
|
+
|
|
214
|
+
def open_lock_file: () -> untyped
|
|
215
|
+
|
|
216
|
+
def now: () -> untyped
|
|
217
|
+
|
|
218
|
+
def monotonic: () -> untyped
|
|
219
|
+
end
|
|
220
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
# The two chromedriver HTTP endpoints session cleanup needs, next to the BiDi protocol: the
|
|
3
|
+
# non-standard +GET /sessions+ (every session on that chromedriver - WebDriver itself has no way to
|
|
4
|
+
# list them, BiDi neither) and +DELETE /session/{id}+ (ends a session and its Chrome even when that
|
|
5
|
+
# Chrome no longer answers BiDi).
|
|
6
|
+
#
|
|
7
|
+
# Built from the new-session URL every other part of bidi2pdf is configured with
|
|
8
|
+
# ("http://host:3000/session").
|
|
9
|
+
class ChromedriverApi
|
|
10
|
+
@session_url: untyped
|
|
11
|
+
|
|
12
|
+
@timeout: untyped
|
|
13
|
+
|
|
14
|
+
@http: untyped
|
|
15
|
+
|
|
16
|
+
# One entry of +GET /sessions+: chromedriver gives no creation time, no owner, and no hint
|
|
17
|
+
# whether a client is still connected (checked against chromedriver 153/154).
|
|
18
|
+
Entry: untyped
|
|
19
|
+
|
|
20
|
+
DEFAULT_TIMEOUT: 30
|
|
21
|
+
|
|
22
|
+
attr_reader session_url: untyped
|
|
23
|
+
|
|
24
|
+
# @param session_url [String] chromedriver's new-session URL (".../session").
|
|
25
|
+
# @param http [#call, nil] (method, url) -> [status, body]; injectable for tests.
|
|
26
|
+
def initialize: (untyped session_url, ?http: untyped?, ?timeout: untyped) -> void
|
|
27
|
+
|
|
28
|
+
# @return [Array<Entry>] every session chromedriver holds right now.
|
|
29
|
+
def sessions: () -> untyped
|
|
30
|
+
|
|
31
|
+
# Ends a session. chromedriver answers 404 ("invalid session id") for one it no longer has.
|
|
32
|
+
#
|
|
33
|
+
# @return [Symbol] :closed, :gone or :failed
|
|
34
|
+
def delete_session: (untyped id) -> untyped
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
# "http://host:3000/session" -> "http://host:3000/sessions"; a chromedriver started with a
|
|
39
|
+
# --url-base keeps its prefix.
|
|
40
|
+
def sessions_url: () -> ::String
|
|
41
|
+
|
|
42
|
+
def net_http: (untyped method, untyped url) -> untyped
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
class CLI < Thor
|
|
3
|
+
# Output of `bidi2pdf sessions` and `bidi2pdf sweep` (ChromeSweeper). Never prints a tab URL or
|
|
4
|
+
# any page content - only ids, ages, tab counts and what was done.
|
|
5
|
+
module SessionCommands
|
|
6
|
+
# CLI flag => ChromeSweeper setting, for the ones that are only passed when given.
|
|
7
|
+
SWEEP_FLAGS: { older_than: :orphan_age, max_sessions: :max_sessions, min_age: :min_age, unresponsive_checks: :unresponsive_checks }
|
|
8
|
+
|
|
9
|
+
# Settings that are counts - Thor hands every numeric over as a Float.
|
|
10
|
+
COUNT_SETTINGS: ::Array[:max_sessions | :unresponsive_checks]
|
|
11
|
+
|
|
12
|
+
private
|
|
13
|
+
|
|
14
|
+
def chrome_sweeper: () -> untyped
|
|
15
|
+
|
|
16
|
+
def sweep_flags: () -> untyped
|
|
17
|
+
|
|
18
|
+
# --check-interval 0 means no checks before the sweep.
|
|
19
|
+
def run_sweep: () -> untyped
|
|
20
|
+
|
|
21
|
+
def session_hash: (untyped info) -> { id: untyped, age: untyped, source: untyped, tabs: untyped, responsive: untyped, live: untyped }
|
|
22
|
+
|
|
23
|
+
def print_sessions: (untyped infos) -> untyped
|
|
24
|
+
|
|
25
|
+
# A live session is not attached to, so there is nothing to say about its tabs.
|
|
26
|
+
def session_state: (untyped info) -> ("live (a running process holds it)" | untyped)
|
|
27
|
+
|
|
28
|
+
def sweep_hash: (untyped result) -> { reason: untyped, sessions: untyped, dry_run: untyped, skipped: untyped, closed: untyped, unresponsive: untyped, limit: untyped, limit_exceeded: untyped, errors: untyped }
|
|
29
|
+
|
|
30
|
+
def print_sweep: (untyped result) -> untyped
|
|
31
|
+
|
|
32
|
+
def print_sweep_problems: (untyped result) -> untyped
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
data/sig/bidi2pdf/cli.rbs
CHANGED
|
@@ -13,6 +13,8 @@ module Bidi2pdf
|
|
|
13
13
|
|
|
14
14
|
include JsonOutput
|
|
15
15
|
|
|
16
|
+
include SessionCommands
|
|
17
|
+
|
|
16
18
|
def self.exit_on_failure?: () -> true
|
|
17
19
|
|
|
18
20
|
def render: () -> (nil | untyped)
|
|
@@ -28,6 +30,10 @@ module Bidi2pdf
|
|
|
28
30
|
# below is what makes `bidi2pdf run recipe.yml` dispatch here.
|
|
29
31
|
def run_recipe: (untyped recipe_path) -> untyped
|
|
30
32
|
|
|
33
|
+
def sessions: () -> untyped
|
|
34
|
+
|
|
35
|
+
def sweep: () -> untyped
|
|
36
|
+
|
|
31
37
|
def template: () -> untyped
|
|
32
38
|
|
|
33
39
|
private
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
class SessionRegistry
|
|
3
|
+
# Keeps the leases of every session this process holds fresh: one thread per process renews
|
|
4
|
+
# them in their registry files every +interval+ seconds, so a sweeper in another process (a
|
|
5
|
+
# second Puma worker, a job worker sharing the registry directory) can tell a live session
|
|
6
|
+
# from one whose process died. The thread runs only while something is held.
|
|
7
|
+
#
|
|
8
|
+
# Fork-aware: a child process (Puma forks its workers) starts with nothing held and without the
|
|
9
|
+
# parent's thread - the parent's sessions are the parent's to renew.
|
|
10
|
+
module Heartbeat
|
|
11
|
+
self.@mutex: untyped
|
|
12
|
+
|
|
13
|
+
self.@held: untyped
|
|
14
|
+
|
|
15
|
+
self.@pid: untyped
|
|
16
|
+
|
|
17
|
+
self.@thread: untyped
|
|
18
|
+
|
|
19
|
+
DEFAULT_INTERVAL: untyped
|
|
20
|
+
|
|
21
|
+
# @return [Numeric] seconds between renewals - keep it well below the sweeper's lease_ttl.
|
|
22
|
+
attr_writer self.interval: untyped
|
|
23
|
+
|
|
24
|
+
def self.interval: () -> untyped
|
|
25
|
+
|
|
26
|
+
def self.hold: (untyped registry, untyped session_id) -> untyped
|
|
27
|
+
|
|
28
|
+
def self.release: (untyped registry, untyped session_id) -> untyped
|
|
29
|
+
|
|
30
|
+
# @return [Array<String>] the session ids this process holds in +registry+'s file.
|
|
31
|
+
def self.held_ids: (untyped registry) -> untyped
|
|
32
|
+
|
|
33
|
+
# Renews every held lease now.
|
|
34
|
+
def self.beat!: (?now: untyped) -> untyped
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def self.beat_loop: () -> untyped
|
|
39
|
+
|
|
40
|
+
def self.held: () -> untyped
|
|
41
|
+
|
|
42
|
+
def self.reset_after_fork: () -> (nil | untyped)
|
|
43
|
+
|
|
44
|
+
def self.synchronize: () { (?) -> untyped } -> untyped
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
# Remembers which sessions bidi2pdf opened on a shared (remote) chromedriver, when, by which
|
|
3
|
+
# process, and whether that process is still alive - in a small JSON file per chromedriver URL - so
|
|
4
|
+
# a process started later, or a ChromeSweeper, can find and close the ones a process that died
|
|
5
|
+
# without a clean shutdown left behind (see SessionSweeper). chromedriver itself keeps a session
|
|
6
|
+
# until someone deletes it, and it drops any tag we could attach to one (a custom capability is
|
|
7
|
+
# accepted but echoed nowhere - checked against chromedriver 153), hence a file.
|
|
8
|
+
#
|
|
9
|
+
# Liveness is a lease: a session the process #hold s is renewed by its Heartbeat every
|
|
10
|
+
# Heartbeat.interval seconds. A session whose lease is fresh belongs to a live process - a render
|
|
11
|
+
# in flight or a warm spare - and no sweeper closes it; one whose lease ran out belongs to a dead
|
|
12
|
+
# process. #record alone writes an entry nobody renews.
|
|
13
|
+
#
|
|
14
|
+
# Everything here is best effort and fail-open: a directory that is not writable, a locked-out or
|
|
15
|
+
# corrupt file only switches orphan tracking off for this process (one warning, one
|
|
16
|
+
# +session_warmer.registry_unavailable.bidi2pdf+ notification) - rendering never depends on it.
|
|
17
|
+
# The file is created 0600 and every change happens under an exclusive lock, so processes sharing
|
|
18
|
+
# the directory (Puma workers, a job worker, a spec run) can record at the same time.
|
|
19
|
+
class SessionRegistry
|
|
20
|
+
@path: untyped
|
|
21
|
+
|
|
22
|
+
@warned: untyped
|
|
23
|
+
|
|
24
|
+
FILE_PREFIX: "bidi2pdf-sessions-"
|
|
25
|
+
|
|
26
|
+
# Seconds a lease stays fresh without a renewal.
|
|
27
|
+
DEFAULT_LEASE_TTL: 60
|
|
28
|
+
|
|
29
|
+
attr_reader path: untyped
|
|
30
|
+
|
|
31
|
+
def self.owner: () -> ::String
|
|
32
|
+
|
|
33
|
+
def initialize: (untyped session_url, ?dir: untyped?) -> void
|
|
34
|
+
|
|
35
|
+
# Records a session the caller opened and keeps its lease fresh while this process lives.
|
|
36
|
+
def hold: (untyped session_id) -> (false | untyped)
|
|
37
|
+
|
|
38
|
+
# Stops renewing the lease but keeps the entry - for a session this process could not close, so
|
|
39
|
+
# a sweeper takes it once the lease ran out.
|
|
40
|
+
def release: (untyped session_id) -> (untyped | nil)
|
|
41
|
+
|
|
42
|
+
# Writes an entry, leased from +created_at+ on but not renewed (see #hold).
|
|
43
|
+
def record: (untyped session_id, ?created_at: untyped) -> (false | untyped)
|
|
44
|
+
|
|
45
|
+
# Forgets a session the caller closed itself (or found gone).
|
|
46
|
+
def forget: (untyped session_id) -> (false | untyped)
|
|
47
|
+
|
|
48
|
+
# Renews the leases of +session_ids+ that are still recorded.
|
|
49
|
+
def renew: (untyped session_ids, ?at: untyped) -> untyped
|
|
50
|
+
|
|
51
|
+
# Every recorded session: { id => opened at (epoch seconds) }.
|
|
52
|
+
def recorded: () -> untyped
|
|
53
|
+
|
|
54
|
+
# The recorded session ids whose lease was renewed within +ttl+ seconds before +now+ - sessions
|
|
55
|
+
# of a live process.
|
|
56
|
+
def leased: (?now: untyped, ?ttl: untyped) -> untyped
|
|
57
|
+
|
|
58
|
+
# The recorded session ids opened at or before +cutoff+ (epoch seconds).
|
|
59
|
+
def recorded_before: (untyped cutoff) -> untyped
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
def read: () -> untyped
|
|
64
|
+
|
|
65
|
+
def update: () { (untyped) -> untyped } -> untyped
|
|
66
|
+
|
|
67
|
+
# A corrupt or foreign file counts as empty rather than as an error; an entry of the first
|
|
68
|
+
# format (just the creation time) has no lease.
|
|
69
|
+
def parse: (untyped text) -> untyped
|
|
70
|
+
|
|
71
|
+
def normalize: (untyped entry) -> ({ "created_at" => untyped } | untyped)
|
|
72
|
+
|
|
73
|
+
def unavailable!: (untyped error) -> (nil | untyped)
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
module Bidi2pdf
|
|
2
|
+
# Closes sessions a SessionWarmer recorded (SessionRegistry) on a shared chromedriver that are
|
|
3
|
+
# older than a cutoff - left behind by a process that died without a clean shutdown, and holding a
|
|
4
|
+
# whole Chrome each until someone deletes them. Run once when a warmer starts.
|
|
5
|
+
#
|
|
6
|
+
# Only recorded sessions are ever touched - another tool's sessions on the same chromedriver never
|
|
7
|
+
# are, and neither is one whose lease a live process keeps renewing (SessionRegistry#hold). A
|
|
8
|
+
# recorded session that is already gone ("invalid session id" / 404) is just forgotten.
|
|
9
|
+
# Fail-open: any error is logged and ends the sweep; it never raises.
|
|
10
|
+
class SessionSweeper
|
|
11
|
+
@session_url: untyped
|
|
12
|
+
|
|
13
|
+
@registry: untyped
|
|
14
|
+
|
|
15
|
+
@http: untyped
|
|
16
|
+
|
|
17
|
+
# @param session_url [String] the chromedriver's new-session URL (".../session").
|
|
18
|
+
# @param registry [SessionRegistry]
|
|
19
|
+
# @param http [#call, nil] (method, url) -> status code; injectable for tests.
|
|
20
|
+
def initialize: (untyped session_url, untyped registry, ?http: untyped?) -> void
|
|
21
|
+
|
|
22
|
+
# Closes recorded sessions opened more than +older_than+ seconds before +now+.
|
|
23
|
+
#
|
|
24
|
+
# @return [Integer] how many sessions were actually closed.
|
|
25
|
+
def sweep: (older_than: untyped, ?now: untyped) -> untyped
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
# True when this call closed the session; a session already gone is forgotten but not counted.
|
|
30
|
+
def closed_now?: (untyped id) -> (false | untyped)
|
|
31
|
+
|
|
32
|
+
def report: (untyped closed) -> (nil | untyped)
|
|
33
|
+
|
|
34
|
+
# chromedriver answers a DELETE of a session it no longer has with 404 "invalid session id".
|
|
35
|
+
def net_http: (untyped method, untyped url) -> untyped
|
|
36
|
+
end
|
|
37
|
+
end
|