@a11ign/screenreader-fleet 0.0.0-reserved.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +94 -2
- package/dist/capture-client.d.mts +49 -0
- package/dist/capture-client.d.mts.map +1 -0
- package/dist/capture-client.mjs +352 -0
- package/dist/capture-client.mjs.map +1 -0
- package/dist/check-worker-code.d.mts +34 -0
- package/dist/check-worker-code.d.mts.map +1 -0
- package/dist/check-worker-code.mjs +142 -0
- package/dist/check-worker-code.mjs.map +1 -0
- package/dist/cli-flags.d.mts +71 -0
- package/dist/cli-flags.d.mts.map +1 -0
- package/dist/cli-flags.mjs +207 -0
- package/dist/cli-flags.mjs.map +1 -0
- package/dist/code-drift.d.mts +140 -0
- package/dist/code-drift.d.mts.map +1 -0
- package/dist/code-drift.mjs +284 -0
- package/dist/code-drift.mjs.map +1 -0
- package/dist/command-line-census.d.mts +33 -0
- package/dist/command-line-census.d.mts.map +1 -0
- package/dist/command-line-census.mjs +96 -0
- package/dist/command-line-census.mjs.map +1 -0
- package/dist/compare-workers.d.mts +3 -0
- package/dist/compare-workers.d.mts.map +1 -0
- package/dist/compare-workers.mjs +332 -0
- package/dist/compare-workers.mjs.map +1 -0
- package/dist/control-plane-isolation.d.mts +45 -0
- package/dist/control-plane-isolation.d.mts.map +1 -0
- package/dist/control-plane-isolation.mjs +67 -0
- package/dist/control-plane-isolation.mjs.map +1 -0
- package/dist/deploy-worker.d.mts +3 -0
- package/dist/deploy-worker.d.mts.map +1 -0
- package/dist/deploy-worker.mjs +333 -0
- package/dist/deploy-worker.mjs.map +1 -0
- package/dist/doctor.d.mts +216 -0
- package/dist/doctor.d.mts.map +1 -0
- package/dist/doctor.mjs +962 -0
- package/dist/doctor.mjs.map +1 -0
- package/dist/fleet-consistency.d.mts +235 -0
- package/dist/fleet-consistency.d.mts.map +1 -0
- package/dist/fleet-consistency.mjs +436 -0
- package/dist/fleet-consistency.mjs.map +1 -0
- package/dist/fleet-env.d.mts +228 -0
- package/dist/fleet-env.d.mts.map +1 -0
- package/dist/fleet-env.mjs +509 -0
- package/dist/fleet-env.mjs.map +1 -0
- package/dist/fleet-scripts.d.mts +11 -0
- package/dist/fleet-scripts.d.mts.map +1 -0
- package/dist/fleet-scripts.mjs +41 -0
- package/dist/fleet-scripts.mjs.map +1 -0
- package/dist/git-safe-env.d.mts +10 -0
- package/dist/git-safe-env.d.mts.map +1 -0
- package/dist/git-safe-env.mjs +44 -0
- package/dist/git-safe-env.mjs.map +1 -0
- package/dist/guest-run.d.mts +26 -0
- package/dist/guest-run.d.mts.map +1 -0
- package/dist/guest-run.mjs +164 -0
- package/dist/guest-run.mjs.map +1 -0
- package/dist/host-address.d.mts +33 -0
- package/dist/host-address.d.mts.map +1 -0
- package/dist/host-address.mjs +105 -0
- package/dist/host-address.mjs.map +1 -0
- package/dist/host-capacity.d.mts +64 -0
- package/dist/host-capacity.d.mts.map +1 -0
- package/dist/host-capacity.mjs +152 -0
- package/dist/host-capacity.mjs.map +1 -0
- package/dist/host-metrics.d.mts +116 -0
- package/dist/host-metrics.d.mts.map +1 -0
- package/dist/host-metrics.mjs +201 -0
- package/dist/host-metrics.mjs.map +1 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -0
- package/dist/local-vm.d.ts +125 -0
- package/dist/local-vm.d.ts.map +1 -0
- package/dist/local-vm.js +360 -0
- package/dist/local-vm.js.map +1 -0
- package/dist/measure-guard.d.mts +34 -0
- package/dist/measure-guard.d.mts.map +1 -0
- package/dist/measure-guard.mjs +73 -0
- package/dist/measure-guard.mjs.map +1 -0
- package/dist/normalise-fleet.d.mts +2 -0
- package/dist/normalise-fleet.d.mts.map +1 -0
- package/dist/normalise-fleet.mjs +76 -0
- package/dist/normalise-fleet.mjs.map +1 -0
- package/dist/npm-cli-executable.d.mts +42 -0
- package/dist/npm-cli-executable.d.mts.map +1 -0
- package/dist/npm-cli-executable.mjs +159 -0
- package/dist/npm-cli-executable.mjs.map +1 -0
- package/dist/probe-outcome.d.mts +89 -0
- package/dist/probe-outcome.d.mts.map +1 -0
- package/dist/probe-outcome.mjs +104 -0
- package/dist/probe-outcome.mjs.map +1 -0
- package/dist/protocol-guard.d.mts +34 -0
- package/dist/protocol-guard.d.mts.map +1 -0
- package/dist/protocol-guard.mjs +121 -0
- package/dist/protocol-guard.mjs.map +1 -0
- package/dist/source-walk.d.mts +12 -0
- package/dist/source-walk.d.mts.map +1 -0
- package/dist/source-walk.mjs +56 -0
- package/dist/source-walk.mjs.map +1 -0
- package/dist/transient-fault.d.mts +6 -0
- package/dist/transient-fault.d.mts.map +1 -0
- package/dist/transient-fault.mjs +86 -0
- package/dist/transient-fault.mjs.map +1 -0
- package/dist/utm-deprecated.d.mts +6 -0
- package/dist/utm-deprecated.d.mts.map +1 -0
- package/dist/utm-deprecated.mjs +23 -0
- package/dist/utm-deprecated.mjs.map +1 -0
- package/dist/worker-code-check.d.mts +29 -0
- package/dist/worker-code-check.d.mts.map +1 -0
- package/dist/worker-code-check.mjs +85 -0
- package/dist/worker-code-check.mjs.map +1 -0
- package/dist/worker-health.d.mts +56 -0
- package/dist/worker-health.d.mts.map +1 -0
- package/dist/worker-health.mjs +73 -0
- package/dist/worker-health.mjs.map +1 -0
- package/dist/worker-http.d.mts +103 -0
- package/dist/worker-http.d.mts.map +1 -0
- package/dist/worker-http.mjs +277 -0
- package/dist/worker-http.mjs.map +1 -0
- package/dist/worker-stats.d.mts +66 -0
- package/dist/worker-stats.d.mts.map +1 -0
- package/dist/worker-stats.mjs +143 -0
- package/dist/worker-stats.mjs.map +1 -0
- package/package.json +96 -4
- package/src/local-worker/autounattend.xml +280 -0
- package/src/local-worker/build-vm.sh +218 -0
- package/src/local-worker/clone-worker.sh +141 -0
- package/src/local-worker/create-utm-vm.sh +202 -0
- package/src/local-worker/fetch-windows-iso.sh +238 -0
- package/src/local-worker/first-boot.cmd +58 -0
- package/src/local-worker/worker-ctl.sh +442 -0
- package/src/provisioning/README.md +28 -0
- package/src/provisioning/apply-foreground-lock-timeout.ps1 +71 -0
- package/src/provisioning/bare-metal/README.md +213 -0
- package/src/provisioning/bare-metal/a11y-bootstrap.service +58 -0
- package/src/provisioning/bare-metal/autounattend.xml +428 -0
- package/src/provisioning/bare-metal/serve-bootstrap.sh +86 -0
- package/src/provisioning/bootstrap-control-plane.sh +463 -0
- package/src/provisioning/bootstrap-windows-worker.ps1 +649 -0
- package/src/provisioning/build-lean-worker-image.ps1 +275 -0
- package/src/provisioning/diagnose-nvda-worker.ps1 +174 -0
- package/src/provisioning/provision-nvda-worker.ps1 +827 -0
- package/src/provisioning/set-display-mode.ps1 +411 -0
- package/src/provisioning/stamp-provision-revision.ps1 +184 -0
|
@@ -0,0 +1,333 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// @ts-check
|
|
3
|
+
// Deploy the worker's code to the guests, in one command, and prove it landed.
|
|
4
|
+
//
|
|
5
|
+
// node scripts/deploy-worker.mjs # every local worker VM, one at a time
|
|
6
|
+
// node scripts/deploy-worker.mjs --vm=a11y-worker-2
|
|
7
|
+
//
|
|
8
|
+
// Why this exists: deploying was a documented twelve-step manual dance — push four files with
|
|
9
|
+
// `utmctl file push`, stop the VM, start it, then run `worker:code` and hope. Two things about that
|
|
10
|
+
// were unacceptable for something we rely on:
|
|
11
|
+
//
|
|
12
|
+
// - **It is easy to get wrong, and I got it wrong.** Pushing three of the four files leaves a guest
|
|
13
|
+
// running a mix; the symptom is a `worker:code` mismatch with no clue which file is stale.
|
|
14
|
+
// - **`utmctl exec` cannot be trusted to restart the worker**, so the reboot is mandatory and easy to
|
|
15
|
+
// skip. Skipping it makes the guest serve the previous code while reporting success — which cost two
|
|
16
|
+
// workers an hour of running stale code once already.
|
|
17
|
+
//
|
|
18
|
+
// So: one command, every hashed file, a real reboot, and a hash check over HTTP afterwards. The hash
|
|
19
|
+
// check is the whole point — it shares no failure mode with the push, which is why `/health.code`
|
|
20
|
+
// exists rather than reading the guest's files back through the same broken channel.
|
|
21
|
+
//
|
|
22
|
+
// Deploys the WORKING TREE, deliberately: that is what you are testing. Roll back by checking out the
|
|
23
|
+
// ref you want and running this again — git is the source of truth for "the previous version", so there
|
|
24
|
+
// is no bespoke backup to go stale.
|
|
25
|
+
import { pathToFileURL } from "node:url";
|
|
26
|
+
import { execFile, execFileSync } from "node:child_process";
|
|
27
|
+
import { sandboxGitEnv } from "./git-safe-env.mjs";
|
|
28
|
+
import { createReadStream, realpathSync } from "node:fs";
|
|
29
|
+
import { promisify } from "node:util";
|
|
30
|
+
import { resolve } from "node:path";
|
|
31
|
+
// By SUBPATH, never the package ROOT: the index re-exports `capture-core.mjs`, which imports guidepup and
|
|
32
|
+
// throws `No available supported screen readers` at import on any host without one. This file only
|
|
33
|
+
// runs on a Mac, where VoiceOver makes that throw invisible — which is exactly why it went unnoticed.
|
|
34
|
+
// `no-win32-imports.test.ts` found it.
|
|
35
|
+
import { WORKER_FILES } from "@a11ign/screenreader-worker/worker-files";
|
|
36
|
+
import { workerSourceDir, codeVersion } from "@a11ign/screenreader-worker/code-version";
|
|
37
|
+
// The WORKING-TREE value, imported rather than regex-scraped — architecture-audit.md §5, item 3.
|
|
38
|
+
// `protocol-version.mjs` is dependency-free for exactly this: safe to import from a portable tree, unlike
|
|
39
|
+
// the package ROOT or `capture-core.mjs` itself, which reach guidepup. The git-HEAD comparison below still
|
|
40
|
+
// has to scrape TEXT, because `git show` returns a historical file's bytes, not something importable.
|
|
41
|
+
import { CAPTURE_PROTOCOL_VERSION as PROTOCOL_IN_TREE } from "@a11ign/screenreader-worker/protocol-version";
|
|
42
|
+
import { fleetScriptPaths } from "./fleet-scripts.mjs";
|
|
43
|
+
import { refuseUnknownFlags, flagValue } from "./cli-flags.mjs";
|
|
44
|
+
import { warnUtmDeprecated } from "./utm-deprecated.mjs";
|
|
45
|
+
import { requestJson } from "./worker-http.mjs";
|
|
46
|
+
import { RECAPTURE_COST } from "./protocol-guard.mjs";
|
|
47
|
+
/**
|
|
48
|
+
* `--allow-protocol-change` is the flag that lets a CAPTURE_PROTOCOL_VERSION bump ship, invalidating
|
|
49
|
+
* every cached capture. A typo silently means "do not allow", which is the safe direction — but
|
|
50
|
+
* `--vm=` mistyped deploys to EVERY guest instead of the one named.
|
|
51
|
+
*
|
|
52
|
+
* An unrecognised flag is otherwise IGNORED, so it runs the default and reports success.
|
|
53
|
+
*/
|
|
54
|
+
refuseUnknownFlags(["--vm=", "--allow-protocol-change"], { entry: import.meta.url, command: "npm run worker:deploy" });
|
|
55
|
+
const run = promisify(execFile);
|
|
56
|
+
// From the worker PACKAGE, not from the cwd. This was `resolve("src/capture/nvda")` and then
|
|
57
|
+
// `resolve("packages/nvda-worker/src")` — a repo-layout guess that had to be edited every time the worker
|
|
58
|
+
// moved, and that silently pointed at nothing whenever the cwd was not the repo root.
|
|
59
|
+
const NVDA_DIR = workerSourceDir();
|
|
60
|
+
// The guest's layout deliberately does NOT mirror the repo's. It is where provisioning put the files and
|
|
61
|
+
// where the scheduled task points, so renaming it means re-provisioning every guest — and M5 moving the host
|
|
62
|
+
// directory to `packages/nvda-worker/src` changed nothing here. All the worker needs is that its files land in
|
|
63
|
+
// one directory together.
|
|
64
|
+
const GUEST_DIR = "C:\\Users\\witness\\a11y-witness\\src\\capture\\nvda";
|
|
65
|
+
// Resolved from THIS module: the fleet scripts ship with this package, so a cwd-relative path was only ever
|
|
66
|
+
// right when run from the repo root.
|
|
67
|
+
const CTL = fleetScriptPaths().workerCtl;
|
|
68
|
+
const LIFECYCLE_TIMEOUT_MS = 420_000;
|
|
69
|
+
// `pool` launches UTM if it is closed and polls every VM's /health. 90s was too short: it timed out
|
|
70
|
+
// mid-deploy while three guests were transitioning, and the whole run died with a bare SIGTERM.
|
|
71
|
+
const POOL_TIMEOUT_MS = 240_000;
|
|
72
|
+
const HEALTH_TIMEOUT_MS = 20_000;
|
|
73
|
+
const only = flagValue(process.argv, "vm");
|
|
74
|
+
/**
|
|
75
|
+
* The files that make up the worker's code version.
|
|
76
|
+
*
|
|
77
|
+
* Imported from the one module that defines them. This used to parse `check-worker-code.mjs`'s SOURCE with a
|
|
78
|
+
* regex for the same list — better than a third copy, which is what the comment here used to argue, but it
|
|
79
|
+
* still broke silently if that loop were ever rewritten, and "a file missing from the list deploys invisibly"
|
|
80
|
+
* is the failure it was guarding against.
|
|
81
|
+
*/
|
|
82
|
+
function hashedFiles() {
|
|
83
|
+
return WORKER_FILES;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The SHARED hasher, not a local copy of it.
|
|
87
|
+
*
|
|
88
|
+
* This used to hash raw bytes while `codeVersion()` normalises CRLF to LF -- and that difference is not
|
|
89
|
+
* cosmetic: a worker whose repo was git-cloned on Windows checks out CRLF, so the two sides hashed
|
|
90
|
+
* different bytes for identical code and the deploy verification reported STALE for ever. Measured on the
|
|
91
|
+
* first bare-metal worker: 31979b551b7a2cfa against a checkout's 22822b7a3a08969c.
|
|
92
|
+
*
|
|
93
|
+
* `code-version.test.ts` claims to enforce "one hasher" but only greps for the file list, so this file
|
|
94
|
+
* satisfied it while keeping its own implementation. Two implementations of a comparison are two chances
|
|
95
|
+
* to disagree, and the whole point of this check is that both sides agree.
|
|
96
|
+
*/
|
|
97
|
+
function localVersion() {
|
|
98
|
+
return codeVersion(NVDA_DIR);
|
|
99
|
+
}
|
|
100
|
+
async function pool() {
|
|
101
|
+
const { stdout } = await run(CTL, ["pool"], { timeout: POOL_TIMEOUT_MS, encoding: "utf8" });
|
|
102
|
+
const all = JSON.parse(stdout);
|
|
103
|
+
return only ? all.filter((/** @type {{ name: string }} */ vm) => vm.name === only) : all;
|
|
104
|
+
}
|
|
105
|
+
/** @param {string} action @param {string} vmName */
|
|
106
|
+
function ctl(action, vmName) {
|
|
107
|
+
return run(CTL, [action], {
|
|
108
|
+
timeout: LIFECYCLE_TIMEOUT_MS, encoding: "utf8",
|
|
109
|
+
env: { ...process.env, A11Y_VM_NAME: vmName },
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Push one file. `utmctl file push` reads the content from stdin, which execFile cannot stream, so the
|
|
114
|
+
* child is spawned and the file piped in.
|
|
115
|
+
*/
|
|
116
|
+
/** @param {string} uuid @param {string} file @returns {Promise<void>} */
|
|
117
|
+
function push(uuid, file) {
|
|
118
|
+
return new Promise((done, fail) => {
|
|
119
|
+
const child = execFile("utmctl", ["file", "push", uuid, `${GUEST_DIR}\\${file}`], (error) => error ? fail(new Error(`push ${file}: ${ /** @type {Error} */(error).message}`)) : done());
|
|
120
|
+
if (child.stdin)
|
|
121
|
+
createReadStream(resolve(NVDA_DIR, file)).pipe(child.stdin);
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
/** @param {string} ip @param {number} port */
|
|
125
|
+
async function healthCode(ip, port) {
|
|
126
|
+
// `requestJson`, not `fetch` -- audit §9's "the HTTP client" row: a second hand-rolled probe of the
|
|
127
|
+
// same worker JSON API, with its own timeout mechanism, buys nothing and carries `fetch`'s
|
|
128
|
+
// undifferentiated `TypeError: fetch failed` in place of a real error CODE (`ECONNREFUSED`,
|
|
129
|
+
// `EHOSTUNREACH`) -- the exact distinction `isTransient` and this repo's own diagnostics rely on
|
|
130
|
+
// elsewhere. This site was found by a tree-wide sweep, not the original audit's four named ones.
|
|
131
|
+
const response = await requestJson(`http://${ip}:${port}/health`, { timeoutMs: HEALTH_TIMEOUT_MS });
|
|
132
|
+
if (!response.ok)
|
|
133
|
+
throw new Error(`HTTP ${response.status} from /health`);
|
|
134
|
+
// `requestJson` returns `undefined` for unparseable JSON rather than throwing (its own docstring: a
|
|
135
|
+
// cache miss is a normal outcome for its usual callers) -- `fetch`'s `.json()` threw, and
|
|
136
|
+
// `healthCodeWhenAwake`'s retry loop relies on that to keep polling on garbage the same as on silence.
|
|
137
|
+
if (response.json === undefined)
|
|
138
|
+
throw new Error(`invalid JSON from http://${ip}:${port}/health`);
|
|
139
|
+
return response.json.code;
|
|
140
|
+
}
|
|
141
|
+
/** How long a rebooted guest gets to start answering before a deploy calls the verification failed. */
|
|
142
|
+
const VERIFY_BUDGET_MS = 240_000;
|
|
143
|
+
const VERIFY_POLL_MS = 10_000;
|
|
144
|
+
/**
|
|
145
|
+
* Read the guest's code hash, waiting for it to finish booting first.
|
|
146
|
+
*
|
|
147
|
+
* A guest that is not answering YET is not a failed deploy, and reading `/health` once immediately after the
|
|
148
|
+
* reboot conflated the two: `worker:deploy` printed "stale or failed" while `npm run worker:code` — run a
|
|
149
|
+
* minute later against the same guest — reported `matches`. That false alarm sent me redeploying guests that
|
|
150
|
+
* had deployed correctly, repeatedly, and the deploy is the tool whose whole job is telling you whether the
|
|
151
|
+
* push landed.
|
|
152
|
+
*
|
|
153
|
+
* Only SILENCE is waited on. A hash that answers and differs is returned straight to the caller, which
|
|
154
|
+
* compares it — so a genuine stale deploy still fails immediately and only a booting guest costs time. Boot
|
|
155
|
+
* times measured on this fleet run from 30 s to 147 s depending on how much Edge-profile hygiene the guest has
|
|
156
|
+
* to do first, so the budget is well above the slowest honest answer.
|
|
157
|
+
*/
|
|
158
|
+
/** @param {string} ip @param {number} port */
|
|
159
|
+
async function healthCodeWhenAwake(ip, port) {
|
|
160
|
+
const deadline = Date.now() + VERIFY_BUDGET_MS;
|
|
161
|
+
let last = "no answer";
|
|
162
|
+
let waited = false;
|
|
163
|
+
while (Date.now() < deadline) {
|
|
164
|
+
try {
|
|
165
|
+
const actual = await healthCode(ip, port);
|
|
166
|
+
if (waited)
|
|
167
|
+
process.stdout.write("\n");
|
|
168
|
+
return actual;
|
|
169
|
+
}
|
|
170
|
+
catch (error) {
|
|
171
|
+
last = error instanceof Error ? error.message : String(error);
|
|
172
|
+
if (!waited)
|
|
173
|
+
process.stdout.write(" waiting for the guest to answer /health ");
|
|
174
|
+
waited = true;
|
|
175
|
+
process.stdout.write(".");
|
|
176
|
+
await new Promise((resolve) => setTimeout(resolve, VERIFY_POLL_MS));
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
if (waited)
|
|
180
|
+
process.stdout.write("\n");
|
|
181
|
+
throw new Error(`${ip}:${port} never answered /health within ${VERIFY_BUDGET_MS / 1000}s (last: ${last})`);
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* Wait until a VM is no longer `stopping`, so the next deploy does not start a guest on top of one still
|
|
185
|
+
* holding its memory.
|
|
186
|
+
*
|
|
187
|
+
* Bounded and non-fatal: if it never settles we continue and let the next deploy report its own failure,
|
|
188
|
+
* because a deploy that hangs forever is worse than one that reports a stale worker.
|
|
189
|
+
*/
|
|
190
|
+
/** @param {string} name @param {number} [limitMs] */
|
|
191
|
+
async function waitUntilSettled(name, limitMs = 120_000) {
|
|
192
|
+
const deadline = Date.now() + limitMs;
|
|
193
|
+
while (Date.now() < deadline) {
|
|
194
|
+
const vm = (await pool()).find((/** @type {{ name: string }} */ v) => v.name === name);
|
|
195
|
+
if (!vm || vm.state !== "stopping")
|
|
196
|
+
return;
|
|
197
|
+
await new Promise((resolve) => setTimeout(resolve, 5_000));
|
|
198
|
+
}
|
|
199
|
+
process.stdout.write(` note: ${name} is still stopping; continuing anyway\n`);
|
|
200
|
+
}
|
|
201
|
+
/** @param {Record<string, any>} vm @param {string[]} files @param {string} expected */
|
|
202
|
+
async function deployTo(vm, files, expected) {
|
|
203
|
+
process.stdout.write(`\n=== ${vm.name} ===\n`);
|
|
204
|
+
// Push needs the guest running; the reboot afterwards is what actually loads the new code.
|
|
205
|
+
await ctl("up", vm.name);
|
|
206
|
+
for (const file of files) {
|
|
207
|
+
await push(vm.uuid, file);
|
|
208
|
+
process.stdout.write(` pushed ${file}\n`);
|
|
209
|
+
}
|
|
210
|
+
process.stdout.write(" rebooting (utmctl exec cannot be trusted to restart the worker) ...\n");
|
|
211
|
+
await ctl("stop", vm.name);
|
|
212
|
+
await ctl("up", vm.name);
|
|
213
|
+
// The restore is in a `finally` because it used to be on the SUCCESS path only, and the failure path is
|
|
214
|
+
// exactly when it matters. A guest whose health check threw was left RUNNING, and the loop then started
|
|
215
|
+
// the next VM on top of it — on a host `doctor` reports as having room for one of two, that guarantees
|
|
216
|
+
// the second times out too. It is then printed as "stale or failed", which reads as a broken guest and
|
|
217
|
+
// sent me looking to rebuild one that boots to ready in 33 s.
|
|
218
|
+
try {
|
|
219
|
+
const fresh = await pool();
|
|
220
|
+
const back = fresh.find((/** @type {{ name: string }} */ v) => v.name === vm.name);
|
|
221
|
+
if (!back?.ip)
|
|
222
|
+
throw new Error(`${vm.name} did not come back with an address`);
|
|
223
|
+
const actual = await healthCodeWhenAwake(back.ip, back.port);
|
|
224
|
+
const ok = actual === expected;
|
|
225
|
+
process.stdout.write(` /health.code ${actual} ${ok ? "== expected" : `!= expected ${expected}`}\n`);
|
|
226
|
+
return ok;
|
|
227
|
+
}
|
|
228
|
+
finally {
|
|
229
|
+
// Put it back where it was found, the same contract the run's lease honours.
|
|
230
|
+
if (vm.state !== "started")
|
|
231
|
+
await ctl("stop", vm.name).catch(() => undefined);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Refuse to deploy a CAPTURE_PROTOCOL_VERSION change unless it is asked for explicitly.
|
|
236
|
+
*
|
|
237
|
+
* This deploys the WORKING TREE, which is right for testing a change and dangerous for one specific
|
|
238
|
+
* change: the protocol version is a capture-cache key input, so shipping a bump invalidates every capture
|
|
239
|
+
* on disk and forces a full recapture (`RECAPTURE_COST`, in protocol-guard.mjs, says how big).
|
|
240
|
+
*
|
|
241
|
+
* The trap is real and was live in this repo. An uncommitted bump in a shared checkout makes
|
|
242
|
+
* `npm run worker:code` report every worker STALE, and the remedy it prints is "redeploy" — which would
|
|
243
|
+
* deploy the bump, wipe the cache, and give no clue why the next run recaptured everything.
|
|
244
|
+
*/
|
|
245
|
+
function guardProtocolChange() {
|
|
246
|
+
const inTree = String(PROTOCOL_IN_TREE);
|
|
247
|
+
let committed;
|
|
248
|
+
try {
|
|
249
|
+
committed = /CAPTURE_PROTOCOL_VERSION = (\d+)/.exec(execFileSync("git", ["-C", NVDA_DIR, "show", "HEAD:./protocol-version.mjs"], { encoding: "utf8", env: sandboxGitEnv() }))?.[1];
|
|
250
|
+
}
|
|
251
|
+
catch {
|
|
252
|
+
// Two very different situations, and one of them is a guard that has quietly stopped guarding: there may
|
|
253
|
+
// be no git checkout at all, or the PATH may have moved (M5 relocated this file from `src/capture/nvda/`;
|
|
254
|
+
// it moved AGAIN out of `capture-core.mjs` into its own file on architecture-audit.md §5, item 3) so
|
|
255
|
+
// `git show` finds nothing. The second would silently disable the most expensive check in this script,
|
|
256
|
+
// which is exactly the shape this repo keeps paying for, so it says so.
|
|
257
|
+
try {
|
|
258
|
+
execFileSync("git", ["rev-parse", "--verify", "HEAD"], { stdio: "ignore", env: sandboxGitEnv() });
|
|
259
|
+
process.stdout.write(" note: cannot compare CAPTURE_PROTOCOL_VERSION against HEAD — "
|
|
260
|
+
+ `${resolve(NVDA_DIR, "protocol-version.mjs")} is not in HEAD.\n`
|
|
261
|
+
+ " Expected for a brand-new or just-moved file; if the path moved, fix it here or this guard is off.\n");
|
|
262
|
+
}
|
|
263
|
+
catch { /* genuinely not a git checkout: nothing to compare, and nothing to warn about */ }
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
if (!inTree || !committed || inTree === committed)
|
|
267
|
+
return;
|
|
268
|
+
if (process.argv.includes("--allow-protocol-change")) {
|
|
269
|
+
process.stdout.write(`\nDeploying CAPTURE_PROTOCOL_VERSION ${committed} -> ${inTree} as requested. ` +
|
|
270
|
+
"Every cached capture is now invalid and the next run will recapture all of them.\n");
|
|
271
|
+
return;
|
|
272
|
+
}
|
|
273
|
+
process.stderr.write(`\nREFUSING TO DEPLOY: the working tree has CAPTURE_PROTOCOL_VERSION = ${inTree}, but HEAD has ` +
|
|
274
|
+
`${committed}.\n\n` +
|
|
275
|
+
"That value is a capture-cache key, so deploying it invalidates all cached captures and forces a\n" +
|
|
276
|
+
`full recapture: ${RECAPTURE_COST}. If a \`worker:code\` STALE report sent you here, the stale\n` +
|
|
277
|
+
"hash is probably caused by this uncommitted bump rather than by the guests being out of date.\n\n" +
|
|
278
|
+
" git stash # deploy without the bump, or\n" +
|
|
279
|
+
" npm run worker:deploy -- --allow-protocol-change # deploy it deliberately\n");
|
|
280
|
+
process.exit(3);
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* Nothing here runs on import.
|
|
284
|
+
*
|
|
285
|
+
* This module used to execute its whole deploy at module scope, so merely importing it — which I did, to read a
|
|
286
|
+
* path constant — ran `guardProtocolChange()` and began enumerating VMs. With guests running it would have
|
|
287
|
+
* pushed files and rebooted them. A program that reboots machines must be invoked, never merely mentioned.
|
|
288
|
+
*
|
|
289
|
+
* `check-worker-code.mjs` got the same treatment. The four remaining scripts in this repo that execute at
|
|
290
|
+
* module scope are pure programs nothing imports; they are left alone deliberately rather than restructured for
|
|
291
|
+
* symmetry.
|
|
292
|
+
*/
|
|
293
|
+
async function main() {
|
|
294
|
+
warnUtmDeprecated("npm run worker:deploy");
|
|
295
|
+
guardProtocolChange();
|
|
296
|
+
const files = hashedFiles();
|
|
297
|
+
const expected = localVersion();
|
|
298
|
+
const vms = await pool();
|
|
299
|
+
if (!vms.length) {
|
|
300
|
+
process.stderr.write(only ? `no local worker VM named ${only}\n` : "no local worker VMs registered\n");
|
|
301
|
+
process.exit(2);
|
|
302
|
+
}
|
|
303
|
+
process.stdout.write(`Deploying ${files.length} file(s) to ${vms.length} worker(s)\n`);
|
|
304
|
+
process.stdout.write(`Files: ${files.join(", ")}\nExpected code: ${expected}\n`);
|
|
305
|
+
const failed = [];
|
|
306
|
+
for (const vm of vms) {
|
|
307
|
+
try {
|
|
308
|
+
if (!await deployTo(vm, files, expected))
|
|
309
|
+
failed.push(vm.name);
|
|
310
|
+
}
|
|
311
|
+
catch (error) {
|
|
312
|
+
process.stdout.write(` FAILED: ${ /** @type {Error} */(error).message}\n`);
|
|
313
|
+
failed.push(vm.name);
|
|
314
|
+
}
|
|
315
|
+
// A `stop` returns before the guest has actually released its memory — one was observed sitting in
|
|
316
|
+
// `stopping` for minutes. Starting the next VM into that overlap is the same over-commitment by a
|
|
317
|
+
// second route, so wait for the host to be quiet before moving on.
|
|
318
|
+
await waitUntilSettled(vm.name);
|
|
319
|
+
}
|
|
320
|
+
process.stdout.write(`\n${vms.length - failed.length}/${vms.length} worker(s) on ${expected}\n`);
|
|
321
|
+
if (failed.length) {
|
|
322
|
+
process.stdout.write(`stale or failed: ${failed.join(", ")}\n`);
|
|
323
|
+
process.stdout.write("Re-run this command; if it persists, the guest is not rebooting — see docs/nvda-worker-runbook.md\n");
|
|
324
|
+
}
|
|
325
|
+
process.exit(failed.length ? 1 : 0);
|
|
326
|
+
}
|
|
327
|
+
// REALPATH'D: `import.meta.url` is resolved through symlinks by Node's ESM loader and `process.argv[1]`
|
|
328
|
+
// is not, so a bin reached via its `.bin` symlink (which is how npm always installs one) mismatched here
|
|
329
|
+
// and this guard silently read false — the tool loaded, did nothing, and exited 0. `/var` and `/tmp` are
|
|
330
|
+
// themselves symlinks on macOS, so this fired every time. Same defect, same fix, as `cli.ts`'s `isProgram`.
|
|
331
|
+
if (import.meta.url === pathToFileURL(process.argv[1] ? realpathSync(process.argv[1]) : "").href)
|
|
332
|
+
await main();
|
|
333
|
+
//# sourceMappingURL=deploy-worker.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deploy-worker.mjs","sourceRoot":"","sources":["../src/deploy-worker.mjs"],"names":[],"mappings":";AACA,YAAY;AACZ,+EAA+E;AAC/E,EAAE;AACF,uFAAuF;AACvF,sDAAsD;AACtD,EAAE;AACF,8FAA8F;AAC9F,oGAAoG;AACpG,8CAA8C;AAC9C,EAAE;AACF,qGAAqG;AACrG,8FAA8F;AAC9F,uGAAuG;AACvG,wGAAwG;AACxG,yDAAyD;AACzD,EAAE;AACF,qGAAqG;AACrG,kGAAkG;AAClG,qFAAqF;AACrF,EAAE;AACF,sGAAsG;AACtG,wGAAwG;AACxG,oCAAoC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACzD,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,0GAA0G;AAC1G,mGAAmG;AACnG,sGAAsG;AACtG,uCAAuC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,0CAA0C,CAAC;AACxE,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,0CAA0C,CAAC;AACxF,iGAAiG;AACjG,0GAA0G;AAC1G,2GAA2G;AAC3G,sGAAsG;AACtG,OAAO,EAAE,wBAAwB,IAAI,gBAAgB,EAAE,MAAM,8CAA8C,CAAC;AAC5G,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD;;;;;;GAMG;AACH,kBAAkB,CAAC,CAAC,OAAO,EAAE,yBAAyB,CAAC,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC,CAAC;AAEvH,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAChC,6FAA6F;AAC7F,0GAA0G;AAC1G,sFAAsF;AACtF,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;AACnC,yGAAyG;AACzG,6GAA6G;AAC7G,+GAA+G;AAC/G,0BAA0B;AAC1B,MAAM,SAAS,GAAG,sDAAsD,CAAC;AACzE,4GAA4G;AAC5G,qCAAqC;AACrC,MAAM,GAAG,GAAG,gBAAgB,EAAE,CAAC,SAAS,CAAC;AACzC,MAAM,oBAAoB,GAAG,OAAO,CAAC;AACrC,oGAAoG;AACpG,gGAAgG;AAChG,MAAM,eAAe,GAAG,OAAO,CAAC;AAChC,MAAM,iBAAiB,GAAG,MAAM,CAAC;AAEjC,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;AAE3C;;;;;;;GAOG;AACH,SAAS,WAAW;IAClB,OAAO,YAAY,CAAC;AACtB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,YAAY;IACnB,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5F,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,+BAA+B,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AAC3F,CAAC;AAED,oDAAoD;AACpD,SAAS,GAAG,CAAC,MAAM,EAAE,MAAM;IACzB,OAAO,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,EAAE;QACxB,OAAO,EAAE,oBAAoB,EAAE,QAAQ,EAAE,MAAM;QAC/C,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE;KAC9C,CAAC,CAAC;AACL,CAAC;AAED;;;GAGG;AACH,yEAAyE;AACzE,SAAS,IAAI,CAAC,IAAI,EAAE,IAAI;IACtB,OAAO,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE;QAChC,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,SAAS,KAAK,IAAI,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAC1F,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAA,oBAAqB,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7F,IAAI,KAAK,CAAC,KAAK;YAAE,gBAAgB,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAC/E,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8CAA8C;AAC9C,KAAK,UAAU,UAAU,CAAC,EAAE,EAAE,IAAI;IAChC,oGAAoG;IACpG,2FAA2F;IAC3F,4FAA4F;IAC5F,iGAAiG;IACjG,iGAAiG;IACjG,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,UAAU,EAAE,IAAI,IAAI,SAAS,EAAE,EAAE,SAAS,EAAE,iBAAiB,EAAE,CAAC,CAAC;IACpG,IAAI,CAAC,QAAQ,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,eAAe,CAAC,CAAC;IAC1E,oGAAoG;IACpG,0FAA0F;IAC1F,uGAAuG;IACvG,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,EAAE,IAAI,IAAI,SAAS,CAAC,CAAC;IAClG,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;AAC5B,CAAC;AAED,uGAAuG;AACvG,MAAM,gBAAgB,GAAG,OAAO,CAAC;AACjC,MAAM,cAAc,GAAG,MAAM,CAAC;AAE9B;;;;;;;;;;;;;GAaG;AACH,8CAA8C;AAC9C,KAAK,UAAU,mBAAmB,CAAC,EAAE,EAAE,IAAI;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;IAC/C,IAAI,IAAI,GAAG,WAAW,CAAC;IACvB,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAC1C,IAAI,MAAM;gBAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9D,IAAI,CAAC,MAAM;gBAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAChF,MAAM,GAAG,IAAI,CAAC;YACd,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAC1B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IACD,IAAI,MAAM;QAAE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,IAAI,KAAK,CAAC,GAAG,EAAE,IAAI,IAAI,kCAAkC,gBAAgB,GAAG,IAAI,YAAY,IAAI,GAAG,CAAC,CAAC;AAC7G,CAAC;AAED;;;;;;GAMG;AACH,qDAAqD;AACrD,KAAK,UAAU,gBAAgB,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IACtC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,+BAA+B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;QACvF,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,KAAK,KAAK,UAAU;YAAE,OAAO;QAC3C,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;IAC7D,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,yCAAyC,CAAC,CAAC;AACjF,CAAC;AAED,uFAAuF;AACvF,KAAK,UAAU,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ;IACzC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,QAAQ,CAAC,CAAC;IAC/C,2FAA2F;IAC3F,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC1B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,yEAAyE,CAAC,CAAC;IAChG,MAAM,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IAC3B,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IAEzB,wGAAwG;IACxG,wGAAwG;IACxG,uGAAuG;IACvG,uGAAuG;IACvG,8DAA8D;IAC9D,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,+BAA+B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,IAAI,CAAC,CAAC;QACnF,IAAI,CAAC,IAAI,EAAE,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,oCAAoC,CAAC,CAAC;QAC/E,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,EAAE,GAAG,MAAM,KAAK,QAAQ,CAAC;QAC/B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,eAAe,QAAQ,EAAE,IAAI,CAAC,CAAC;QACrG,OAAO,EAAE,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,6EAA6E;QAC7E,IAAI,EAAE,CAAC,KAAK,KAAK,SAAS;YAAE,MAAM,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAChF,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,mBAAmB;IAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAC;IACxC,IAAI,SAAS,CAAC;IACd,IAAI,CAAC;QACH,SAAS,GAAG,kCAAkC,CAAC,IAAI,CACjD,YAAY,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,6BAA6B,CAAC,EACzE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,yGAAyG;QACzG,0GAA0G;QAC1G,qGAAqG;QACrG,uGAAuG;QACvG,wEAAwE;QACxE,IAAI,CAAC;YACH,YAAY,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC;YAClG,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,iEAAiE;kBAC/D,GAAG,OAAO,CAAC,QAAQ,EAAE,sBAAsB,CAAC,oBAAoB;kBAChE,6GAA6G,CAAC,CAAC;QACrH,CAAC;QAAC,MAAM,CAAC,CAAC,iFAAiF,CAAC,CAAC;QAC7F,OAAO;IACT,CAAC;IACD,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO;IAC1D,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,EAAE,CAAC;QACrD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,wCAAwC,SAAS,OAAO,MAAM,iBAAiB;YAClG,oFAAoF,CAAC,CAAC;QACxF,OAAO;IACT,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,yEAAyE,MAAM,iBAAiB;QAChG,GAAG,SAAS,OAAO;QACnB,mGAAmG;QACnG,mBAAmB,cAAc,gEAAgE;QACjG,mGAAmG;QACnG,wEAAwE;QACxE,iFAAiF,CAAC,CAAC;IACrF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED;;;;;;;;;;GAUG;AACH,KAAK,UAAU,IAAI;IACjB,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;IAC3C,mBAAmB,EAAE,CAAC;IAEtB,MAAM,KAAK,GAAG,WAAW,EAAE,CAAC;IAC5B,MAAM,QAAQ,GAAG,YAAY,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,MAAM,IAAI,EAAE,CAAC;IACzB,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QAChB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,4BAA4B,IAAI,IAAI,CAAC,CAAC,CAAC,kCAAkC,CAAC,CAAC;QACvG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,KAAK,CAAC,MAAM,eAAe,GAAG,CAAC,MAAM,cAAc,CAAC,CAAC;IACvF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,oBAAoB,QAAQ,IAAI,CAAC,CAAC;IAEjF,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAA,oBAAqB,CAAC,KAAK,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC;YAC5E,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,mGAAmG;QACnG,kGAAkG;QAClG,mEAAmE;QACnE,MAAM,gBAAgB,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,iBAAiB,QAAQ,IAAI,CAAC,CAAC;IACjG,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qGAAqG,CAAC,CAAC;IAC9H,CAAC;IACD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,wGAAwG;AACxG,yGAAyG;AACzG,yGAAyG;AACzG,4GAA4G;AAC5G,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI;IAAE,MAAM,IAAI,EAAE,CAAC"}
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* The remedy for a failed local-pool query, as a runnable command plus the advice that is not one.
|
|
4
|
+
*
|
|
5
|
+
* #1059: THE DEPRECATION REFUSAL GETS THE FLEET, NOT THE SCRIPT THAT JUST REFUSED. `worker-ctl.sh` refuses
|
|
6
|
+
* on a machine the deprecation is aimed at and says so -- *"Capture on the bare-metal fleet instead:
|
|
7
|
+
* npm run fleet:status"* -- and the `fix:` line beside it said to re-run the script. **Following it
|
|
8
|
+
* exactly reproduces the refusal**, which is the one thing a refusal must never do, and it contradicted
|
|
9
|
+
* the message printed directly above it.
|
|
10
|
+
* @param {string} observed
|
|
11
|
+
* @returns {{ fix: string|null, note: string|null }}
|
|
12
|
+
*/
|
|
13
|
+
export function workerControlFix(observed: string): {
|
|
14
|
+
fix: string | null;
|
|
15
|
+
note: string | null;
|
|
16
|
+
};
|
|
17
|
+
/**
|
|
18
|
+
* Pure: does `resolvedRealPath` (already realpath'd) live under `thisCheckoutRoot` (also realpath'd)?
|
|
19
|
+
* Both must be realpath'd BEFORE calling this, never inside it -- comparing a symlinked path against a
|
|
20
|
+
* realpath'd root would report every worktree as foreign to itself, since a worktree's own files are
|
|
21
|
+
* reached through no symlink while its `node_modules` is one.
|
|
22
|
+
*
|
|
23
|
+
* @param {string} resolvedRealPath
|
|
24
|
+
* @param {string} thisCheckoutRootReal
|
|
25
|
+
* @returns {boolean}
|
|
26
|
+
*/
|
|
27
|
+
export function resolvesToThisCheckout(resolvedRealPath: string, thisCheckoutRootReal: string): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Pure: which checkout root does a resolved `packages/<name>/dist/...` path belong to? Everything before
|
|
30
|
+
* the first `/packages/` -- this repo's own, fixed layout, not a guess. `null` when the path does not look
|
|
31
|
+
* like it, which a caller must treat as "could not tell", never as "this checkout".
|
|
32
|
+
*
|
|
33
|
+
* @param {string} resolvedRealPath
|
|
34
|
+
* @returns {string | null}
|
|
35
|
+
*/
|
|
36
|
+
export function checkoutRootFor(resolvedRealPath: string): string | null;
|
|
37
|
+
/**
|
|
38
|
+
* Whether `tsc --build --dry` -- the SAME authority the real build uses, deferred to rather than
|
|
39
|
+
* reimplemented -- considers `tsconfigPath`'s own outputs up to date.
|
|
40
|
+
*
|
|
41
|
+
* NOT a timestamp comparison, and that correction cost a wrong first version of this check (#256, live-
|
|
42
|
+
* measured): a directory's mtime does not move on rewrite; a file's mtime moves on `git checkout` with no
|
|
43
|
+
* content change at all, which is the ordinary case of switching branches; and `tsc --build` itself is
|
|
44
|
+
* content-addressed for the files it actually recompiles, so a real build can be legitimately up to date
|
|
45
|
+
* with source files whose mtimes are newer than its outputs. Measured on two live worktrees straight
|
|
46
|
+
* after a branch switch: several source files newer than `dist/index.js`, and `tsc --build --dry` still
|
|
47
|
+
* correctly reported "is up to date". A raw mtime comparison would have flagged both as stale --
|
|
48
|
+
* permanently, on every worktree, the moment `git checkout` runs -- which is exactly the "readiness
|
|
49
|
+
* command that cries wolf" this file's own `advise` doc warns against.
|
|
50
|
+
*
|
|
51
|
+
* `null`, not `false`, when the run failed outright or its report never mentioned this project at all --
|
|
52
|
+
* "could not tell" and "not up to date" need opposite responses (investigate vs. rebuild), and this repo's
|
|
53
|
+
* own rule is that a lookup failure is never silently read as a clean answer.
|
|
54
|
+
*
|
|
55
|
+
* @param {string} tsconfigPath
|
|
56
|
+
* @param {{ run?: (cmd: string, args: string[]) => string }} [deps]
|
|
57
|
+
* @returns {boolean | null}
|
|
58
|
+
*/
|
|
59
|
+
export function tscProjectUpToDate(tsconfigPath: string, { run }?: {
|
|
60
|
+
run?: (cmd: string, args: string[]) => string;
|
|
61
|
+
}): boolean | null;
|
|
62
|
+
/**
|
|
63
|
+
* The agreement sentence, with WHICH FIELDS AGREED DERIVED rather than retyped — #1997.
|
|
64
|
+
*
|
|
65
|
+
* "OF N CONFIGURED", because agreement among a SUBSET is not agreement. Unreachable guests are skipped
|
|
66
|
+
* by the caller — correctly, that check is not their business — so without the denominator "3 guests
|
|
67
|
+
* agree" reads as a whole fleet on a fleet of five, which is the examined-nothing shape one step in from
|
|
68
|
+
* zero.
|
|
69
|
+
*
|
|
70
|
+
* AND THE FIELD NAMES ARE NOT TYPED HERE. This line used to read "agree on browser, screen reader, OS
|
|
71
|
+
* and protocol": four names, by hand, beside a `MUST_MATCH` that has ten. `guidepupVersion`,
|
|
72
|
+
* `architecture`, `browserProfile`, `screenReaderSettings`, `provisionRevision` and `displayMode` were
|
|
73
|
+
* all compared and none of them was mentioned, and the remediation string repeated the same four — so
|
|
74
|
+
* the sentence had been making a POSITIVE, false claim about its own scope since the fifth field was
|
|
75
|
+
* added, and no test could notice because nothing tied the words to the list. A sentence enumerating
|
|
76
|
+
* what a machine compared is a second copy of that machine's predicate; derived, it cannot go stale.
|
|
77
|
+
*
|
|
78
|
+
* `fleet:status` says nothing about field coverage and this said something untrue about it, which is why
|
|
79
|
+
* #1997's fix has to reach both. Never a FAIL either way: a mismatched pool is worse than a matched one
|
|
80
|
+
* and far better than no pool, and a diagnostic must not be the thing that takes the fleet offline.
|
|
81
|
+
*
|
|
82
|
+
* AND THE LIST IS WHAT EVERY COMPARED GUEST REPORTED, NOT WHAT ANY ONE OF THEM DID — #2034, carrying
|
|
83
|
+
* #2019's ruling into the second of the two commands #1997 named. `fields.compared` is TRUE-IF-ANYBODY,
|
|
84
|
+
* so a field ONE guest of three reported was named inside a list introduced by the words "guests agree
|
|
85
|
+
* on", and the reporter count contradicting it sat in the same return value. Measured 2026-09-22 at
|
|
86
|
+
* #2033's head: `coverage: {"field":"displayMode","reported":1,"asked":3}` beside `3 of 3 guests agree
|
|
87
|
+
* on 10 compared field(s) (..., displayMode)`. One guest's display was read. Naming it is worse than
|
|
88
|
+
* counting it, because the naming is what #1997 added to make the sentence actionable.
|
|
89
|
+
*
|
|
90
|
+
* THREE FACTS, THREE SENTENCES, and that split is #2019's ruling rather than a style choice: `N of N`
|
|
91
|
+
* is agreement, `k of N` is *some boxes did not report it* and sends a reader to the BOXES, `0 of N` is
|
|
92
|
+
* *nobody could be asked* and sends them to the FIELD. Collapsing the middle one into either of the
|
|
93
|
+
* outer two is the defect this fixes in one direction and #1997's in the other.
|
|
94
|
+
*
|
|
95
|
+
* DERIVED FROM `coverage`, AND NOT FROM `compared`/`unchecked` — the same call `fleet:status` makes
|
|
96
|
+
* (`fieldCoverageGap`, #2019). The two lists are that same measurement thresholded at "anybody", so
|
|
97
|
+
* reading the whole case off one and the partial case off the other gives one fact two sources that can
|
|
98
|
+
* disagree. The lists stay in the return value, where a caller greps them for the remedy.
|
|
99
|
+
*
|
|
100
|
+
* @param {{ agreeing: number, configured: number,
|
|
101
|
+
* fields: { compared: string[], unchecked: string[],
|
|
102
|
+
* coverage?: { field: string, reported: number, asked: number }[] } }} input
|
|
103
|
+
* @returns {string}
|
|
104
|
+
*/
|
|
105
|
+
export function fleetAgreementLine({ agreeing, configured, fields }: {
|
|
106
|
+
agreeing: number;
|
|
107
|
+
configured: number;
|
|
108
|
+
fields: {
|
|
109
|
+
compared: string[];
|
|
110
|
+
unchecked: string[];
|
|
111
|
+
coverage?: {
|
|
112
|
+
field: string;
|
|
113
|
+
reported: number;
|
|
114
|
+
asked: number;
|
|
115
|
+
}[];
|
|
116
|
+
};
|
|
117
|
+
}): string;
|
|
118
|
+
/**
|
|
119
|
+
* #1059: A COMMAND, OR `null`. Never a sentence.
|
|
120
|
+
*
|
|
121
|
+
* `--json`'s `next_command` is the field CLAUDE.md tells an agent to read and obey, so anything in it that
|
|
122
|
+
* is not runnable makes an automated reader loop -- which is exactly what happened when a failing `worker`
|
|
123
|
+
* check put *"unlock the Mac if it is locked, then re-run …"* here.
|
|
124
|
+
*
|
|
125
|
+
* **`null` and an unrunnable string are different reports**, and a JSON consumer can act on the first: it
|
|
126
|
+
* means read the checks. `contention` has no command because waiting is not one, and it says so with
|
|
127
|
+
* `null` and a `note` rather than with an imperative nobody can execute.
|
|
128
|
+
* THE PARAMETER TYPE IS WHAT THIS FUNCTION READS, not the whole check shape: `ok` and `fix`, and nothing
|
|
129
|
+
* else. A test injecting a two-field object is stating exactly the inputs the verdict depends on, and a
|
|
130
|
+
* wider type would have made it carry an `id` and a `detail` the answer cannot possibly turn on.
|
|
131
|
+
* @param {{ok: boolean, fix: string|null}[]} [checkList]
|
|
132
|
+
* @returns {string|null}
|
|
133
|
+
*/
|
|
134
|
+
export function nextCommand(checkList?: {
|
|
135
|
+
ok: boolean;
|
|
136
|
+
fix: string | null;
|
|
137
|
+
}[]): string | null;
|
|
138
|
+
/**
|
|
139
|
+
* Is `line` something a shell can run? The shape `next_command` must have, and prose must not.
|
|
140
|
+
*
|
|
141
|
+
* #1059: the field carried *"unlock the Mac if it is locked, then re-run …"*, and CLAUDE.md tells an agent
|
|
142
|
+
* to read it and do that. A shape check is the only thing that can tell a command from an imperative
|
|
143
|
+
* sentence without running it: **a command begins with an executable token** — a pnpm/npm/node/git invocation,
|
|
144
|
+
* a path, or a `VAR=value` prefix — **and an English sentence begins with a verb or an article.**
|
|
145
|
+
* @param {string|null} line
|
|
146
|
+
* @returns {boolean}
|
|
147
|
+
*/
|
|
148
|
+
export function isRunnableCommand(line: string | null): boolean;
|
|
149
|
+
/**
|
|
150
|
+
* Only when RUN, never on import.
|
|
151
|
+
*
|
|
152
|
+
* Every check here probes something real -- it spawns the Python scorer, polls each worker's `/health`,
|
|
153
|
+
* looks for strays on the pages port and reads the run's progress file -- and then calls `process.exit`.
|
|
154
|
+
* So importing this file ran the whole diagnostic against the fleet and then terminated the IMPORTING
|
|
155
|
+
* process with doctor's verdict.
|
|
156
|
+
*
|
|
157
|
+
* A brace-depth scan for dangerous calls at module scope reports this file CLEAN, because the work is one
|
|
158
|
+
* call deeper inside `checkJudge`/`checkWorker`/`checkDatasetPages`. Indirection is that check's blind
|
|
159
|
+
* spot, which is why these guards were placed by reading each file rather than by running a tool over them.
|
|
160
|
+
*/
|
|
161
|
+
/**
|
|
162
|
+
* `ready` over the checks that GATE. Exported so the case nobody runs -- a clean clone whose only failing
|
|
163
|
+
* check is `dataset` -- is drivable without a clean clone.
|
|
164
|
+
* @param {{name: string, ok: boolean}[]} checkList
|
|
165
|
+
* @returns {boolean}
|
|
166
|
+
*/
|
|
167
|
+
export function readyFrom(checkList: {
|
|
168
|
+
name: string;
|
|
169
|
+
ok: boolean;
|
|
170
|
+
}[]): boolean;
|
|
171
|
+
/**
|
|
172
|
+
* #1082: A `--json` RUN THAT CANNOT PRODUCE JSON STILL PRODUCES JSON.
|
|
173
|
+
*
|
|
174
|
+
* Measured on `1e74e3d0`: a check threw, `doctor --json` exited 1 with **zero bytes on stdout** and 1,155
|
|
175
|
+
* bytes of stack on stderr — where a `--json` consumer never looks. **"Could not ask" and "no output" are
|
|
176
|
+
* different for a caller**, and only the first is actionable; the second is indistinguishable from a
|
|
177
|
+
* command that was never run.
|
|
178
|
+
*
|
|
179
|
+
* `2>&1` IS NOT THE FIX HERE, which is what makes this different from #1068's watch job. That job's stdout
|
|
180
|
+
* is prose, so merging stderr in was free. This stdout is a PARSED format, and redirecting into it
|
|
181
|
+
* produces invalid JSON — **worse than nothing, because a consumer that parses gets a syntax error rather
|
|
182
|
+
* than a document.** The fix has to be in the tool.
|
|
183
|
+
*
|
|
184
|
+
* `checks` is present and EMPTY rather than absent OR PARTIAL — and the partial list is the one actually
|
|
185
|
+
* worth refusing. A consumer reading `.checks[]` off an absent key crashes; off a partial one it reads a
|
|
186
|
+
* list that looks exactly like a complete verdict, with **no way to tell a check that is missing because
|
|
187
|
+
* it passed from one that is missing because the run died under it.** Empty says "no verdict" and cannot
|
|
188
|
+
* be mistaken for a short one.
|
|
189
|
+
* @param {unknown} error
|
|
190
|
+
* @returns {{ready: false, error: string, checks: never[]}}
|
|
191
|
+
*/
|
|
192
|
+
export function errorDocument(error: unknown): {
|
|
193
|
+
ready: false;
|
|
194
|
+
error: string;
|
|
195
|
+
checks: never[];
|
|
196
|
+
};
|
|
197
|
+
/**
|
|
198
|
+
* The whole run, as a function of its steps and its streams, returning an exit code.
|
|
199
|
+
*
|
|
200
|
+
* @param {{ steps?: (() => unknown)[], json?: boolean, out?: (line: string) => void,
|
|
201
|
+
* err?: (line: string) => void }} [deps]
|
|
202
|
+
* @returns {Promise<number>}
|
|
203
|
+
*/
|
|
204
|
+
export function doctorRun(deps?: {
|
|
205
|
+
steps?: (() => unknown)[];
|
|
206
|
+
json?: boolean;
|
|
207
|
+
out?: (line: string) => void;
|
|
208
|
+
err?: (line: string) => void;
|
|
209
|
+
}): Promise<number>;
|
|
210
|
+
export function addCheck(name: string, ok: boolean, detail: string, remedy?: string | null | {
|
|
211
|
+
fix: string | null;
|
|
212
|
+
note?: string | null;
|
|
213
|
+
}): void;
|
|
214
|
+
export function allChecks(): string[];
|
|
215
|
+
export function gatingChecks(): string[];
|
|
216
|
+
//# sourceMappingURL=doctor.d.mts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"doctor.d.mts","sourceRoot":"","sources":["../src/doctor.mjs"],"names":[],"mappings":";AAiLA;;;;;;;;;;GAUG;AACH,2CAHW,MAAM,GACJ;IAAE,GAAG,EAAE,MAAM,GAAC,IAAI,CAAC;IAAC,IAAI,EAAE,MAAM,GAAC,IAAI,CAAA;CAAE,CAmBnD;AA4GD;;;;;;;;;GASG;AACH,yDAJW,MAAM,wBACN,MAAM,GACJ,OAAO,CAKnB;AAED;;;;;;;GAOG;AACH,kDAHW,MAAM,GACJ,MAAM,GAAG,IAAI,CAKzB;AAKD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,iDAJW,MAAM,YACN;IAAE,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,MAAM,CAAA;CAAE,GAC/C,OAAO,GAAG,IAAI,CAkB1B;AAuSD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,qEALW;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IACrC,MAAM,EAAE;QAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAC;QACxC,QAAQ,CAAC,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,EAAE,CAAA;KAAE,CAAA;CAAE,GAC7E,MAAM,CAkBlB;AA+GD;;;;;;;;;;;;;;;GAeG;AACH,wCAHW;IAAC,EAAE,EAAE,OAAO,CAAC;IAAC,GAAG,EAAE,MAAM,GAAC,IAAI,CAAA;CAAC,EAAE,GAC/B,MAAM,GAAC,IAAI,CAMvB;AAED;;;;;;;;;GASG;AACH,wCAHW,MAAM,GAAC,IAAI,GACT,OAAO,CAMnB;AAED;;;;;;;;;;;GAWG;AACH;;;;;GAKG;AACH,qCAHW;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,OAAO,CAAA;CAAC,EAAE,GAC3B,OAAO,CAInB;AAYD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,qCAHW,OAAO,GACL;IAAC,KAAK,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,KAAK,EAAE,CAAA;CAAC,CAK1D;AAED;;;;;;GAMG;AACH,iCAJW;IAAE,KAAK,CAAC,EAAE,CAAC,MAAM,OAAO,CAAC,EAAE,CAAC;IAAC,IAAI,CAAC,EAAE,OAAO,CAAC;IAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACxE,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;CAAE,GAC9B,OAAO,CAAC,MAAM,CAAC,CAgB3B;AAxzBM,+BANI,MAAM,MACN,OAAO,UACP,MAAM,WACN,MAAM,GAAC,IAAI,GAAC;IAAC,GAAG,EAAE,MAAM,GAAC,IAAI,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,GAAC,IAAI,CAAA;CAAC,QAa5D;AAsvBM,sCAA0C;AAG1C,yCAAmG"}
|