@a11ign/screenreader-fleet 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capture-client.d.mts +0 -1
- package/dist/capture-client.mjs +144 -306
- package/dist/check-worker-code.d.mts +0 -1
- package/dist/check-worker-code.mjs +42 -141
- package/dist/cli-flags.d.mts +0 -1
- package/dist/cli-flags.mjs +33 -179
- package/dist/code-drift.d.mts +0 -1
- package/dist/command-line-census.d.mts +0 -1
- package/dist/compare-workers.d.mts +0 -1
- package/dist/compare-workers.mjs +383 -255
- package/dist/control-plane-isolation.d.mts +0 -1
- package/dist/deploy-worker.d.mts +0 -1
- package/dist/deploy-worker.mjs +101 -242
- package/dist/doctor.d.mts +0 -1
- package/dist/doctor.mjs +361 -809
- package/dist/fleet-consistency.d.mts +0 -1
- package/dist/fleet-consistency.mjs +155 -379
- package/dist/fleet-env.d.mts +0 -1
- package/dist/fleet-env.mjs +148 -438
- package/dist/fleet-scripts.d.mts +0 -1
- package/dist/git-safe-env.d.mts +0 -1
- package/dist/guest-run.d.mts +0 -1
- package/dist/host-address.d.mts +0 -1
- package/dist/host-address.mjs +19 -90
- package/dist/host-capacity.d.mts +0 -1
- package/dist/host-capacity.mjs +22 -136
- package/dist/host-metrics.d.mts +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.mjs +231 -0
- package/dist/local-vm.d.ts +0 -1
- package/dist/measure-guard.d.mts +0 -1
- package/dist/normalise-fleet.d.mts +0 -1
- package/dist/npm-cli-executable.d.mts +0 -1
- package/dist/probe-outcome.d.mts +0 -1
- package/dist/probe-outcome.mjs +50 -96
- package/dist/protocol-guard.d.mts +0 -1
- package/dist/source-walk.d.mts +0 -1
- package/dist/src_fleet-scripts_mjs.mjs +16 -0
- package/dist/src_git-safe-env_mjs.mjs +9 -0
- package/dist/src_utm-deprecated_mjs.mjs +4 -0
- package/dist/transient-fault.d.mts +0 -1
- package/dist/transient-fault.mjs +21 -81
- package/dist/utm-deprecated.d.mts +0 -1
- package/dist/worker-code-check.d.mts +0 -1
- package/dist/worker-code-check.mjs +127 -76
- package/dist/worker-health.d.mts +0 -1
- package/dist/worker-health.mjs +16 -61
- package/dist/worker-http.d.mts +0 -1
- package/dist/worker-http.mjs +42 -234
- package/dist/worker-stats.d.mts +0 -1
- package/package.json +12 -5
- package/dist/capture-client.d.mts.map +0 -1
- package/dist/capture-client.mjs.map +0 -1
- package/dist/check-worker-code.d.mts.map +0 -1
- package/dist/check-worker-code.mjs.map +0 -1
- package/dist/cli-flags.d.mts.map +0 -1
- package/dist/cli-flags.mjs.map +0 -1
- package/dist/code-drift.d.mts.map +0 -1
- package/dist/code-drift.mjs +0 -284
- package/dist/code-drift.mjs.map +0 -1
- package/dist/command-line-census.d.mts.map +0 -1
- package/dist/command-line-census.mjs +0 -96
- package/dist/command-line-census.mjs.map +0 -1
- package/dist/compare-workers.d.mts.map +0 -1
- package/dist/compare-workers.mjs.map +0 -1
- package/dist/control-plane-isolation.d.mts.map +0 -1
- package/dist/control-plane-isolation.mjs +0 -67
- package/dist/control-plane-isolation.mjs.map +0 -1
- package/dist/deploy-worker.d.mts.map +0 -1
- package/dist/deploy-worker.mjs.map +0 -1
- package/dist/doctor.d.mts.map +0 -1
- package/dist/doctor.mjs.map +0 -1
- package/dist/fleet-consistency.d.mts.map +0 -1
- package/dist/fleet-consistency.mjs.map +0 -1
- package/dist/fleet-env.d.mts.map +0 -1
- package/dist/fleet-env.mjs.map +0 -1
- package/dist/fleet-scripts.d.mts.map +0 -1
- package/dist/fleet-scripts.mjs +0 -41
- package/dist/fleet-scripts.mjs.map +0 -1
- package/dist/git-safe-env.d.mts.map +0 -1
- package/dist/git-safe-env.mjs +0 -44
- package/dist/git-safe-env.mjs.map +0 -1
- package/dist/guest-run.d.mts.map +0 -1
- package/dist/guest-run.mjs +0 -164
- package/dist/guest-run.mjs.map +0 -1
- package/dist/host-address.d.mts.map +0 -1
- package/dist/host-address.mjs.map +0 -1
- package/dist/host-capacity.d.mts.map +0 -1
- package/dist/host-capacity.mjs.map +0 -1
- package/dist/host-metrics.d.mts.map +0 -1
- package/dist/host-metrics.mjs +0 -201
- package/dist/host-metrics.mjs.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -25
- package/dist/index.js.map +0 -1
- package/dist/local-vm.d.ts.map +0 -1
- package/dist/local-vm.js +0 -360
- package/dist/local-vm.js.map +0 -1
- package/dist/measure-guard.d.mts.map +0 -1
- package/dist/measure-guard.mjs +0 -73
- package/dist/measure-guard.mjs.map +0 -1
- package/dist/normalise-fleet.d.mts.map +0 -1
- package/dist/normalise-fleet.mjs +0 -76
- package/dist/normalise-fleet.mjs.map +0 -1
- package/dist/npm-cli-executable.d.mts.map +0 -1
- package/dist/npm-cli-executable.mjs +0 -159
- package/dist/npm-cli-executable.mjs.map +0 -1
- package/dist/probe-outcome.d.mts.map +0 -1
- package/dist/probe-outcome.mjs.map +0 -1
- package/dist/protocol-guard.d.mts.map +0 -1
- package/dist/protocol-guard.mjs +0 -121
- package/dist/protocol-guard.mjs.map +0 -1
- package/dist/source-walk.d.mts.map +0 -1
- package/dist/source-walk.mjs +0 -56
- package/dist/source-walk.mjs.map +0 -1
- package/dist/transient-fault.d.mts.map +0 -1
- package/dist/transient-fault.mjs.map +0 -1
- package/dist/utm-deprecated.d.mts.map +0 -1
- package/dist/utm-deprecated.mjs +0 -23
- package/dist/utm-deprecated.mjs.map +0 -1
- package/dist/worker-code-check.d.mts.map +0 -1
- package/dist/worker-code-check.mjs.map +0 -1
- package/dist/worker-health.d.mts.map +0 -1
- package/dist/worker-health.mjs.map +0 -1
- package/dist/worker-http.d.mts.map +0 -1
- package/dist/worker-http.mjs.map +0 -1
- package/dist/worker-stats.d.mts.map +0 -1
- package/dist/worker-stats.mjs +0 -143
- package/dist/worker-stats.mjs.map +0 -1
package/dist/doctor.mjs
CHANGED
|
@@ -1,962 +1,514 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// @ts-check
|
|
3
|
-
// Can I run right now? One command, one answer.
|
|
4
|
-
//
|
|
5
|
-
// pnpm run doctor human-readable, with the fix for anything broken
|
|
6
|
-
// pnpm run doctor -- --json machine-readable, for an agent
|
|
7
|
-
//
|
|
8
|
-
// This exists because "is the environment ready" took five commands and some inference, and
|
|
9
|
-
// the inference went wrong: a healthy VM was reported as a corrupted one because `utmctl`
|
|
10
|
-
// answers `unknown` when the UTM app is closed. Every check below therefore reports what it
|
|
11
|
-
// observed AND the exact command that fixes it, so nothing has to be deduced.
|
|
12
|
-
//
|
|
13
|
-
// Exit codes: 0 ready, 1 something is broken (details in the report).
|
|
14
2
|
import { execFile, execFileSync } from "node:child_process";
|
|
15
3
|
import { promisify } from "node:util";
|
|
16
|
-
import { existsSync, readFileSync, realpathSync, statSync } from "node:fs";
|
|
17
|
-
import { resolve } from "node:path";
|
|
4
|
+
import { existsSync as external_node_fs_existsSync, readFileSync, realpathSync as external_node_fs_realpathSync, statSync } from "node:fs";
|
|
5
|
+
import { delimiter as external_node_path_delimiter, dirname as external_node_path_dirname, join as external_node_path_join, resolve } from "node:path";
|
|
18
6
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
19
7
|
import { homedir } from "node:os";
|
|
20
8
|
import { createRequire } from "node:module";
|
|
21
|
-
// The canonical scrubber for this package -- `packages/guards/src/git-env.mjs` re-exports the same function for the
|
|
22
|
-
// repo-root scripts. One helper, two entry points, so a git spawn cannot inherit GIT_DIR by either door.
|
|
23
|
-
import { sandboxGitEnv } from "./git-safe-env.mjs";
|
|
24
9
|
import { availableHostMemoryMb, workersHostCanRun } from "./host-capacity.mjs";
|
|
25
10
|
import { fleetConsistency, describeMismatches } from "./fleet-consistency.mjs";
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
import { fleetScriptPaths } from "./fleet-scripts.mjs";
|
|
29
|
-
import { configuredWorkers, namedInventoryWorkers } from "./fleet-env.mjs";
|
|
11
|
+
import { fleetScriptPaths } from "./src_fleet-scripts_mjs.mjs";
|
|
12
|
+
import { namedInventoryWorkers, configuredWorkers } from "./fleet-env.mjs";
|
|
30
13
|
import { refuseUnknownFlags } from "./cli-flags.mjs";
|
|
31
|
-
import { pnpmCliInvocation } from "./npm-cli-executable.mjs";
|
|
32
14
|
import { requestJson } from "./worker-http.mjs";
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
const
|
|
15
|
+
import { sandboxGitEnv } from "./src_git-safe-env_mjs.mjs";
|
|
16
|
+
import { assessWorker } from "./worker-health.mjs";
|
|
17
|
+
function controlPlaneIsolation({ hasNodeModules, hasFleetKey, packages, isWorkspace = false }) {
|
|
18
|
+
if (!hasNodeModules || !hasFleetKey) return {
|
|
19
|
+
violated: false,
|
|
20
|
+
why: hasFleetKey ? "fleet key present, no node_modules beside it — ADR 0012 holds" : "no fleet key here, so nothing to isolate it from"
|
|
21
|
+
};
|
|
22
|
+
const count = void 0 === packages ? "" : ` (${packages} packages)`;
|
|
23
|
+
const remedy = isWorkspace ? "This is a WORKSPACE, so the dependencies belong here and the KEY does not. Dispatch fleet work through the control plane instead of holding `a11y-witness_ed25519` beside 100 MB of packages you did not audit — see docs/control-plane-plan.md L3." : "Nothing on a control plane needs them — `code-version.mjs` has no bare imports and `deploy.yml` imports it by path. Remove them: `rm -rf ~/a11y-witness/node_modules` (measured 2026-08-29: codeVersion was byte-identical afterwards).";
|
|
24
|
+
return {
|
|
25
|
+
violated: true,
|
|
26
|
+
why: `ADR 0012 VIOLATED: node_modules${count} sits beside the fleet SSH key. A compromised transitive dependency here can reach the key that reconfigures every Windows worker, each of which auto-logs-in to an unlocked desktop. ${remedy}`
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
function pnpmCliInvocation(args) {
|
|
30
|
+
const fromParent = process.env.npm_execpath ?? "";
|
|
31
|
+
if (/pnpm\.c?js$/.test(fromParent) && external_node_fs_existsSync(fromParent)) return {
|
|
32
|
+
command: process.execPath,
|
|
33
|
+
args: [
|
|
34
|
+
fromParent,
|
|
35
|
+
...args
|
|
36
|
+
]
|
|
37
|
+
};
|
|
38
|
+
const shim = onPath("pnpm");
|
|
39
|
+
if (null !== shim) return shimInvocation(shim, external_node_path_join("node_modules", "pnpm", "bin", "pnpm.cjs"), args);
|
|
40
|
+
const corepack = onPath("corepack");
|
|
41
|
+
if (null !== corepack) return shimInvocation(corepack, external_node_path_join("node_modules", "corepack", "dist", "corepack.js"), [
|
|
42
|
+
"pnpm",
|
|
43
|
+
...args
|
|
44
|
+
]);
|
|
45
|
+
throw new Error("could not find pnpm: `npm_execpath` is not a pnpm script, and neither `pnpm` nor `corepack` is on PATH. `packageManager` in the root package.json names the version -- `corepack enable` or `pnpm/action-setup` provides it.");
|
|
46
|
+
}
|
|
47
|
+
function onPath(name) {
|
|
48
|
+
for (const dir of (process.env.PATH ?? "").split(external_node_path_delimiter))if ("" !== dir) {
|
|
49
|
+
for (const candidate of [
|
|
50
|
+
external_node_path_join(dir, name),
|
|
51
|
+
external_node_path_join(dir, `${name}.cmd`)
|
|
52
|
+
])if (external_node_fs_existsSync(candidate)) return candidate;
|
|
53
|
+
}
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
function shimInvocation(shim, script, args) {
|
|
57
|
+
if (!shim.endsWith(".cmd")) return {
|
|
58
|
+
command: shim,
|
|
59
|
+
args
|
|
60
|
+
};
|
|
61
|
+
const found = [
|
|
62
|
+
external_node_path_join(external_node_path_dirname(shim), script),
|
|
63
|
+
external_node_path_join(external_node_path_dirname(process.execPath), script)
|
|
64
|
+
].find((path)=>external_node_fs_existsSync(path));
|
|
65
|
+
if (void 0 === found) throw new Error(`${shim} is a .cmd shim and its script ${script} is not beside it or beside node`);
|
|
66
|
+
return {
|
|
67
|
+
command: process.execPath,
|
|
68
|
+
args: [
|
|
69
|
+
found,
|
|
70
|
+
...args
|
|
71
|
+
]
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
refuseUnknownFlags([
|
|
75
|
+
"--json"
|
|
76
|
+
], {
|
|
77
|
+
entry: import.meta.url,
|
|
78
|
+
command: "pnpm run doctor"
|
|
79
|
+
});
|
|
80
|
+
const doctor_run = promisify(execFile);
|
|
41
81
|
const JSON_OUT = process.argv.includes("--json");
|
|
42
|
-
// A11Y_WORKERS (plural) is how a bare-metal fleet is configured -- bootstrap-control-plane.sh tells you
|
|
43
|
-
// to set exactly that -- and this read only A11Y_WORKER. So doctor reported "no A11Y_WORKER set and no
|
|
44
|
-
// local VM tooling here" against a healthy fleet of ten machines, and doctor is the FIRST command
|
|
45
|
-
// CLAUDE.md tells an agent to run. The environment was fine and the entry point said it was broken.
|
|
46
|
-
//
|
|
47
|
-
// Parsed by fleet-env.mjs, which is the ONE place that answers "what is the fleet" -- see there for why
|
|
48
|
-
// the precedence had to be settled: doctor and worker:code disagreed, so they could report on two
|
|
49
|
-
// different sets of machines with nothing to say so.
|
|
50
82
|
const WORKERS_ENV = configuredWorkers();
|
|
51
83
|
const PAGES_PORT = Number(process.env.DATASET_PAGES_PORT || 5050);
|
|
52
|
-
// The lifecycle script ships beside this one in the fleet package. It was `../scripts/local-worker/...`,
|
|
53
|
-
// resolved from this module — correct while doctor lived in `scripts/`, and silently wrong the moment it
|
|
54
|
-
// moved: doctor then reported "no local VM tooling here" on a host with three registered VMs, which reads as
|
|
55
|
-
// a broken environment rather than a broken path.
|
|
56
84
|
const CTL = fleetScriptPaths().workerCtl;
|
|
57
|
-
// Resolved from THIS module, never the cwd. The scorer being resolved against `process.cwd()` is the
|
|
58
|
-
// defect that made a fresh clone unable to run its own default judge (see packages/scorer/src/index.ts),
|
|
59
|
-
// so nothing here may repeat it.
|
|
60
85
|
const SCORER_MODEL_DIR = fileURLToPath(new URL("../../scorer/models/screenreader-scorer/", import.meta.url));
|
|
61
|
-
// Same rule as SCORER_MODEL_DIR above: resolved from THIS module, never the cwd. `@a11ign/lab`
|
|
62
|
-
// owns the canonical `runs/` resolution (`packages/lab/src/dataset-paths.mjs`), but `lab` depends on
|
|
63
|
-
// `worker-fleet`, so this package cannot import it without a cycle — this is the same computation,
|
|
64
|
-
// duplicated for that reason rather than left cwd-anchored.
|
|
65
86
|
const DATASET = resolve(fileURLToPath(new URL("../../../", import.meta.url)), "runs/screenreader-dataset");
|
|
66
87
|
const PROBE_TIMEOUT_MS = 8000;
|
|
67
|
-
/** @type {{name: string, id: string, ok: boolean, detail: string, fix: string|null, note?: string|null,
|
|
68
|
-
* advisory?: boolean}[]} */
|
|
69
88
|
const checks = [];
|
|
70
|
-
/**
|
|
71
|
-
* DOES THIS CHECK DECIDE `ready`? -- #1073, product-manager's ruling: **the dataset check must REPORT and
|
|
72
|
-
* not gate.**
|
|
73
|
-
*
|
|
74
|
-
* `const ready = checks.every((c) => c.ok)` made every check a gate, so a freshly cloned checkout with a
|
|
75
|
-
* worker configured read **NOT READY** — because the stranger had not generated a TRAINING CORPUS they
|
|
76
|
-
* have no reason to want. `doctor` is the first command the README names and CLAUDE.md tells an agent to
|
|
77
|
-
* obey its `next_command`; a verdict of NOT READY on a machine that is ready for the documented purpose
|
|
78
|
-
* **teaches the reader that the verdict is not about them**, which is #1059's defect one level up.
|
|
79
|
-
*
|
|
80
|
-
* DECLARED HERE AND ENFORCED BY `add`, WHICH THROWS ON AN UNDECLARED NAME. A list that merely sat beside
|
|
81
|
-
* the checks would drift from them silently; a list a new check cannot bypass cannot. **The author of the
|
|
82
|
-
* next check has to say which kind it is**, which is the property the row asked for — not the location.
|
|
83
|
-
*
|
|
84
|
-
* THE GATE NARROWS, IT DOES NOT VANISH. Everything a capture run actually needs still decides: a
|
|
85
|
-
* readiness command that is always ready is worse than one that is never ready, because it is believed.
|
|
86
|
-
* `dataset` is the single entry that reports without deciding, and its absence is real information on the
|
|
87
|
-
* lab path -- which is why it is still PRINTED with its fix.
|
|
88
|
-
* @type {Readonly<Record<string, boolean>>}
|
|
89
|
-
*/
|
|
90
89
|
const GATES = Object.freeze({
|
|
91
|
-
worker: true,
|
|
92
|
-
fleet: true,
|
|
93
|
-
judge: true,
|
|
94
|
-
pages: true,
|
|
95
|
-
run: true,
|
|
96
|
-
contention: true,
|
|
97
|
-
isolation: true,
|
|
90
|
+
worker: true,
|
|
91
|
+
fleet: true,
|
|
92
|
+
judge: true,
|
|
93
|
+
pages: true,
|
|
94
|
+
run: true,
|
|
95
|
+
contention: true,
|
|
96
|
+
isolation: true,
|
|
98
97
|
"dist-freshness": true,
|
|
99
98
|
"dist-resolution": true,
|
|
100
|
-
dataset: false,
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
"primary checkout": false, // a capture runs fine in a linked worktree; this guards the SHARED tree's
|
|
105
|
-
// read-only rule, which is a different kind of cannot-proceed and not one
|
|
106
|
-
// that stops a run. It reports, loudly, via `advise` on the unmarked case.
|
|
107
|
-
"fleet reach": false, // it fires only when SOME box is unreachable and says "the run will dispatch
|
|
108
|
-
// to the rest" -- a partial fleet is a smaller fleet, not no fleet. `fleet`
|
|
109
|
-
// above is the check that decides whether there is a fleet at all.
|
|
110
|
-
"host memory": false, // reports how many workers the host has room for; the run starts fewer
|
|
111
|
-
// rather than failing, which is the whole point of the number.
|
|
99
|
+
dataset: false,
|
|
100
|
+
"primary checkout": false,
|
|
101
|
+
"fleet reach": false,
|
|
102
|
+
"host memory": false
|
|
112
103
|
});
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
* @param {string} name
|
|
131
|
-
* @param {boolean} ok
|
|
132
|
-
* @param {string} detail
|
|
133
|
-
* @param {string|null|{fix: string|null, note?: string|null}} [remedy] a runnable command, `null` when
|
|
134
|
-
* none can be constructed, or `{ fix, note }` when there is advice a shell cannot run
|
|
135
|
-
*/
|
|
136
|
-
export const addCheck = (name, ok, detail, remedy = null) => {
|
|
137
|
-
// #1073: an undeclared check cannot inherit a default. The throw is the forcing function.
|
|
138
|
-
if (!Object.hasOwn(GATES, name)) {
|
|
139
|
-
throw new Error(`doctor: check "${name}" declares no entry in GATES -- say whether it decides `
|
|
140
|
-
+ "`ready` (a capture cannot happen without it) or only reports (true/false in GATES).");
|
|
141
|
-
}
|
|
142
|
-
const { fix, note } = typeof remedy === "string" || remedy === null
|
|
143
|
-
? { fix: remedy, note: null }
|
|
144
|
-
: { note: null, ...remedy };
|
|
145
|
-
checks.push({ name, id: name, ok, detail, fix, note });
|
|
104
|
+
const addCheck = (name, ok, detail, remedy = null)=>{
|
|
105
|
+
if (!Object.hasOwn(GATES, name)) throw new Error(`doctor: check "${name}" declares no entry in GATES -- say whether it decides \`ready\` (a capture cannot happen without it) or only reports (true/false in GATES).`);
|
|
106
|
+
const { fix, note } = "string" == typeof remedy || null === remedy ? {
|
|
107
|
+
fix: remedy,
|
|
108
|
+
note: null
|
|
109
|
+
} : {
|
|
110
|
+
note: null,
|
|
111
|
+
...remedy
|
|
112
|
+
};
|
|
113
|
+
checks.push({
|
|
114
|
+
name,
|
|
115
|
+
id: name,
|
|
116
|
+
ok,
|
|
117
|
+
detail,
|
|
118
|
+
fix,
|
|
119
|
+
note
|
|
120
|
+
});
|
|
146
121
|
};
|
|
147
|
-
/** The name every call site uses; `addCheck` is the same function, exported so a test can drive the
|
|
148
|
-
* GATES refusal through the real thing rather than through a copy of its rule. */
|
|
149
122
|
const add = addCheck;
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
.map((value) => String(value ?? "").trim())
|
|
165
|
-
.find(Boolean) || "unknown command failure";
|
|
123
|
+
const advise = (name, detail, fix = null)=>checks.push({
|
|
124
|
+
name,
|
|
125
|
+
id: name,
|
|
126
|
+
ok: true,
|
|
127
|
+
advisory: true,
|
|
128
|
+
detail,
|
|
129
|
+
fix
|
|
130
|
+
});
|
|
131
|
+
function commandError(error) {
|
|
132
|
+
const observed = [
|
|
133
|
+
error?.stderr,
|
|
134
|
+
error?.stdout,
|
|
135
|
+
error?.message
|
|
136
|
+
].map((value)=>String(value ?? "").trim()).find(Boolean) || "unknown command failure";
|
|
166
137
|
return observed.replace(/\s+/g, " ").slice(0, 400);
|
|
167
138
|
}
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
fix: "pnpm run fleet:status",
|
|
185
|
-
note: "the local UTM VM path is deprecated and refused to run; capture on the bare-metal fleet. "
|
|
186
|
-
+ `To use the deprecated local VM anyway: A11Y_LOCAL_VM=1 ${CTL} pool`,
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
if (/no VM named|no worker VM registered/i.test(observed)) {
|
|
190
|
-
return { fix: `${CTL} pool`,
|
|
191
|
-
note: "UTM has no registered worker VM; re-register the existing a11y-worker*.utm bundles in UTM first" };
|
|
192
|
-
}
|
|
193
|
-
return existsSync("/Applications/UTM.app")
|
|
194
|
-
? { fix: `${CTL} pool`, note: "unlock the Mac first if it is locked" }
|
|
195
|
-
: { fix: `${CTL} pool`, note: "this launches UTM if it is installed" };
|
|
139
|
+
function workerControlFix(observed) {
|
|
140
|
+
if (/DEPRECATED|refusing: set A11Y_LOCAL_VM/i.test(observed)) return {
|
|
141
|
+
fix: "pnpm run fleet:status",
|
|
142
|
+
note: `the local UTM VM path is deprecated and refused to run; capture on the bare-metal fleet. To use the deprecated local VM anyway: A11Y_LOCAL_VM=1 ${CTL} pool`
|
|
143
|
+
};
|
|
144
|
+
if (/no VM named|no worker VM registered/i.test(observed)) return {
|
|
145
|
+
fix: `${CTL} pool`,
|
|
146
|
+
note: "UTM has no registered worker VM; re-register the existing a11y-worker*.utm bundles in UTM first"
|
|
147
|
+
};
|
|
148
|
+
return external_node_fs_existsSync("/Applications/UTM.app") ? {
|
|
149
|
+
fix: `${CTL} pool`,
|
|
150
|
+
note: "unlock the Mac first if it is locked"
|
|
151
|
+
} : {
|
|
152
|
+
fix: `${CTL} pool`,
|
|
153
|
+
note: "this launches UTM if it is installed"
|
|
154
|
+
};
|
|
196
155
|
}
|
|
197
|
-
async function shell(
|
|
198
|
-
const { stdout } = await
|
|
156
|
+
async function shell(cmd, args, timeout = 30000) {
|
|
157
|
+
const { stdout } = await doctor_run(cmd, args, {
|
|
158
|
+
timeout,
|
|
159
|
+
encoding: "utf8"
|
|
160
|
+
});
|
|
199
161
|
return stdout.trim();
|
|
200
162
|
}
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
if (!response.ok)
|
|
208
|
-
throw new Error(`HTTP ${response.status}`);
|
|
209
|
-
// `requestJson` returns `undefined` for unparseable JSON rather than throwing (its own docstring: a
|
|
210
|
-
// cache miss is a normal outcome for its usual callers) -- `fetch`'s `.json()` throws, and this function's
|
|
211
|
-
// callers rely on that to distinguish "answered with garbage" from "answered correctly". Restored here.
|
|
212
|
-
if (response.json === undefined)
|
|
213
|
-
throw new Error(`invalid JSON from ${url}`);
|
|
163
|
+
async function httpJson(url) {
|
|
164
|
+
const response = await requestJson(url, {
|
|
165
|
+
timeoutMs: PROBE_TIMEOUT_MS
|
|
166
|
+
});
|
|
167
|
+
if (!response.ok) throw new Error(`HTTP ${response.status}`);
|
|
168
|
+
if (void 0 === response.json) throw new Error(`invalid JSON from ${url}`);
|
|
214
169
|
return response.json;
|
|
215
170
|
}
|
|
216
|
-
// --- the checks -----------------------------------------------------------
|
|
217
|
-
/**
|
|
218
|
-
* IS THE FLEET KEY SITTING NEXT TO 100 MB OF PACKAGES NOBODY AUDITED? — ADR 0012, checked rather than
|
|
219
|
-
* asserted.
|
|
220
|
-
*
|
|
221
|
-
* Found on 2026-08-29 to be violated on BOTH machines the ADR is about: the control plane carried 56 MB
|
|
222
|
-
* and 121 packages beside the key, and this laptop carries 103 MB beside the same key plus the lab key.
|
|
223
|
-
* The document was accurate about the intent and described a system that did not exist — which is worse
|
|
224
|
-
* than no document, because it is read as a guarantee.
|
|
225
|
-
*
|
|
226
|
-
* Reported by `doctor` because that is the command whose whole promise is that every check names its own
|
|
227
|
-
* fix, and because a check nobody runs is one this repo has learned not to write.
|
|
228
|
-
*/
|
|
229
|
-
/**
|
|
230
|
-
* IS THIS CHECKOUT MARKED AS THE PRIMARY, and does that match what it looks like?
|
|
231
|
-
*
|
|
232
|
-
* The primary-checkout guards (`pre-commit`, `post-checkout`) are OPT-IN as of #198: they fire only where
|
|
233
|
-
* `git config --local a11y.primaryCheckout` is `true`. That is the correct default — inferring it from
|
|
234
|
-
* `.git` being a directory made the hook fire on the lab, which is an ordinary clone, and broke every
|
|
235
|
-
* `lab:job -e ref=<branch>`.
|
|
236
|
-
*
|
|
237
|
-
* But an opt-in guard nobody can find the switch for is an OFF guard, and "unmarked" must not read the
|
|
238
|
-
* same as "safe". So this reports the state on every run rather than only when something is wrong — the
|
|
239
|
-
* `isolation` check above takes the same shape for the same reason: a debt that is reported every run is
|
|
240
|
-
* a known one, and a debt reported never is a forgotten one.
|
|
241
|
-
*
|
|
242
|
-
* ADVISORY, never a hard failure. `doctor` exits 0 when a RUN can proceed, and an unmarked checkout can
|
|
243
|
-
* run perfectly well — it is the fleet-driving machine's protection that is missing, not its capability.
|
|
244
|
-
* A doctor that refused READY over this would be ignored, which is how a guard gets switched off.
|
|
245
|
-
*
|
|
246
|
-
* It does not GUESS which machine deserves the mark. `doctor` runs on laptops, worktrees, the lab and CI,
|
|
247
|
-
* and telling four of those five to mark themselves would be the #198 defect wearing an advisory's
|
|
248
|
-
* clothes. It states what is true and names the command; the operator decides.
|
|
249
|
-
*/
|
|
250
171
|
function checkPrimaryCheckoutMark() {
|
|
251
172
|
const root = resolve(fileURLToPath(new URL(".", import.meta.url)), "..", "..", "..");
|
|
252
|
-
// A linked worktree's `.git` is a FILE (`gitdir: ...`) and it SHARES `.git/config` with the repository
|
|
253
|
-
// it was created from -- so the mark alone reads true inside every worktree off a marked primary. Both
|
|
254
|
-
// conditions, exactly as `scripts/git-hooks/lib/is-primary-checkout.sh` requires them; a doctor that
|
|
255
|
-
// disagreed with the hook it reports on would be worse than silent.
|
|
256
|
-
/** A linked worktree's `.git` is a FILE; an absent one is neither, and neither is the primary. */
|
|
257
173
|
let linked;
|
|
258
174
|
try {
|
|
259
175
|
linked = !statSync(resolve(root, ".git")).isDirectory();
|
|
260
|
-
}
|
|
261
|
-
catch {
|
|
176
|
+
} catch {
|
|
262
177
|
linked = false;
|
|
263
178
|
}
|
|
264
|
-
/** `git config --get` exits 1 on an absent key: not marked, not an error. */
|
|
265
179
|
let marked;
|
|
266
180
|
try {
|
|
267
|
-
marked = execFileSync("git", [
|
|
268
|
-
|
|
269
|
-
|
|
181
|
+
marked = "true" === execFileSync("git", [
|
|
182
|
+
"config",
|
|
183
|
+
"--local",
|
|
184
|
+
"--get",
|
|
185
|
+
"a11y.primaryCheckout"
|
|
186
|
+
], {
|
|
187
|
+
cwd: root,
|
|
188
|
+
encoding: "utf8",
|
|
189
|
+
env: sandboxGitEnv()
|
|
190
|
+
}).trim();
|
|
191
|
+
} catch {
|
|
270
192
|
marked = false;
|
|
271
193
|
}
|
|
272
|
-
if (linked)
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
if (marked) {
|
|
276
|
-
return add("primary checkout", true, "MARKED — pre-commit refuses commits here and post-checkout keeps it detached at origin/main");
|
|
277
|
-
}
|
|
278
|
-
advise("primary checkout", "not marked, so the primary-checkout guards are INERT here. Correct for the lab, a worker or a "
|
|
279
|
-
+ "colleague's clone; wrong for the machine that drives the fleet.", "pnpm run primary:mark -- --set (only on the fleet-driving checkout — see docs/primary-checkout.md)");
|
|
194
|
+
if (linked) return add("primary checkout", true, "a linked worktree — never the primary, whatever the shared .git/config says");
|
|
195
|
+
if (marked) return add("primary checkout", true, "MARKED — pre-commit refuses commits here and post-checkout keeps it detached at origin/main");
|
|
196
|
+
advise("primary checkout", "not marked, so the primary-checkout guards are INERT here. Correct for the lab, a worker or a colleague's clone; wrong for the machine that drives the fleet.", "pnpm run primary:mark -- --set (only on the fleet-driving checkout — see docs/primary-checkout.md)");
|
|
280
197
|
}
|
|
281
198
|
function checkControlPlaneIsolation() {
|
|
282
|
-
// `~` is a SHELL expansion, not a filesystem one: `existsSync("~/.ssh/...")` is always false, which
|
|
283
|
-
// would make this guard report every machine as compliant. The silent-pass failure mode, in the guard
|
|
284
|
-
// written because a document silently passed.
|
|
285
199
|
const raw = process.env.A11Y_SSH_KEY || "~/.ssh/a11y-witness_ed25519";
|
|
286
200
|
const keyPath = raw.startsWith("~/") ? resolve(homedir(), raw.slice(2)) : raw;
|
|
287
|
-
const hasFleetKey =
|
|
201
|
+
const hasFleetKey = external_node_fs_existsSync(keyPath);
|
|
288
202
|
const root = resolve(fileURLToPath(new URL(".", import.meta.url)), "..", "..", "..");
|
|
289
|
-
const hasNodeModules =
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
if (!verdict.violated)
|
|
298
|
-
return add("isolation", true, verdict.why);
|
|
203
|
+
const hasNodeModules = external_node_fs_existsSync(resolve(root, "node_modules"));
|
|
204
|
+
const isWorkspace = external_node_fs_existsSync(resolve(root, "packages")) && external_node_fs_existsSync(resolve(root, "package.json"));
|
|
205
|
+
const verdict = controlPlaneIsolation({
|
|
206
|
+
hasNodeModules,
|
|
207
|
+
hasFleetKey,
|
|
208
|
+
isWorkspace
|
|
209
|
+
});
|
|
210
|
+
if (!verdict.violated) return add("isolation", true, verdict.why);
|
|
299
211
|
advise("isolation", verdict.why, "docs/control-plane-plan.md L3 — drive the control plane rather than holding its keys");
|
|
300
212
|
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
* reached through no symlink while its `node_modules` is one.
|
|
306
|
-
*
|
|
307
|
-
* @param {string} resolvedRealPath
|
|
308
|
-
* @param {string} thisCheckoutRootReal
|
|
309
|
-
* @returns {boolean}
|
|
310
|
-
*/
|
|
311
|
-
export function resolvesToThisCheckout(resolvedRealPath, thisCheckoutRootReal) {
|
|
312
|
-
return resolvedRealPath === thisCheckoutRootReal
|
|
313
|
-
|| resolvedRealPath.startsWith(thisCheckoutRootReal.endsWith("/") ? thisCheckoutRootReal : `${thisCheckoutRootReal}/`);
|
|
314
|
-
}
|
|
315
|
-
/**
|
|
316
|
-
* Pure: which checkout root does a resolved `packages/<name>/dist/...` path belong to? Everything before
|
|
317
|
-
* the first `/packages/` -- this repo's own, fixed layout, not a guess. `null` when the path does not look
|
|
318
|
-
* like it, which a caller must treat as "could not tell", never as "this checkout".
|
|
319
|
-
*
|
|
320
|
-
* @param {string} resolvedRealPath
|
|
321
|
-
* @returns {string | null}
|
|
322
|
-
*/
|
|
323
|
-
export function checkoutRootFor(resolvedRealPath) {
|
|
213
|
+
function resolvesToThisCheckout(resolvedRealPath, thisCheckoutRootReal) {
|
|
214
|
+
return resolvedRealPath === thisCheckoutRootReal || resolvedRealPath.startsWith(thisCheckoutRootReal.endsWith("/") ? thisCheckoutRootReal : `${thisCheckoutRootReal}/`);
|
|
215
|
+
}
|
|
216
|
+
function checkoutRootFor(resolvedRealPath) {
|
|
324
217
|
const idx = resolvedRealPath.indexOf("/packages/");
|
|
325
|
-
return
|
|
326
|
-
}
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
* reimplemented -- considers `tsconfigPath`'s own outputs up to date.
|
|
332
|
-
*
|
|
333
|
-
* NOT a timestamp comparison, and that correction cost a wrong first version of this check (#256, live-
|
|
334
|
-
* measured): a directory's mtime does not move on rewrite; a file's mtime moves on `git checkout` with no
|
|
335
|
-
* content change at all, which is the ordinary case of switching branches; and `tsc --build` itself is
|
|
336
|
-
* content-addressed for the files it actually recompiles, so a real build can be legitimately up to date
|
|
337
|
-
* with source files whose mtimes are newer than its outputs. Measured on two live worktrees straight
|
|
338
|
-
* after a branch switch: several source files newer than `dist/index.js`, and `tsc --build --dry` still
|
|
339
|
-
* correctly reported "is up to date". A raw mtime comparison would have flagged both as stale --
|
|
340
|
-
* permanently, on every worktree, the moment `git checkout` runs -- which is exactly the "readiness
|
|
341
|
-
* command that cries wolf" this file's own `advise` doc warns against.
|
|
342
|
-
*
|
|
343
|
-
* `null`, not `false`, when the run failed outright or its report never mentioned this project at all --
|
|
344
|
-
* "could not tell" and "not up to date" need opposite responses (investigate vs. rebuild), and this repo's
|
|
345
|
-
* own rule is that a lookup failure is never silently read as a clean answer.
|
|
346
|
-
*
|
|
347
|
-
* @param {string} tsconfigPath
|
|
348
|
-
* @param {{ run?: (cmd: string, args: string[]) => string }} [deps]
|
|
349
|
-
* @returns {boolean | null}
|
|
350
|
-
*/
|
|
351
|
-
export function tscProjectUpToDate(tsconfigPath, { run = defaultTscRun } = {}) {
|
|
352
|
-
/** @type {string} */
|
|
218
|
+
return -1 === idx ? null : resolvedRealPath.slice(0, idx);
|
|
219
|
+
}
|
|
220
|
+
const defaultTscRun = (cmd, args)=>execFileSync(cmd, args, {
|
|
221
|
+
encoding: "utf8"
|
|
222
|
+
});
|
|
223
|
+
function tscProjectUpToDate(tsconfigPath, { run = defaultTscRun } = {}) {
|
|
353
224
|
let output;
|
|
354
225
|
try {
|
|
355
|
-
const tsc = pnpmCliInvocation([
|
|
226
|
+
const tsc = pnpmCliInvocation([
|
|
227
|
+
"exec",
|
|
228
|
+
"tsc",
|
|
229
|
+
"--build",
|
|
230
|
+
"--dry",
|
|
231
|
+
tsconfigPath
|
|
232
|
+
]);
|
|
356
233
|
output = run(tsc.command, tsc.args);
|
|
234
|
+
} catch (error) {
|
|
235
|
+
output = error?.stdout ?? "";
|
|
236
|
+
if (!output) return null;
|
|
357
237
|
}
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
// a missing tsconfig, a syntax error blocking even the dry check -- and its stdout, if any, is still
|
|
361
|
-
// worth reading rather than discarded.
|
|
362
|
-
output = /** @type {{stdout?: string}} */ (error)?.stdout ?? "";
|
|
363
|
-
if (!output)
|
|
364
|
-
return null;
|
|
365
|
-
}
|
|
366
|
-
const line = output.split("\n").find((l) => l.includes(tsconfigPath));
|
|
367
|
-
if (!line)
|
|
368
|
-
return null;
|
|
238
|
+
const line = output.split("\n").find((l)=>l.includes(tsconfigPath));
|
|
239
|
+
if (!line) return null;
|
|
369
240
|
return /is up to date/.test(line);
|
|
370
241
|
}
|
|
371
|
-
/**
|
|
372
|
-
* ", N commit(s) behind origin/main" or "" -- best-effort, and silently empty on any failure (not a git
|
|
373
|
-
* checkout at all, no `origin/main`, `otherRoot` unknown). This is a NOTE on an already-advisory finding,
|
|
374
|
-
* not itself a fact `doctor` promises; the resolution mismatch is reported either way.
|
|
375
|
-
*
|
|
376
|
-
* @param {string | null} otherRoot
|
|
377
|
-
* @returns {string}
|
|
378
|
-
*/
|
|
379
242
|
function behindOriginMainNote(otherRoot) {
|
|
380
|
-
if (!otherRoot)
|
|
381
|
-
return "";
|
|
243
|
+
if (!otherRoot) return "";
|
|
382
244
|
try {
|
|
383
|
-
const behindBy = execFileSync("git", [
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
245
|
+
const behindBy = execFileSync("git", [
|
|
246
|
+
"rev-list",
|
|
247
|
+
"--count",
|
|
248
|
+
"HEAD..origin/main"
|
|
249
|
+
], {
|
|
250
|
+
cwd: otherRoot,
|
|
251
|
+
encoding: "utf8",
|
|
252
|
+
env: sandboxGitEnv()
|
|
253
|
+
}).trim();
|
|
254
|
+
return "0" === behindBy ? "" : `, ${behindBy} commit(s) behind origin/main`;
|
|
255
|
+
} catch {
|
|
387
256
|
return "";
|
|
388
257
|
}
|
|
389
258
|
}
|
|
390
|
-
/**
|
|
391
|
-
* WHOSE dist a cross-package import actually resolves to, and is IT stale (#256) -- both computed from
|
|
392
|
-
* the exact SPECIFIER a real import site in this repo uses, never the bare package name. CLAUDE.md's own
|
|
393
|
-
* recorded lesson: "resolving @a11ign/judge does not prove @a11ign/judge/rules came from your
|
|
394
|
-
* tree" -- a package can export subpaths from elsewhere, so resolving the root proves nothing about a
|
|
395
|
-
* subpath. `@a11ign/judge/rules` is a real specifier this repo imports
|
|
396
|
-
* (`packages/lab/scripts/score-rules.ts` and others), not a synthetic probe.
|
|
397
|
-
*
|
|
398
|
-
* ADVISORY, never a hard failure -- same reasoning as `isolation` above: a worktree resolving to the
|
|
399
|
-
* primary's dist can still run every command correctly today, and a doctor that refused READY over an
|
|
400
|
-
* environmental fact would be ignored, which is how a guard gets switched off. It is reported every run
|
|
401
|
-
* so a stale answer is a known condition, not a silent one.
|
|
402
|
-
*/
|
|
403
259
|
function checkCrossPackageDist() {
|
|
404
260
|
const specifier = "@a11ign/judge/rules";
|
|
405
261
|
const thisCheckoutRoot = resolve(fileURLToPath(new URL(".", import.meta.url)), "..", "..", "..");
|
|
406
|
-
/** @type {string} */
|
|
407
262
|
let resolvedRealPath;
|
|
408
263
|
try {
|
|
409
|
-
resolvedRealPath =
|
|
410
|
-
}
|
|
411
|
-
|
|
412
|
-
return advise("dist-resolution", `could not resolve ${specifier} to check whose dist it comes from -- `
|
|
413
|
-
+ `${ /** @type {Error} */(error).message}`, "pnpm run build");
|
|
414
|
-
}
|
|
415
|
-
if (resolvesToThisCheckout(resolvedRealPath, realpathSync(thisCheckoutRoot))) {
|
|
416
|
-
add("dist-resolution", true, `${specifier} resolves to this checkout's own dist`);
|
|
264
|
+
resolvedRealPath = external_node_fs_realpathSync(createRequire(import.meta.url).resolve(specifier));
|
|
265
|
+
} catch (error) {
|
|
266
|
+
return advise("dist-resolution", `could not resolve ${specifier} to check whose dist it comes from -- ${error.message}`, "pnpm run build");
|
|
417
267
|
}
|
|
268
|
+
if (resolvesToThisCheckout(resolvedRealPath, external_node_fs_realpathSync(thisCheckoutRoot))) add("dist-resolution", true, `${specifier} resolves to this checkout's own dist`);
|
|
418
269
|
else {
|
|
419
270
|
const behindNote = behindOriginMainNote(checkoutRootFor(resolvedRealPath));
|
|
420
271
|
advise("dist-resolution", `${specifier} resolves to ${resolvedRealPath} (NOT this checkout${behindNote})`, "pnpm run primary:update && pnpm run build # if that is the primary checkout");
|
|
421
272
|
}
|
|
422
|
-
// THE HALF A RESOLUTION CHECK ALONE MISSES: resolving to your OWN tree is no protection if your own
|
|
423
|
-
// dist is stale -- so this checks freshness of whichever checkout the specifier ACTUALLY resolved to,
|
|
424
|
-
// not always this one. `tsc --build --dry`, never a raw mtime comparison -- see `tscProjectUpToDate`'s
|
|
425
|
-
// own doc for the live-measured reason.
|
|
426
273
|
const distRoot = checkoutRootFor(resolvedRealPath) ?? thisCheckoutRoot;
|
|
427
274
|
const tsconfigPath = resolve(distRoot, "packages/judge/tsconfig.json");
|
|
428
275
|
const upToDate = tscProjectUpToDate(tsconfigPath);
|
|
429
|
-
if (
|
|
430
|
-
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
return add("dist-freshness", true, `packages/judge under ${distRoot} is up to date (tsc --build --dry)`);
|
|
434
|
-
}
|
|
435
|
-
advise("dist-freshness", `packages/judge under ${distRoot} is NOT up to date (tsc --build --dry) -- a `
|
|
436
|
-
+ "build compiled before the source it now reflects", "pnpm run build # in that checkout");
|
|
437
|
-
}
|
|
438
|
-
// The DEFAULT here was "codex", and every part of that was wrong. `judge.ts` has no codex case at all —
|
|
439
|
-
// it offers local, anthropic and openai — so with JUDGE_BACKEND unset (the normal case) this told the
|
|
440
|
-
// operator to "install Codex and run: codex login" for a backend the product cannot use. And setting
|
|
441
|
-
// JUDGE_BACKEND=local fell into the other branch and checked JUDGE_BASE_URL, which local does not need.
|
|
442
|
-
// Both answers were wrong, in a command whose whole promise is that every check names its own fix.
|
|
443
|
-
//
|
|
444
|
-
// Mirrors judge.ts's default deliberately: a doctor that disagrees with the thing it inspects is worse
|
|
445
|
-
// than no doctor.
|
|
276
|
+
if (null === upToDate) return advise("dist-freshness", `could not ask tsc whether ${tsconfigPath} is up to date`, "pnpm run build");
|
|
277
|
+
if (upToDate) return add("dist-freshness", true, `packages/judge under ${distRoot} is up to date (tsc --build --dry)`);
|
|
278
|
+
advise("dist-freshness", `packages/judge under ${distRoot} is NOT up to date (tsc --build --dry) -- a build compiled before the source it now reflects`, "pnpm run build # in that checkout");
|
|
279
|
+
}
|
|
446
280
|
async function checkJudge() {
|
|
447
|
-
// `||`, not `??`: an env var set to the EMPTY string is how CI passes "unset", and `??` only defaults
|
|
448
|
-
// on nullish — so an empty JUDGE_BACKEND matched no backend and reported a typo that nobody made.
|
|
449
281
|
const backend = (process.env.JUDGE_BACKEND || "local").toLowerCase();
|
|
450
|
-
if (
|
|
282
|
+
if ("local" === backend) {
|
|
451
283
|
const weights = resolve(SCORER_MODEL_DIR, "model.safetensors");
|
|
452
|
-
return add("judge",
|
|
284
|
+
return add("judge", external_node_fs_existsSync(weights), external_node_fs_existsSync(weights) ? "backend=local, trained scorer present" : "backend=local, but the trained scorer is missing", `expected weights at ${weights} — they ship in the repo, so this means an incomplete checkout`);
|
|
453
285
|
}
|
|
454
|
-
if (
|
|
455
|
-
const key =
|
|
286
|
+
if ("anthropic" === backend || "openai" === backend) {
|
|
287
|
+
const key = "anthropic" === backend ? "ANTHROPIC_API_KEY" : "JUDGE_BASE_URL";
|
|
456
288
|
return add("judge", !!process.env[key], `backend=${backend}`, `export ${key}=...`);
|
|
457
289
|
}
|
|
458
|
-
// Refuse an unknown backend rather than reporting on one that will not run — the same rule action.yml
|
|
459
|
-
// applies, because a typo must not quietly change which judge assessed the page.
|
|
460
290
|
add("judge", false, `backend=${backend} is not one of local, anthropic, openai`, "unset JUDGE_BACKEND to use the default local scorer");
|
|
461
291
|
}
|
|
462
|
-
// Workers, as a POOL, and with the right idea of what "ready" means.
|
|
463
|
-
//
|
|
464
|
-
// This check used to look at one VM and fail if it was not started. That was true before runs
|
|
465
|
-
// managed the VM themselves; now a stopped worker is the correct resting state -- a run starts
|
|
466
|
-
// what it needs and puts it back afterwards -- so reporting it as FAIL told an agent the
|
|
467
|
-
// environment was broken when it was idle. One did exactly that: it went hunting for a
|
|
468
|
-
// decommissioned worker on another host and then reached for the UTM GUI.
|
|
469
|
-
//
|
|
470
|
-
// Ready means "a run can proceed", not "everything is already running".
|
|
471
292
|
async function checkWorker() {
|
|
472
|
-
if (WORKERS_ENV.length)
|
|
473
|
-
return checkConfiguredFleet(WORKERS_ENV);
|
|
474
|
-
// THE INVENTORY IS A FLEET, and `doctor` could not see one.
|
|
475
|
-
//
|
|
476
|
-
// It resolved A11Y_WORKERS, then the local UTM pool, then gave up — so on a Mac with any registered VM
|
|
477
|
-
// it reported the DEPRECATED local guests and never `inventory.yml`, which every other fleet command
|
|
478
|
-
// treats as the source of truth. Measured 2026-08-29 on one machine, at one moment: `doctor` said
|
|
479
|
-
// "2 worker(s), all stopped — READY" while `worker:code` said "checking 5 worker(s) from inventory.yml"
|
|
480
|
-
// and `fleet:status` showed those five busy with a corpus run.
|
|
481
|
-
//
|
|
482
|
-
// Three commands describing three different fleets, and `doctor` is the one CLAUDE.md tells an agent to
|
|
483
|
-
// run FIRST. Its `next_command` said `training:capture`, which would have captured on the wrong machines.
|
|
484
|
-
//
|
|
485
|
-
// THE INVENTORY WINS OUTRIGHT. The local UTM pool is deprecated — it was a testing arrangement — so it
|
|
486
|
-
// is a fallback for a machine with no inventory, never a contender with one. Anything else reproduces
|
|
487
|
-
// the divergence above on any developer Mac that still has a bundle registered.
|
|
293
|
+
if (WORKERS_ENV.length) return checkConfiguredFleet(WORKERS_ENV);
|
|
488
294
|
const inventory = namedInventoryWorkers();
|
|
489
|
-
if (inventory.length)
|
|
490
|
-
|
|
491
|
-
if (process.platform !== "darwin" || !existsSync(CTL)) {
|
|
492
|
-
return add("worker", false, "no A11Y_WORKERS set, no inventory.yml fleet, and no local VM tooling here", "set A11Y_WORKERS=http://host:8765[,http://host2:8765], or see docs/getting-started.md");
|
|
493
|
-
}
|
|
295
|
+
if (inventory.length) return checkConfiguredFleet(inventory);
|
|
296
|
+
if ("darwin" !== process.platform || !external_node_fs_existsSync(CTL)) return add("worker", false, "no A11Y_WORKERS set, no inventory.yml fleet, and no local VM tooling here", "set A11Y_WORKERS=http://host:8765[,http://host2:8765], or see docs/getting-started.md");
|
|
494
297
|
let pool;
|
|
495
298
|
try {
|
|
496
|
-
pool = JSON.parse(await shell(CTL, [
|
|
497
|
-
|
|
498
|
-
|
|
299
|
+
pool = JSON.parse(await shell(CTL, [
|
|
300
|
+
"pool"
|
|
301
|
+
], 90000));
|
|
302
|
+
} catch (e) {
|
|
499
303
|
return add("worker", false, `could not query the local pool (${commandError(e)})`, workerControlFix(commandError(e)));
|
|
500
304
|
}
|
|
501
|
-
if (!pool.length)
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
const
|
|
505
|
-
const
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
else if (healthy.length) {
|
|
513
|
-
add("worker", true, `${healthy.length}/${pool.length} ready — ${summary}`);
|
|
514
|
-
}
|
|
515
|
-
else {
|
|
516
|
-
add("worker", true, `${pool.length} worker(s), all stopped — a run starts them automatically (${summary})`);
|
|
517
|
-
}
|
|
518
|
-
// Same shape as a configured fleet, so the two diagnostics below have ONE implementation. They were
|
|
519
|
-
// pure functions over /health JSON that only the UTM branch could reach, which meant a bare-metal
|
|
520
|
-
// fleet -- the direction this project is going -- got neither.
|
|
521
|
-
const reachable = pool.filter((/** @type {any} */ v) => v.healthy && v.ip)
|
|
522
|
-
.map((/** @type {any} */ v) => ({ name: v.name, url: `http://${v.ip}:${v.port}` }));
|
|
305
|
+
if (!pool.length) return add("worker", false, "no worker VM registered", "UTM has no registered worker VM; re-register an existing a11y-worker*.utm bundle, or build one from docs/getting-started.md");
|
|
306
|
+
const running = pool.filter((vm)=>"started" === vm.state);
|
|
307
|
+
const healthy = pool.filter((vm)=>vm.healthy);
|
|
308
|
+
const brokenlyRunning = running.filter((vm)=>!vm.healthy);
|
|
309
|
+
const summary = pool.map((vm)=>`${vm.name}=${vm.healthy ? vm.ip : vm.state}`).join(" ");
|
|
310
|
+
if (brokenlyRunning.length) add("worker", false, `${summary} — ${brokenlyRunning.map((v)=>v.name).join(", ")} running but not answering`, `Start-ScheduledTask -TaskName a11ysrv on that guest, or ${CTL} stop && ${CTL} up`);
|
|
311
|
+
else healthy.length ? add("worker", true, `${healthy.length}/${pool.length} ready — ${summary}`) : add("worker", true, `${pool.length} worker(s), all stopped — a run starts them automatically (${summary})`);
|
|
312
|
+
const reachable = pool.filter((v)=>v.healthy && v.ip).map((v)=>({
|
|
313
|
+
name: v.name,
|
|
314
|
+
url: `http://${v.ip}:${v.port}`
|
|
315
|
+
}));
|
|
523
316
|
const probed = await probeAll(reachable);
|
|
524
317
|
await checkDegradedWorkers(probed);
|
|
525
318
|
checkFleetConsistency(probed, pool.length);
|
|
526
319
|
checkHostCapacity(pool);
|
|
527
|
-
const busy = pool.filter((
|
|
528
|
-
if (busy.length) {
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
}
|
|
320
|
+
const busy = pool.filter((vm)=>vm.busy);
|
|
321
|
+
if (busy.length) add("contention", false, `${busy.map((v)=>v.name).join(", ")} busy with a capture — another shell or agent is using the pool`, {
|
|
322
|
+
fix: null,
|
|
323
|
+
note: "wait for it, or you will both see the other's restarts as breakage"
|
|
324
|
+
});
|
|
533
325
|
}
|
|
534
|
-
/**
|
|
535
|
-
* The fleet named by A11Y_WORKERS: probe every one, report per worker, never fail on a single loss.
|
|
536
|
-
*
|
|
537
|
-
* "Ready means a run can proceed" is this file's own rule, and for a fleet that means AT LEAST ONE
|
|
538
|
-
* worker answering -- not all of them. The dispatcher already evicts a worker after three consecutive
|
|
539
|
-
* failures and requeues its cases (capture-decisions.mjs), so one dead machine costs throughput, not
|
|
540
|
-
* the run. Failing the whole check for it would tell an agent the environment is broken when nine
|
|
541
|
-
* workers are sitting idle and ready, which is the exact mistake the comment above checkWorker
|
|
542
|
-
* describes for a stopped VM.
|
|
543
|
-
*/
|
|
544
|
-
/**
|
|
545
|
-
* Ask every worker once, and keep the failures as data.
|
|
546
|
-
*
|
|
547
|
-
* Shared because both fleet branches feed the same two diagnostics, and because `doctor` used to probe
|
|
548
|
-
* `/health` THREE times per worker — once here, once for degradation, once for consistency — so the three
|
|
549
|
-
* sections could describe three different moments. A box that went busy between them was reported ready by
|
|
550
|
-
* one and silently skipped by the next.
|
|
551
|
-
*
|
|
552
|
-
* @param {{name: string, url: string}[]} workers
|
|
553
|
-
*/
|
|
554
326
|
async function probeAll(workers) {
|
|
555
327
|
const probed = [];
|
|
556
|
-
for (const w of workers) {
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
328
|
+
for (const w of workers)try {
|
|
329
|
+
probed.push({
|
|
330
|
+
...w,
|
|
331
|
+
health: await httpJson(`${w.url}/health`)
|
|
332
|
+
});
|
|
333
|
+
} catch (e) {
|
|
334
|
+
probed.push({
|
|
335
|
+
...w,
|
|
336
|
+
health: null,
|
|
337
|
+
error: e.message
|
|
338
|
+
});
|
|
563
339
|
}
|
|
564
340
|
return probed;
|
|
565
341
|
}
|
|
566
|
-
async function checkConfiguredFleet(
|
|
342
|
+
async function checkConfiguredFleet(workers) {
|
|
567
343
|
const probed = await probeAll(workers);
|
|
568
|
-
const reachable = probed.filter((p)
|
|
569
|
-
const ready = reachable.filter((p)
|
|
570
|
-
const state = (
|
|
571
|
-
if (!p.health)
|
|
572
|
-
|
|
573
|
-
if (p.health.busy)
|
|
574
|
-
return "busy";
|
|
344
|
+
const reachable = probed.filter((p)=>p.health);
|
|
345
|
+
const ready = reachable.filter((p)=>p.health.ready);
|
|
346
|
+
const state = (p)=>{
|
|
347
|
+
if (!p.health) return "unreachable";
|
|
348
|
+
if (p.health.busy) return "busy";
|
|
575
349
|
return p.health.ready ? "ready" : "not-ready";
|
|
576
350
|
};
|
|
577
|
-
const summary = probed.map((p)
|
|
578
|
-
if (!reachable.length) {
|
|
579
|
-
return add("worker", false, `${workers.length} configured, none answering — ${summary}`, "check those machines are up and their a11ysrv task is running; "
|
|
580
|
-
+ `curl ${workers[0].url}/health from this host`);
|
|
581
|
-
}
|
|
582
|
-
// A worker that answers but is not ready is normal right after a boot and clears on its own --
|
|
583
|
-
// /health's own note says so -- which is why this reports the count rather than failing on it.
|
|
351
|
+
const summary = probed.map((p)=>`${p.name}=${state(p)}`).join(" ");
|
|
352
|
+
if (!reachable.length) return add("worker", false, `${workers.length} configured, none answering — ${summary}`, `check those machines are up and their a11ysrv task is running; curl ${workers[0].url}/health from this host`);
|
|
584
353
|
add("worker", true, `${ready.length}/${workers.length} ready — ${summary}`);
|
|
585
|
-
const unreachable = probed.filter((p)
|
|
586
|
-
if (unreachable.length) {
|
|
587
|
-
add("fleet reach", true, `${unreachable.length} not answering: ${unreachable.map((p) => p.name).join(", ")}`
|
|
588
|
-
+ " — the run will dispatch to the rest");
|
|
589
|
-
}
|
|
590
|
-
// ONE PROBE, PASSED DOWN. These re-probed `/health` themselves, so `doctor` made three requests per
|
|
591
|
-
// worker and the three sections could describe three different moments — a box that went busy or
|
|
592
|
-
// unreachable between them was reported ready by one and silently skipped by the next.
|
|
354
|
+
const unreachable = probed.filter((p)=>!p.health);
|
|
355
|
+
if (unreachable.length) add("fleet reach", true, `${unreachable.length} not answering: ${unreachable.map((p)=>p.name).join(", ")} — the run will dispatch to the rest`);
|
|
593
356
|
await checkDegradedWorkers(probed);
|
|
594
357
|
await checkFleetConsistency(probed, workers.length);
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
// #1059: no `fix`, because waiting is not a command. The advice is a note and `next_command` is null.
|
|
600
|
-
{ fix: null, note: "wait for it, or you will both see the other's restarts as breakage" });
|
|
601
|
-
}
|
|
358
|
+
if (reachable.length && reachable.every((p)=>p.health.busy)) add("contention", false, `all ${reachable.length} reachable worker(s) busy — another run or agent has the fleet`, {
|
|
359
|
+
fix: null,
|
|
360
|
+
note: "wait for it, or you will both see the other's restarts as breakage"
|
|
361
|
+
});
|
|
602
362
|
}
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
// none, so this never surfaced anywhere. Measured on this pool: one worker needed a recovery on 4 of 4
|
|
607
|
-
// captures (nvdaStart 19.1s each, WALL 122.9s) beside one that needed none (WALL 40.6s). Reported, not
|
|
608
|
-
// failed: a degraded worker is slow, not broken, and pulling it costs more throughput than it saves.
|
|
609
|
-
async function checkDegradedWorkers(/** @type {any} */ probed) {
|
|
610
|
-
for (const w of probed) {
|
|
611
|
-
if (!w.health)
|
|
612
|
-
continue; // unreachable is already the worker check's business
|
|
363
|
+
async function checkDegradedWorkers(probed) {
|
|
364
|
+
for (const w of probed){
|
|
365
|
+
if (!w.health) continue;
|
|
613
366
|
const { degraded, reason } = assessWorker(w.health.vitals);
|
|
614
|
-
if (degraded) {
|
|
615
|
-
add(`worker ${w.name}`, true, `DEGRADED — ${reason}`, `re-provision ${w.name}: packages/worker-fleet/src/provisioning/provision-nvda-worker.ps1, elevated,`
|
|
616
|
-
+ " in the interactive session");
|
|
617
|
-
}
|
|
367
|
+
if (degraded) add(`worker ${w.name}`, true, `DEGRADED — ${reason}`, `re-provision ${w.name}: packages/worker-fleet/src/provisioning/provision-nvda-worker.ps1, elevated, in the interactive session`);
|
|
618
368
|
}
|
|
619
369
|
}
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
* behind, and StartupBoostEnabled read 1 on two guests and 0 on a third -- and BOTH were caught by a
|
|
627
|
-
* human reading a console by eye. That is not a detection mechanism.
|
|
628
|
-
*
|
|
629
|
-
* Never a FAIL. A run on slightly mismatched guests is worse than one on matched guests and far better
|
|
630
|
-
* than no run, and a diagnostic must not be the thing that takes the pool offline.
|
|
631
|
-
*/
|
|
632
|
-
function checkFleetConsistency(/** @type {any} */ probed, /** @type {number} */ configured) {
|
|
633
|
-
const guests = probed.filter((/** @type {any} */ w) => w.health)
|
|
634
|
-
.map((/** @type {any} */ w) => ({ worker: w.url, environment: w.health.environment, policy: undefined }));
|
|
370
|
+
function checkFleetConsistency(probed, configured) {
|
|
371
|
+
const guests = probed.filter((w)=>w.health).map((w)=>({
|
|
372
|
+
worker: w.url,
|
|
373
|
+
environment: w.health.environment,
|
|
374
|
+
policy: void 0
|
|
375
|
+
}));
|
|
635
376
|
const { consistent, mismatches, fields } = fleetConsistency(guests);
|
|
636
|
-
if (guests.length < 2)
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
}
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
add("fleet", true, `INCONSISTENT — ${describeMismatches(mismatches).join("; ")}`, "re-provision the odd one out so every worker reports the same "
|
|
646
|
-
+ `${mismatches.map((/** @type {any} */ m) => m.field).join(", ")}`);
|
|
647
|
-
}
|
|
648
|
-
/**
|
|
649
|
-
* The agreement sentence, with WHICH FIELDS AGREED DERIVED rather than retyped — #1997.
|
|
650
|
-
*
|
|
651
|
-
* "OF N CONFIGURED", because agreement among a SUBSET is not agreement. Unreachable guests are skipped
|
|
652
|
-
* by the caller — correctly, that check is not their business — so without the denominator "3 guests
|
|
653
|
-
* agree" reads as a whole fleet on a fleet of five, which is the examined-nothing shape one step in from
|
|
654
|
-
* zero.
|
|
655
|
-
*
|
|
656
|
-
* AND THE FIELD NAMES ARE NOT TYPED HERE. This line used to read "agree on browser, screen reader, OS
|
|
657
|
-
* and protocol": four names, by hand, beside a `MUST_MATCH` that has ten. `guidepupVersion`,
|
|
658
|
-
* `architecture`, `browserProfile`, `screenReaderSettings`, `provisionRevision` and `displayMode` were
|
|
659
|
-
* all compared and none of them was mentioned, and the remediation string repeated the same four — so
|
|
660
|
-
* the sentence had been making a POSITIVE, false claim about its own scope since the fifth field was
|
|
661
|
-
* added, and no test could notice because nothing tied the words to the list. A sentence enumerating
|
|
662
|
-
* what a machine compared is a second copy of that machine's predicate; derived, it cannot go stale.
|
|
663
|
-
*
|
|
664
|
-
* `fleet:status` says nothing about field coverage and this said something untrue about it, which is why
|
|
665
|
-
* #1997's fix has to reach both. Never a FAIL either way: a mismatched pool is worse than a matched one
|
|
666
|
-
* and far better than no pool, and a diagnostic must not be the thing that takes the fleet offline.
|
|
667
|
-
*
|
|
668
|
-
* AND THE LIST IS WHAT EVERY COMPARED GUEST REPORTED, NOT WHAT ANY ONE OF THEM DID — #2034, carrying
|
|
669
|
-
* #2019's ruling into the second of the two commands #1997 named. `fields.compared` is TRUE-IF-ANYBODY,
|
|
670
|
-
* so a field ONE guest of three reported was named inside a list introduced by the words "guests agree
|
|
671
|
-
* on", and the reporter count contradicting it sat in the same return value. Measured 2026-09-22 at
|
|
672
|
-
* #2033's head: `coverage: {"field":"displayMode","reported":1,"asked":3}` beside `3 of 3 guests agree
|
|
673
|
-
* on 10 compared field(s) (..., displayMode)`. One guest's display was read. Naming it is worse than
|
|
674
|
-
* counting it, because the naming is what #1997 added to make the sentence actionable.
|
|
675
|
-
*
|
|
676
|
-
* THREE FACTS, THREE SENTENCES, and that split is #2019's ruling rather than a style choice: `N of N`
|
|
677
|
-
* is agreement, `k of N` is *some boxes did not report it* and sends a reader to the BOXES, `0 of N` is
|
|
678
|
-
* *nobody could be asked* and sends them to the FIELD. Collapsing the middle one into either of the
|
|
679
|
-
* outer two is the defect this fixes in one direction and #1997's in the other.
|
|
680
|
-
*
|
|
681
|
-
* DERIVED FROM `coverage`, AND NOT FROM `compared`/`unchecked` — the same call `fleet:status` makes
|
|
682
|
-
* (`fieldCoverageGap`, #2019). The two lists are that same measurement thresholded at "anybody", so
|
|
683
|
-
* reading the whole case off one and the partial case off the other gives one fact two sources that can
|
|
684
|
-
* disagree. The lists stay in the return value, where a caller greps them for the remedy.
|
|
685
|
-
*
|
|
686
|
-
* @param {{ agreeing: number, configured: number,
|
|
687
|
-
* fields: { compared: string[], unchecked: string[],
|
|
688
|
-
* coverage?: { field: string, reported: number, asked: number }[] } }} input
|
|
689
|
-
* @returns {string}
|
|
690
|
-
*/
|
|
691
|
-
export function fleetAgreementLine({ agreeing, configured, fields }) {
|
|
377
|
+
if (guests.length < 2) return;
|
|
378
|
+
if (consistent) return void add("fleet", true, fleetAgreementLine({
|
|
379
|
+
agreeing: guests.length,
|
|
380
|
+
configured,
|
|
381
|
+
fields
|
|
382
|
+
}));
|
|
383
|
+
add("fleet", true, `INCONSISTENT — ${describeMismatches(mismatches).join("; ")}`, `re-provision the odd one out so every worker reports the same ${mismatches.map((m)=>m.field).join(", ")}`);
|
|
384
|
+
}
|
|
385
|
+
function fleetAgreementLine({ agreeing, configured, fields }) {
|
|
692
386
|
const rest = agreeing < configured ? " — the rest could not be asked" : "";
|
|
693
387
|
const coverage = fields.coverage;
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
const gaps = [partialClause(coverage), uncheckedClause(coverage)].filter((clause) => clause !== "");
|
|
705
|
-
return `${agreeing} of ${configured} guests agree on ${whole.length} of ${coverage.length} `
|
|
706
|
-
+ `field(s)${named}${rest}${gaps.join("")}`;
|
|
707
|
-
}
|
|
708
|
-
/** "it"/"them" for a clause that names a list, so one gap does not read as a plural. */
|
|
709
|
-
const itOrThem = (/** @type {number} */ count) => (count === 1 ? "it" : "them");
|
|
710
|
-
/**
|
|
711
|
-
* #2034's clause: the fields SOME compared guests reported and others did not, each with its `k of N`.
|
|
712
|
-
*
|
|
713
|
-
* NAMED WITH ITS COUNT, never counted — the same choice `fleet:status`' `partialClause` makes, and for
|
|
714
|
-
* the same reason #1997 gave for naming the zero case. "1 field was partly reported" sends a reader back
|
|
715
|
-
* to `doctor`; "displayMode (1 of 3 reported it)" tells them two boxes owe an answer, which is the
|
|
716
|
-
* finding — either the converge did not reach them, or their probe for the field failed (#1953).
|
|
717
|
-
*
|
|
718
|
-
* @param {{ field: string, reported: number, asked: number }[]} coverage
|
|
719
|
-
* @returns {string} empty when no field is partly reported
|
|
720
|
-
*/
|
|
388
|
+
if (void 0 === coverage) return `${agreeing} of ${configured} guests agree, and no field coverage was supplied, so WHICH fields were compared, and by HOW MANY guests, was never asked${rest}`;
|
|
389
|
+
const whole = coverage.filter(({ reported, asked })=>reported === asked).map(({ field })=>field);
|
|
390
|
+
const named = 0 === whole.length ? "" : ` (${whole.join(", ")})`;
|
|
391
|
+
const gaps = [
|
|
392
|
+
partialClause(coverage),
|
|
393
|
+
uncheckedClause(coverage)
|
|
394
|
+
].filter((clause)=>"" !== clause);
|
|
395
|
+
return `${agreeing} of ${configured} guests agree on ${whole.length} of ${coverage.length} field(s)${named}${rest}${gaps.join("")}`;
|
|
396
|
+
}
|
|
397
|
+
const itOrThem = (count)=>1 === count ? "it" : "them";
|
|
721
398
|
function partialClause(coverage) {
|
|
722
|
-
const partial = coverage.filter(({ reported, asked })
|
|
723
|
-
if (partial.length
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
+ `${itOrThem(partial.length)}: ${named.join(", ")}`;
|
|
728
|
-
}
|
|
729
|
-
/**
|
|
730
|
-
* #1997's clause: the fields asked of everybody and answered by nobody.
|
|
731
|
-
*
|
|
732
|
-
* NAMED, not counted: "one field was not compared" sends a reader back to doctor, and "displayMode was
|
|
733
|
-
* not compared" sends them to the deploy that would report it.
|
|
734
|
-
*
|
|
735
|
-
* @param {{ field: string, reported: number, asked: number }[]} coverage
|
|
736
|
-
* @returns {string} empty when every asked field drew at least one value
|
|
737
|
-
*/
|
|
399
|
+
const partial = coverage.filter(({ reported, asked })=>reported > 0 && reported < asked);
|
|
400
|
+
if (0 === partial.length) return "";
|
|
401
|
+
const named = partial.map(({ field, reported, asked })=>`${field} (${reported} of ${asked} reported it)`);
|
|
402
|
+
return `; reported by only SOME of the compared guests, so agreement says nothing about ${itOrThem(partial.length)}: ${named.join(", ")}`;
|
|
403
|
+
}
|
|
738
404
|
function uncheckedClause(coverage) {
|
|
739
|
-
const unchecked = coverage.filter(({ reported })
|
|
740
|
-
if (unchecked.length
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
}
|
|
745
|
-
// Can this host actually hold the pool it has registered?
|
|
746
|
-
//
|
|
747
|
-
// Not a fault, and never a FAIL: a capped pool runs fine, just narrower. It is reported because the
|
|
748
|
-
// alternative is invisible. Three guests on this 36 GB Mac made every capture 1.6x slower than one
|
|
749
|
-
// and produced mute-NVDA failures, and from outside that reads as "the workers are degrading" rather
|
|
750
|
-
// than "the host is out of memory" — which is exactly how it was misread for a day.
|
|
751
|
-
function checkHostCapacity(/** @type {any} */ pool) {
|
|
405
|
+
const unchecked = coverage.filter(({ reported })=>0 === reported).map(({ field })=>field);
|
|
406
|
+
if (0 === unchecked.length) return "";
|
|
407
|
+
return `; NOT compared on any guest, so agreement says nothing about ${itOrThem(unchecked.length)}: ` + unchecked.join(", ");
|
|
408
|
+
}
|
|
409
|
+
function checkHostCapacity(pool) {
|
|
752
410
|
const availableMb = availableHostMemoryMb();
|
|
753
|
-
if (
|
|
754
|
-
|
|
755
|
-
// Guests already up have paid for their memory and are not counted in `availableMb`, so they are
|
|
756
|
-
// added back — otherwise a running worker makes the host look smaller than it is.
|
|
757
|
-
const running = pool.filter((/** @type {any} */ vm) => vm.state === "started").length;
|
|
411
|
+
if (null === availableMb || !pool.length) return;
|
|
412
|
+
const running = pool.filter((vm)=>"started" === vm.state).length;
|
|
758
413
|
const poolSize = pool.length;
|
|
759
|
-
const limit = workersHostCanRun({
|
|
414
|
+
const limit = workersHostCanRun({
|
|
415
|
+
availableMb,
|
|
416
|
+
alreadyRunning: running
|
|
417
|
+
});
|
|
760
418
|
const detail = `~${availableMb} MB available — room for ${Math.min(limit, poolSize)} of ${poolSize} worker(s)`;
|
|
761
|
-
add("host memory", true, limit >= poolSize
|
|
762
|
-
? detail
|
|
763
|
-
: `${detail}; the rest stay stopped so the run does not swap (override: A11Y_MAX_WORKERS)`);
|
|
419
|
+
add("host memory", true, limit >= poolSize ? detail : `${detail}; the rest stay stopped so the run does not swap (override: A11Y_MAX_WORKERS)`);
|
|
764
420
|
}
|
|
765
|
-
// Dataset capture needs the pages served, and the guest must be able to reach them — the
|
|
766
|
-
// host's localhost is not reachable from inside the VM. The capture command leases the page
|
|
767
|
-
// server for the run, so an idle host with no listener on this port is ready, not broken.
|
|
768
421
|
async function checkDatasetPages() {
|
|
769
422
|
const manifestPath = resolve(DATASET, "manifest.json");
|
|
770
|
-
if (!
|
|
771
|
-
return add("dataset", false, "no manifest — the dataset has not been generated", "pnpm run training:generate");
|
|
772
|
-
}
|
|
773
|
-
// Ask for a REAL page, not `/`.
|
|
774
|
-
//
|
|
775
|
-
// "Something answers on :5050" is not the same as "our pages are being served", and the difference
|
|
776
|
-
// has already cost a dataset: a stray server on that port reported "Capture complete: 3/3 cases"
|
|
777
|
-
// while every transcript read "Error code: 404". A leftover `npx serve` from another directory
|
|
778
|
-
// answers the root happily and 404s every case. Four of them were running on this host today, which
|
|
779
|
-
// is how likely that is.
|
|
423
|
+
if (!external_node_fs_existsSync(manifestPath)) return add("dataset", false, "no manifest — the dataset has not been generated", "pnpm run training:generate");
|
|
780
424
|
const sample = JSON.parse(readFileSync(manifestPath, "utf8")).cases?.[0]?.id;
|
|
781
425
|
const probe = sample ? `${sample}/good.html` : "";
|
|
782
426
|
try {
|
|
783
|
-
const response = await fetch(`http://localhost:${PAGES_PORT}/${probe}`, {
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
}
|
|
427
|
+
const response = await fetch(`http://localhost:${PAGES_PORT}/${probe}`, {
|
|
428
|
+
signal: AbortSignal.timeout(PROBE_TIMEOUT_MS)
|
|
429
|
+
});
|
|
430
|
+
if (!response.ok) return add("pages", false, `:${PAGES_PORT} answers but returns HTTP ${response.status} for ${probe} — wrong directory`, "something else holds the port. Stop it; training:capture will lease the dataset server automatically");
|
|
787
431
|
add("pages", true, `serving the dataset on :${PAGES_PORT} (verified ${probe || "/"})`);
|
|
788
|
-
}
|
|
789
|
-
catch {
|
|
432
|
+
} catch {
|
|
790
433
|
add("pages", true, `nothing serving on :${PAGES_PORT} — training:capture leases it automatically`);
|
|
791
434
|
}
|
|
792
435
|
}
|
|
793
|
-
// A run left mid-flight is the difference between "start" and "--resume", and getting it
|
|
794
|
-
// wrong either re-captures for hours or silently skips work.
|
|
795
436
|
function checkRunState() {
|
|
796
437
|
const progress = resolve(DATASET, "capture-progress.json");
|
|
797
|
-
if (!
|
|
798
|
-
return add("run", true, "no capture run recorded");
|
|
438
|
+
if (!external_node_fs_existsSync(progress)) return add("run", true, "no capture run recorded");
|
|
799
439
|
const p = JSON.parse(readFileSync(progress, "utf8"));
|
|
800
|
-
if (!p.startedAt)
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
// --- report ---------------------------------------------------------------
|
|
809
|
-
// The single most useful line for anything automated: what to run next. A list of green ticks
|
|
810
|
-
// still leaves a caller deciding, and deciding is where they go wrong.
|
|
811
|
-
/**
|
|
812
|
-
* #1059: A COMMAND, OR `null`. Never a sentence.
|
|
813
|
-
*
|
|
814
|
-
* `--json`'s `next_command` is the field CLAUDE.md tells an agent to read and obey, so anything in it that
|
|
815
|
-
* is not runnable makes an automated reader loop -- which is exactly what happened when a failing `worker`
|
|
816
|
-
* check put *"unlock the Mac if it is locked, then re-run …"* here.
|
|
817
|
-
*
|
|
818
|
-
* **`null` and an unrunnable string are different reports**, and a JSON consumer can act on the first: it
|
|
819
|
-
* means read the checks. `contention` has no command because waiting is not one, and it says so with
|
|
820
|
-
* `null` and a `note` rather than with an imperative nobody can execute.
|
|
821
|
-
* THE PARAMETER TYPE IS WHAT THIS FUNCTION READS, not the whole check shape: `ok` and `fix`, and nothing
|
|
822
|
-
* else. A test injecting a two-field object is stating exactly the inputs the verdict depends on, and a
|
|
823
|
-
* wider type would have made it carry an `id` and a `detail` the answer cannot possibly turn on.
|
|
824
|
-
* @param {{ok: boolean, fix: string|null}[]} [checkList]
|
|
825
|
-
* @returns {string|null}
|
|
826
|
-
*/
|
|
827
|
-
export function nextCommand(checkList = checks) {
|
|
828
|
-
const broken = checkList.find((c) => !c.ok);
|
|
829
|
-
if (!broken)
|
|
830
|
-
return "pnpm run training:capture";
|
|
440
|
+
if (!p.startedAt) return add("run", true, "no capture run recorded");
|
|
441
|
+
if (!p.finishedAt) return add("run", false, `a run is UNFINISHED (started ${p.startedAt})`, "pnpm run training:wait, or pnpm run training:capture -- --resume --no-cache");
|
|
442
|
+
const failed = Object.values(p.cases ?? {}).filter((c)=>"failed" === c.status).length;
|
|
443
|
+
add("run", 0 === failed, `last run ${p.outcome ?? "finished"}`, failed ? "pnpm run training:capture -- --resume --no-cache" : null);
|
|
444
|
+
}
|
|
445
|
+
function nextCommand(checkList = checks) {
|
|
446
|
+
const broken = checkList.find((c)=>!c.ok);
|
|
447
|
+
if (!broken) return "pnpm run training:capture";
|
|
831
448
|
return broken.fix ?? null;
|
|
832
449
|
}
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
*
|
|
836
|
-
* #1059: the field carried *"unlock the Mac if it is locked, then re-run …"*, and CLAUDE.md tells an agent
|
|
837
|
-
* to read it and do that. A shape check is the only thing that can tell a command from an imperative
|
|
838
|
-
* sentence without running it: **a command begins with an executable token** — a pnpm/npm/node/git invocation,
|
|
839
|
-
* a path, or a `VAR=value` prefix — **and an English sentence begins with a verb or an article.**
|
|
840
|
-
* @param {string|null} line
|
|
841
|
-
* @returns {boolean}
|
|
842
|
-
*/
|
|
843
|
-
export function isRunnableCommand(line) {
|
|
844
|
-
if (line === null)
|
|
845
|
-
return false; // absent is not unrunnable; the caller distinguishes them
|
|
450
|
+
function isRunnableCommand(line) {
|
|
451
|
+
if (null === line) return false;
|
|
846
452
|
const first = line.trim().split(/\s+/)[0] ?? "";
|
|
847
453
|
return /^(?:[A-Z][A-Z0-9_]*=\S*|pnpm|npm|npx|node|git|\.?\.?\/\S+|[a-z0-9_-]+\.(?:sh|mjs|js|ts|py))$/.test(first);
|
|
848
454
|
}
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
* @param {{name: string, ok: boolean}[]} checkList
|
|
865
|
-
* @returns {boolean}
|
|
866
|
-
*/
|
|
867
|
-
export function readyFrom(checkList) {
|
|
868
|
-
return checkList.every((c) => c.ok || GATES[c.name] === false);
|
|
869
|
-
}
|
|
870
|
-
/** Every DECLARED check name, gating or not -- so a test can compare the two sets without a literal. */
|
|
871
|
-
export const allChecks = () => Object.keys(GATES);
|
|
872
|
-
/** Which checks decide `ready`, for a test that must not retype the list. */
|
|
873
|
-
export const gatingChecks = () => Object.entries(GATES).filter(([, gates]) => gates).map(([name]) => name);
|
|
874
|
-
/** The checks `doctor` runs, in order. Injectable so a throwing one can be driven without breaking a tree. */
|
|
875
|
-
const DEFAULT_STEPS = [checkPrimaryCheckoutMark, checkControlPlaneIsolation, checkCrossPackageDist,
|
|
876
|
-
checkJudge, checkWorker, checkDatasetPages, checkRunState];
|
|
877
|
-
/**
|
|
878
|
-
* #1082: A `--json` RUN THAT CANNOT PRODUCE JSON STILL PRODUCES JSON.
|
|
879
|
-
*
|
|
880
|
-
* Measured on `1e74e3d0`: a check threw, `doctor --json` exited 1 with **zero bytes on stdout** and 1,155
|
|
881
|
-
* bytes of stack on stderr — where a `--json` consumer never looks. **"Could not ask" and "no output" are
|
|
882
|
-
* different for a caller**, and only the first is actionable; the second is indistinguishable from a
|
|
883
|
-
* command that was never run.
|
|
884
|
-
*
|
|
885
|
-
* `2>&1` IS NOT THE FIX HERE, which is what makes this different from #1068's watch job. That job's stdout
|
|
886
|
-
* is prose, so merging stderr in was free. This stdout is a PARSED format, and redirecting into it
|
|
887
|
-
* produces invalid JSON — **worse than nothing, because a consumer that parses gets a syntax error rather
|
|
888
|
-
* than a document.** The fix has to be in the tool.
|
|
889
|
-
*
|
|
890
|
-
* `checks` is present and EMPTY rather than absent OR PARTIAL — and the partial list is the one actually
|
|
891
|
-
* worth refusing. A consumer reading `.checks[]` off an absent key crashes; off a partial one it reads a
|
|
892
|
-
* list that looks exactly like a complete verdict, with **no way to tell a check that is missing because
|
|
893
|
-
* it passed from one that is missing because the run died under it.** Empty says "no verdict" and cannot
|
|
894
|
-
* be mistaken for a short one.
|
|
895
|
-
* @param {unknown} error
|
|
896
|
-
* @returns {{ready: false, error: string, checks: never[]}}
|
|
897
|
-
*/
|
|
898
|
-
export function errorDocument(error) {
|
|
455
|
+
function readyFrom(checkList) {
|
|
456
|
+
return checkList.every((c)=>c.ok || false === GATES[c.name]);
|
|
457
|
+
}
|
|
458
|
+
const allChecks = ()=>Object.keys(GATES);
|
|
459
|
+
const gatingChecks = ()=>Object.entries(GATES).filter(([, gates])=>gates).map(([name])=>name);
|
|
460
|
+
const DEFAULT_STEPS = [
|
|
461
|
+
checkPrimaryCheckoutMark,
|
|
462
|
+
checkControlPlaneIsolation,
|
|
463
|
+
checkCrossPackageDist,
|
|
464
|
+
checkJudge,
|
|
465
|
+
checkWorker,
|
|
466
|
+
checkDatasetPages,
|
|
467
|
+
checkRunState
|
|
468
|
+
];
|
|
469
|
+
function errorDocument(error) {
|
|
899
470
|
const message = error instanceof Error ? error.message : String(error);
|
|
900
|
-
return {
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
* @returns {Promise<number>}
|
|
908
|
-
*/
|
|
909
|
-
export async function doctorRun(deps = {}) {
|
|
471
|
+
return {
|
|
472
|
+
ready: false,
|
|
473
|
+
error: message,
|
|
474
|
+
checks: []
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
async function doctorRun(deps = {}) {
|
|
910
478
|
const { steps = DEFAULT_STEPS, json = JSON_OUT, out = console.log, err = console.error } = deps;
|
|
911
479
|
try {
|
|
912
|
-
for (const step of steps)
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
// NOTHING BUT JSON REACHES STDOUT IN `--json` MODE. A `console.log` here would corrupt the document
|
|
917
|
-
// for every consumer, which is the failure this exists to prevent rather than to introduce.
|
|
918
|
-
if (json)
|
|
919
|
-
out(JSON.stringify(errorDocument(error), null, 2));
|
|
920
|
-
// The HUMAN path keeps the stack. Only the parsed format has to give it up, and it gives it up for a
|
|
921
|
-
// document a consumer can read -- not to make the failure quieter.
|
|
922
|
-
else
|
|
923
|
-
err(error instanceof Error && error.stack ? error.stack : `doctor: ${errorDocument(error).error}`);
|
|
480
|
+
for (const step of steps)await step();
|
|
481
|
+
} catch (error) {
|
|
482
|
+
if (json) out(JSON.stringify(errorDocument(error), null, 2));
|
|
483
|
+
else err(error instanceof Error && error.stack ? error.stack : `doctor: ${errorDocument(error).error}`);
|
|
924
484
|
return 1;
|
|
925
485
|
}
|
|
926
|
-
return renderDoctor({
|
|
486
|
+
return renderDoctor({
|
|
487
|
+
json,
|
|
488
|
+
out
|
|
489
|
+
});
|
|
927
490
|
}
|
|
928
|
-
/**
|
|
929
|
-
* @param {{ json: boolean, out: (line: string) => void }} deps
|
|
930
|
-
* @returns {number}
|
|
931
|
-
*/
|
|
932
491
|
function renderDoctor({ json, out }) {
|
|
933
492
|
const ready = readyFrom(checks);
|
|
934
|
-
if (json) {
|
|
935
|
-
|
|
936
|
-
|
|
493
|
+
if (json) out(JSON.stringify({
|
|
494
|
+
ready,
|
|
495
|
+
next_command: nextCommand(),
|
|
496
|
+
checks: checks
|
|
497
|
+
}, null, 2));
|
|
937
498
|
else {
|
|
938
|
-
for (const c of checks)
|
|
499
|
+
for (const c of checks){
|
|
939
500
|
out(`${c.advisory ? "DEBT" : c.ok ? "OK " : "FAIL"} ${c.name.padEnd(11)} ${c.detail}`);
|
|
940
|
-
if ((!c.ok || c.advisory) && c.fix)
|
|
941
|
-
|
|
942
|
-
// #1059: the advice a shell cannot run is PRINTED, just not in the field something executes.
|
|
943
|
-
if ((!c.ok || c.advisory) && c.note)
|
|
944
|
-
out(` note: ${c.note}`);
|
|
501
|
+
if ((!c.ok || c.advisory) && c.fix) out(` fix: ${c.fix}`);
|
|
502
|
+
if ((!c.ok || c.advisory) && c.note) out(` note: ${c.note}`);
|
|
945
503
|
}
|
|
946
504
|
out(`\n${ready ? "READY" : "NOT READY — see the fixes above"}`);
|
|
947
505
|
const next = nextCommand();
|
|
948
|
-
|
|
949
|
-
out(next === null ? "next: no single command — read the fixes and notes above" : `next: ${next}`);
|
|
506
|
+
out(null === next ? "next: no single command — read the fixes and notes above" : `next: ${next}`);
|
|
950
507
|
}
|
|
951
508
|
return ready ? 0 : 1;
|
|
952
509
|
}
|
|
953
510
|
async function main() {
|
|
954
511
|
process.exit(await doctorRun());
|
|
955
512
|
}
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
// and this guard silently read false — the tool loaded, did nothing, and exited 0. `/var` and `/tmp` are
|
|
959
|
-
// themselves symlinks on macOS, so this fired every time. Same defect, same fix, as `cli.ts`'s `isProgram`.
|
|
960
|
-
if (import.meta.url === pathToFileURL(process.argv[1] ? realpathSync(process.argv[1]) : "").href)
|
|
961
|
-
await main();
|
|
962
|
-
//# sourceMappingURL=doctor.mjs.map
|
|
513
|
+
if (import.meta.url === pathToFileURL(process.argv[1] ? external_node_fs_realpathSync(process.argv[1]) : "").href) await main();
|
|
514
|
+
export { addCheck, allChecks, checkoutRootFor, doctorRun, errorDocument, fleetAgreementLine, gatingChecks, isRunnableCommand, nextCommand, readyFrom, resolvesToThisCheckout, tscProjectUpToDate, workerControlFix };
|