@a11ign/screenreader-fleet 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capture-client.d.mts +0 -1
- package/dist/capture-client.mjs +144 -306
- package/dist/check-worker-code.d.mts +0 -1
- package/dist/check-worker-code.mjs +42 -141
- package/dist/cli-flags.d.mts +0 -1
- package/dist/cli-flags.mjs +33 -179
- package/dist/code-drift.d.mts +0 -1
- package/dist/command-line-census.d.mts +0 -1
- package/dist/compare-workers.d.mts +0 -1
- package/dist/compare-workers.mjs +383 -255
- package/dist/control-plane-isolation.d.mts +0 -1
- package/dist/deploy-worker.d.mts +0 -1
- package/dist/deploy-worker.mjs +101 -242
- package/dist/doctor.d.mts +0 -1
- package/dist/doctor.mjs +361 -809
- package/dist/fleet-consistency.d.mts +0 -1
- package/dist/fleet-consistency.mjs +155 -379
- package/dist/fleet-env.d.mts +0 -1
- package/dist/fleet-env.mjs +148 -438
- package/dist/fleet-scripts.d.mts +0 -1
- package/dist/git-safe-env.d.mts +0 -1
- package/dist/guest-run.d.mts +0 -1
- package/dist/host-address.d.mts +0 -1
- package/dist/host-address.mjs +19 -90
- package/dist/host-capacity.d.mts +0 -1
- package/dist/host-capacity.mjs +22 -136
- package/dist/host-metrics.d.mts +0 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.mjs +231 -0
- package/dist/local-vm.d.ts +0 -1
- package/dist/measure-guard.d.mts +0 -1
- package/dist/normalise-fleet.d.mts +0 -1
- package/dist/npm-cli-executable.d.mts +0 -1
- package/dist/probe-outcome.d.mts +0 -1
- package/dist/probe-outcome.mjs +50 -96
- package/dist/protocol-guard.d.mts +0 -1
- package/dist/source-walk.d.mts +0 -1
- package/dist/src_fleet-scripts_mjs.mjs +16 -0
- package/dist/src_git-safe-env_mjs.mjs +9 -0
- package/dist/src_utm-deprecated_mjs.mjs +4 -0
- package/dist/transient-fault.d.mts +0 -1
- package/dist/transient-fault.mjs +21 -81
- package/dist/utm-deprecated.d.mts +0 -1
- package/dist/worker-code-check.d.mts +0 -1
- package/dist/worker-code-check.mjs +127 -76
- package/dist/worker-health.d.mts +0 -1
- package/dist/worker-health.mjs +16 -61
- package/dist/worker-http.d.mts +0 -1
- package/dist/worker-http.mjs +42 -234
- package/dist/worker-stats.d.mts +0 -1
- package/package.json +12 -5
- package/dist/capture-client.d.mts.map +0 -1
- package/dist/capture-client.mjs.map +0 -1
- package/dist/check-worker-code.d.mts.map +0 -1
- package/dist/check-worker-code.mjs.map +0 -1
- package/dist/cli-flags.d.mts.map +0 -1
- package/dist/cli-flags.mjs.map +0 -1
- package/dist/code-drift.d.mts.map +0 -1
- package/dist/code-drift.mjs +0 -284
- package/dist/code-drift.mjs.map +0 -1
- package/dist/command-line-census.d.mts.map +0 -1
- package/dist/command-line-census.mjs +0 -96
- package/dist/command-line-census.mjs.map +0 -1
- package/dist/compare-workers.d.mts.map +0 -1
- package/dist/compare-workers.mjs.map +0 -1
- package/dist/control-plane-isolation.d.mts.map +0 -1
- package/dist/control-plane-isolation.mjs +0 -67
- package/dist/control-plane-isolation.mjs.map +0 -1
- package/dist/deploy-worker.d.mts.map +0 -1
- package/dist/deploy-worker.mjs.map +0 -1
- package/dist/doctor.d.mts.map +0 -1
- package/dist/doctor.mjs.map +0 -1
- package/dist/fleet-consistency.d.mts.map +0 -1
- package/dist/fleet-consistency.mjs.map +0 -1
- package/dist/fleet-env.d.mts.map +0 -1
- package/dist/fleet-env.mjs.map +0 -1
- package/dist/fleet-scripts.d.mts.map +0 -1
- package/dist/fleet-scripts.mjs +0 -41
- package/dist/fleet-scripts.mjs.map +0 -1
- package/dist/git-safe-env.d.mts.map +0 -1
- package/dist/git-safe-env.mjs +0 -44
- package/dist/git-safe-env.mjs.map +0 -1
- package/dist/guest-run.d.mts.map +0 -1
- package/dist/guest-run.mjs +0 -164
- package/dist/guest-run.mjs.map +0 -1
- package/dist/host-address.d.mts.map +0 -1
- package/dist/host-address.mjs.map +0 -1
- package/dist/host-capacity.d.mts.map +0 -1
- package/dist/host-capacity.mjs.map +0 -1
- package/dist/host-metrics.d.mts.map +0 -1
- package/dist/host-metrics.mjs +0 -201
- package/dist/host-metrics.mjs.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -25
- package/dist/index.js.map +0 -1
- package/dist/local-vm.d.ts.map +0 -1
- package/dist/local-vm.js +0 -360
- package/dist/local-vm.js.map +0 -1
- package/dist/measure-guard.d.mts.map +0 -1
- package/dist/measure-guard.mjs +0 -73
- package/dist/measure-guard.mjs.map +0 -1
- package/dist/normalise-fleet.d.mts.map +0 -1
- package/dist/normalise-fleet.mjs +0 -76
- package/dist/normalise-fleet.mjs.map +0 -1
- package/dist/npm-cli-executable.d.mts.map +0 -1
- package/dist/npm-cli-executable.mjs +0 -159
- package/dist/npm-cli-executable.mjs.map +0 -1
- package/dist/probe-outcome.d.mts.map +0 -1
- package/dist/probe-outcome.mjs.map +0 -1
- package/dist/protocol-guard.d.mts.map +0 -1
- package/dist/protocol-guard.mjs +0 -121
- package/dist/protocol-guard.mjs.map +0 -1
- package/dist/source-walk.d.mts.map +0 -1
- package/dist/source-walk.mjs +0 -56
- package/dist/source-walk.mjs.map +0 -1
- package/dist/transient-fault.d.mts.map +0 -1
- package/dist/transient-fault.mjs.map +0 -1
- package/dist/utm-deprecated.d.mts.map +0 -1
- package/dist/utm-deprecated.mjs +0 -23
- package/dist/utm-deprecated.mjs.map +0 -1
- package/dist/worker-code-check.d.mts.map +0 -1
- package/dist/worker-code-check.mjs.map +0 -1
- package/dist/worker-health.d.mts.map +0 -1
- package/dist/worker-health.mjs.map +0 -1
- package/dist/worker-http.d.mts.map +0 -1
- package/dist/worker-http.mjs.map +0 -1
- package/dist/worker-stats.d.mts.map +0 -1
- package/dist/worker-stats.mjs +0 -143
- package/dist/worker-stats.mjs.map +0 -1
package/dist/fleet-scripts.d.mts
CHANGED
package/dist/git-safe-env.d.mts
CHANGED
package/dist/guest-run.d.mts
CHANGED
|
@@ -23,4 +23,3 @@ export function wrapScript(body: string, outputFile: string): string;
|
|
|
23
23
|
export function isComplete(output: string | null): boolean;
|
|
24
24
|
/** Written by the wrapper as its last act. Absent means still running, or it died. */
|
|
25
25
|
export const DONE_SENTINEL: "---GUEST-RUN-DONE---";
|
|
26
|
-
//# sourceMappingURL=guest-run.d.mts.map
|
package/dist/host-address.d.mts
CHANGED
package/dist/host-address.mjs
CHANGED
|
@@ -1,105 +1,34 @@
|
|
|
1
|
-
// @ts-check
|
|
2
|
-
/**
|
|
3
|
-
* This host's address as a WORKER sees it — one definition, because there were five.
|
|
4
|
-
*
|
|
5
|
-
* A worker cannot reach the host's `localhost`, so anything the host serves to a worker (the dataset
|
|
6
|
-
* page server, above all) must be addressed by the host's LAN IP. `local-vm.ts` worked this out
|
|
7
|
-
* correctly and said why:
|
|
8
|
-
*
|
|
9
|
-
* > Derived by matching interfaces against the guest's address rather than assuming the usual
|
|
10
|
-
* > `x.y.z.1`, because guessing an address that silently does not answer is worse than reporting
|
|
11
|
-
* > that we could not find one.
|
|
12
|
-
*
|
|
13
|
-
* Four harnesses then reimplemented it inline as exactly the assumption that comment rejects:
|
|
14
|
-
*
|
|
15
|
-
* new URL(worker).hostname.replace(/\.\d+$/, ".1")
|
|
16
|
-
*
|
|
17
|
-
* That is right for UTM, where the Mac really is the `.1` gateway of the shared network, and wrong
|
|
18
|
-
* for every other network. Measured: against a bare-metal worker on the fleet's own LAN it produced
|
|
19
|
-
* the ROUTER's address instead — so `evidence:check` refused to run because the pages
|
|
20
|
-
* were not being served. On a fleet of bare-metal boxes it fails every time.
|
|
21
|
-
*
|
|
22
|
-
* The failure mode is why this matters more than tidiness. `guestReachableUrl`'s own note records
|
|
23
|
-
* what happened last time this was wrong: the guest fetched its own localhost, Edge showed
|
|
24
|
-
* "localhost refused to connect", the title check rejected the capture, and three attempts were
|
|
25
|
-
* burned per page before giving up — a page-server problem wearing a capture problem's costume.
|
|
26
|
-
*
|
|
27
|
-
* Plain `.mjs` on purpose: `capture-check.mjs` runs under bare node on the guest (see
|
|
28
|
-
* `run-capture-check.cmd`) and cannot import TypeScript.
|
|
29
|
-
*/
|
|
30
1
|
import { networkInterfaces } from "node:os";
|
|
31
2
|
const IPV4_OCTETS = 4;
|
|
32
|
-
|
|
33
|
-
/** @param {string} ip */
|
|
34
|
-
export function ipv4ToInt(ip) {
|
|
3
|
+
function ipv4ToInt(ip) {
|
|
35
4
|
const parts = String(ip).split(".").map(Number);
|
|
36
|
-
if (parts.length !== IPV4_OCTETS || parts.some((n)
|
|
37
|
-
|
|
38
|
-
return parts.reduce((acc, octet) => acc * 256 + octet, 0);
|
|
5
|
+
if (parts.length !== IPV4_OCTETS || parts.some((n)=>!Number.isInteger(n) || n < 0 || n > 255)) return null;
|
|
6
|
+
return parts.reduce((acc, octet)=>256 * acc + octet, 0);
|
|
39
7
|
}
|
|
40
|
-
|
|
41
|
-
* This host's address on the same subnet as `guestIp`, or undefined if we have no interface onto it.
|
|
42
|
-
*
|
|
43
|
-
* Undefined rather than a guess: a caller that is told "we do not know" can ask to be told, whereas
|
|
44
|
-
* a caller handed a plausible-but-dead address discovers it as a timeout somewhere else entirely.
|
|
45
|
-
*
|
|
46
|
-
* @param {string} guestIp
|
|
47
|
-
* @returns {string | undefined}
|
|
48
|
-
*/
|
|
49
|
-
export function hostAddressFor(guestIp) {
|
|
8
|
+
function hostAddressFor(guestIp) {
|
|
50
9
|
const guest = ipv4ToInt(guestIp);
|
|
51
|
-
if (
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
const mask = ipv4ToInt(a.netmask);
|
|
59
|
-
if (host === null || mask === null)
|
|
60
|
-
continue;
|
|
61
|
-
// Both sides go through the same coercion, so addresses above 2^31 comparing as negative
|
|
62
|
-
// is harmless here.
|
|
63
|
-
if ((host & mask) === (guest & mask))
|
|
64
|
-
return a.address;
|
|
10
|
+
if (null === guest) return;
|
|
11
|
+
for (const addresses of Object.values(networkInterfaces()))for (const a of addresses ?? []){
|
|
12
|
+
if ("IPv4" !== a.family || a.internal) continue;
|
|
13
|
+
const host = ipv4ToInt(a.address);
|
|
14
|
+
const mask = ipv4ToInt(a.netmask);
|
|
15
|
+
if (null !== host && null !== mask) {
|
|
16
|
+
if ((host & mask) === (guest & mask)) return a.address;
|
|
65
17
|
}
|
|
66
18
|
}
|
|
67
|
-
return undefined;
|
|
68
19
|
}
|
|
69
|
-
|
|
70
|
-
* This host's address as seen from a worker at `workerUrl`, when the two share a subnet.
|
|
71
|
-
*
|
|
72
|
-
* @param {string} workerUrl
|
|
73
|
-
* @returns {string | undefined}
|
|
74
|
-
*/
|
|
75
|
-
export function hostAddressForWorker(workerUrl) {
|
|
20
|
+
function hostAddressForWorker(workerUrl) {
|
|
76
21
|
try {
|
|
77
22
|
const { hostname } = new URL(workerUrl);
|
|
78
|
-
return ipv4ToInt(hostname)
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
return undefined;
|
|
23
|
+
return null === ipv4ToInt(hostname) ? void 0 : hostAddressFor(hostname);
|
|
24
|
+
} catch {
|
|
25
|
+
return;
|
|
82
26
|
}
|
|
83
27
|
}
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
*
|
|
87
|
-
* Throws rather than returning something unusable. The four call sites all feed this straight into a
|
|
88
|
-
* capture request, and a wrong page URL does not present as a page-server fault — it presents as
|
|
89
|
-
* captures that read an error page, which is evidence rot rather than an outage.
|
|
90
|
-
*
|
|
91
|
-
* @param {string} workerUrl
|
|
92
|
-
* @param {number | string} [port]
|
|
93
|
-
* @returns {string}
|
|
94
|
-
*/
|
|
95
|
-
export function hostPagesBase(workerUrl, port = process.env.DATASET_PAGES_PORT || 5050) {
|
|
96
|
-
if (process.env.DATASET_BASE_URL)
|
|
97
|
-
return process.env.DATASET_BASE_URL.replace(/\/$/, "");
|
|
28
|
+
function hostPagesBase(workerUrl, port = process.env.DATASET_PAGES_PORT || 5050) {
|
|
29
|
+
if (process.env.DATASET_BASE_URL) return process.env.DATASET_BASE_URL.replace(/\/$/, "");
|
|
98
30
|
const address = hostAddressForWorker(workerUrl);
|
|
99
|
-
if (!address) {
|
|
100
|
-
throw new Error(`Cannot work out this host's address as seen from ${workerUrl}: no local interface shares its subnet. `
|
|
101
|
-
+ "Set DATASET_BASE_URL to the URL the worker should fetch dataset pages from.");
|
|
102
|
-
}
|
|
31
|
+
if (!address) throw new Error(`Cannot work out this host's address as seen from ${workerUrl}: no local interface shares its subnet. Set DATASET_BASE_URL to the URL the worker should fetch dataset pages from.`);
|
|
103
32
|
return `http://${address}:${port}`;
|
|
104
33
|
}
|
|
105
|
-
|
|
34
|
+
export { hostAddressFor, hostAddressForWorker, hostPagesBase, ipv4ToInt };
|
package/dist/host-capacity.d.mts
CHANGED
package/dist/host-capacity.mjs
CHANGED
|
@@ -1,152 +1,38 @@
|
|
|
1
|
-
// @ts-check
|
|
2
|
-
/**
|
|
3
|
-
* How many worker VMs will actually fit on this Mac?
|
|
4
|
-
*
|
|
5
|
-
* The pool used to start every VM it found. On a 36 GB host that meant three guests, and measurement
|
|
6
|
-
* says three do not fit: with all three up a capture took 44.5 s, and with one up it took 27.4 s of
|
|
7
|
-
* the same page on the same worker — a 1.6x penalty paid by every capture in the run. The guests were
|
|
8
|
-
* being swapped out from under NVDA, which also produced "NVDA is running but not speaking" failures
|
|
9
|
-
* and blackouts on /health. More workers were making the run slower AND less reliable.
|
|
10
|
-
*
|
|
11
|
-
* So capacity is a property of the host, not a count of the VMs that happen to be registered.
|
|
12
|
-
*
|
|
13
|
-
* `.mjs` rather than `.ts` because `npm run doctor` runs under plain node — it is the first thing an
|
|
14
|
-
* agent runs and must not depend on a transpiler — while the pool lease is TypeScript. Same reason
|
|
15
|
-
* capture-decisions.mjs is .mjs.
|
|
16
|
-
*/
|
|
17
1
|
import { execFileSync } from "node:child_process";
|
|
18
2
|
import { totalmem } from "node:os";
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
*
|
|
25
|
-
* 4096 MB configured -> 8,048-8,127 MB host
|
|
26
|
-
* 3072 MB configured -> 5,494 MB host <- what the guests now run at
|
|
27
|
-
* 2560 MB configured -> 4,952 MB host <- measured, and too small: the guest pages
|
|
28
|
-
*
|
|
29
|
-
* 3072 MB is the setting because it is the smallest that does not page. The guest commits ~1,859 MB, so
|
|
30
|
-
* 2560 leaves too little above it: capture phases went from a 12.1 s median (IQR 0.4, 0 recoveries in
|
|
31
|
-
* 10) to 36.6 s (IQR 38, 4 recoveries in 10). Less RAM stopped being cheaper the moment Windows started
|
|
32
|
-
* swapping, and four cramped workers measured worse than three comfortable ones.
|
|
33
|
-
*
|
|
34
|
-
* The ~1.8x multiplier is QEMU's own overhead on top of guest RAM that Windows dirties and never gives
|
|
35
|
-
* back (there is no balloon driver). That is why the CONFIGURED size is the lever and the guest's usage
|
|
36
|
-
* is not: Windows expands to fill whatever ceiling it is given.
|
|
37
|
-
*
|
|
38
|
-
* This constant has been wrong twice, in the same direction. 7,600 was an underestimate from a short
|
|
39
|
-
* sample; 8,100 was right for 4096 MB guests and became wrong the moment they were re-sized. Re-measure
|
|
40
|
-
* it whenever guest RAM changes -- it is a property of the configuration, not of the software.
|
|
41
|
-
*
|
|
42
|
-
* Deliberately the high end of the range. Under-committing costs a little parallelism; over-committing
|
|
43
|
-
* costs correctness, because a swapped-out guest fails captures rather than merely slowing them.
|
|
44
|
-
*/
|
|
45
|
-
const MEMORY_PER_WORKER_MB = 5_600;
|
|
46
|
-
/**
|
|
47
|
-
* What the host's own software needs, beyond the run.
|
|
48
|
-
*
|
|
49
|
-
* Measured on this Mac while it was doing nothing unusual: Wispr Flow 2.3 GB, tessl 2.0 GB, stable
|
|
50
|
-
* 1.3 GB, mds_stores 1.1 GB, WindowServer 1.1 GB, Codex 0.9 GB and the rest — about 11 GB before a
|
|
51
|
-
* single guest starts. This module only ever runs on macOS (see availableHostMemoryMb), so it is
|
|
52
|
-
* always somebody's desktop, never a dedicated hypervisor.
|
|
53
|
-
*/
|
|
54
|
-
const HOST_APPS_RESERVE_MB = 12_000;
|
|
55
|
-
/**
|
|
56
|
-
* Memory left for the host itself.
|
|
57
|
-
*
|
|
58
|
-
* The Mac is somebody's desktop, not a dedicated hypervisor — Chrome, Spotify and the editor were all
|
|
59
|
-
* resident during the measurements above. Taking the last of the available memory pushes the HOST into
|
|
60
|
-
* swap, which slows every guest at once.
|
|
61
|
-
*/
|
|
62
|
-
const HOST_HEADROOM_MB = 3_000;
|
|
63
|
-
/**
|
|
64
|
-
* Available memory on macOS, in MB, or null if it cannot be read.
|
|
65
|
-
*
|
|
66
|
-
* `os.freemem()` is useless here: it reported 402 MB on a host that comfortably had ~12 GB to give,
|
|
67
|
-
* because macOS counts compressed and inactive pages as used. The pages that can actually be handed
|
|
68
|
-
* out without swapping are free + inactive + speculative + purgeable, which is what `vm_stat` reports.
|
|
69
|
-
*
|
|
70
|
-
* Returns null rather than throwing on any surprise — a capacity check that cannot read the host must
|
|
71
|
-
* not stop a run. This codebase already applies that rule to foregroundLockTimeout(): a broken
|
|
72
|
-
* diagnostic taking the pool offline is worse than the fault it looks for.
|
|
73
|
-
*/
|
|
74
|
-
export function availableHostMemoryMb() {
|
|
75
|
-
if (process.platform !== "darwin")
|
|
76
|
-
return null;
|
|
3
|
+
const MEMORY_PER_WORKER_MB = 5600;
|
|
4
|
+
const HOST_APPS_RESERVE_MB = 12000;
|
|
5
|
+
const HOST_HEADROOM_MB = 3000;
|
|
6
|
+
function availableHostMemoryMb() {
|
|
7
|
+
if ("darwin" !== process.platform) return null;
|
|
77
8
|
try {
|
|
78
|
-
const output = execFileSync("vm_stat", {
|
|
9
|
+
const output = execFileSync("vm_stat", {
|
|
10
|
+
encoding: "utf8",
|
|
11
|
+
timeout: 5000
|
|
12
|
+
});
|
|
79
13
|
const pageSize = Number(/page size of (\d+) bytes/.exec(output)?.[1]);
|
|
80
|
-
if (!Number.isFinite(pageSize))
|
|
81
|
-
|
|
82
|
-
/** @param {string} label */
|
|
83
|
-
const pagesFor = (label) => Number(new RegExp(`Pages ${label}:\\s+(\\d+)`).exec(output)?.[1] ?? 0);
|
|
14
|
+
if (!Number.isFinite(pageSize)) return null;
|
|
15
|
+
const pagesFor = (label)=>Number(new RegExp(`Pages ${label}:\\s+(\\d+)`).exec(output)?.[1] ?? 0);
|
|
84
16
|
const pages = pagesFor("free") + pagesFor("inactive") + pagesFor("speculative") + pagesFor("purgeable");
|
|
85
|
-
return Math.round(
|
|
86
|
-
}
|
|
87
|
-
catch {
|
|
17
|
+
return Math.round(pages * pageSize / 1048576);
|
|
18
|
+
} catch {
|
|
88
19
|
return null;
|
|
89
20
|
}
|
|
90
21
|
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
return Math.round(totalmem() / (1024 * 1024));
|
|
22
|
+
function totalHostMemoryMb() {
|
|
23
|
+
return Math.round(totalmem() / 1048576);
|
|
94
24
|
}
|
|
95
|
-
|
|
96
|
-
* The most workers this machine can EVER hold, from physical RAM alone.
|
|
97
|
-
*
|
|
98
|
-
* This exists because the dynamic estimate below is computed from `vm_stat`, and `vm_stat` is
|
|
99
|
-
* distorted by exactly the situation it needs to detect. Once guests are swapped out, their pages are
|
|
100
|
-
* counted as compressed/inactive — which `availableHostMemoryMb` reports as *available* — so a host
|
|
101
|
-
* three guests deep in swap advertised 13.7 GB free while two of the three could not answer an HTTP
|
|
102
|
-
* health check within 75 seconds. Total RAM is immune to that feedback loop.
|
|
103
|
-
*
|
|
104
|
-
* @returns {number} workers, from physical memory
|
|
105
|
-
*/
|
|
106
|
-
export function workerCeilingFromTotalRam(totalMb = totalHostMemoryMb()) {
|
|
25
|
+
function workerCeilingFromTotalRam(totalMb = totalHostMemoryMb()) {
|
|
107
26
|
return Math.floor((totalMb - HOST_APPS_RESERVE_MB - HOST_HEADROOM_MB) / MEMORY_PER_WORKER_MB);
|
|
108
27
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
*
|
|
112
|
-
* **A running worker is not automatically an affordable one.** This used to return
|
|
113
|
-
* `alreadyRunning + canStart` on the reasoning that guests already up had "paid for" their memory and
|
|
114
|
-
* `availableMb` was what remained after them. That is true of a healthy host and false of the one case
|
|
115
|
-
* that matters: when the host is in swap, the running guests are being paid for out of disk, and adding
|
|
116
|
-
* their count back ratifies the very over-commitment that is breaking the run. The result could never
|
|
117
|
-
* be lower than the number of VMs already up, so the cap was structurally unable to hold back a pool
|
|
118
|
-
* somebody had already started — which is how three guests came to share a 36 GB Mac, drive 6.6 GB of
|
|
119
|
-
* swap, and black out two of the three workers mid-run.
|
|
120
|
-
*
|
|
121
|
-
* So the answer is the lower of the dynamic estimate and the physical-RAM ceiling, and it is allowed to
|
|
122
|
-
* come out below `alreadyRunning`. The run then dispatches to fewer workers than are up, which is the
|
|
123
|
-
* conservative direction: the extra guest wastes memory but no longer takes work.
|
|
124
|
-
*
|
|
125
|
-
* Never returns less than one: a run with no workers is a worse outcome than a slow one.
|
|
126
|
-
*
|
|
127
|
-
* @param {{ availableMb: number | null, alreadyRunning: number, totalMb?: number }} host
|
|
128
|
-
* @returns {number}
|
|
129
|
-
*/
|
|
130
|
-
export function workersHostCanRun({ availableMb, alreadyRunning, totalMb = totalHostMemoryMb() }) {
|
|
131
|
-
if (availableMb === null)
|
|
132
|
-
return Number.POSITIVE_INFINITY; // unreadable: do not constrain the run
|
|
28
|
+
function workersHostCanRun({ availableMb, alreadyRunning, totalMb = totalHostMemoryMb() }) {
|
|
29
|
+
if (null === availableMb) return 1 / 0;
|
|
133
30
|
const spareMb = availableMb - HOST_HEADROOM_MB;
|
|
134
31
|
const dynamic = alreadyRunning + Math.max(0, Math.floor(spareMb / MEMORY_PER_WORKER_MB));
|
|
135
32
|
return Math.max(1, Math.min(dynamic, workerCeilingFromTotalRam(totalMb)));
|
|
136
33
|
}
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
* @param {{ limit: number, wanted: number, availableMb: number | null }} cap
|
|
141
|
-
* @returns {string | null}
|
|
142
|
-
*/
|
|
143
|
-
export function capacityReason({ limit, wanted, availableMb }) {
|
|
144
|
-
if (!Number.isFinite(limit) || limit >= wanted)
|
|
145
|
-
return null;
|
|
146
|
-
return `host has ${totalHostMemoryMb()} MB of RAM and ~${availableMb} MB available, and each worker ` +
|
|
147
|
-
`costs ~${MEMORY_PER_WORKER_MB} MB, so ${limit} of ${wanted} local workers will be used. ` +
|
|
148
|
-
"Measured: an over-committed host made every capture 1.6x slower, caused mute-NVDA failures, and " +
|
|
149
|
-
"once starved two of three guests until they stopped answering /health at all. " +
|
|
150
|
-
"Override with A11Y_MAX_WORKERS if you know better.";
|
|
34
|
+
function capacityReason({ limit, wanted, availableMb }) {
|
|
35
|
+
if (!Number.isFinite(limit) || limit >= wanted) return null;
|
|
36
|
+
return `host has ${totalHostMemoryMb()} MB of RAM and ~${availableMb} MB available, and each worker costs ~${MEMORY_PER_WORKER_MB} MB, so ${limit} of ${wanted} local workers will be used. Measured: an over-committed host made every capture 1.6x slower, caused mute-NVDA failures, and once starved two of three guests until they stopped answering /health at all. Override with A11Y_MAX_WORKERS if you know better.`;
|
|
151
37
|
}
|
|
152
|
-
|
|
38
|
+
export { availableHostMemoryMb, capacityReason, totalHostMemoryMb, workerCeilingFromTotalRam, workersHostCanRun };
|
package/dist/host-metrics.d.mts
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -20,4 +20,3 @@ export type { AfterRun, WorkerLease, PoolLease } from "./local-vm.js";
|
|
|
20
20
|
* they live: it had four, and one of them was wrong, which broke `leaseWorker` silently.
|
|
21
21
|
*/
|
|
22
22
|
export declare function fleetScriptPaths(): Record<string, string>;
|
|
23
|
-
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.mjs
ADDED
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
2
|
+
import { promisify } from "node:util";
|
|
3
|
+
import { networkInterfaces } from "node:os";
|
|
4
|
+
import { capacityReason, availableHostMemoryMb, workersHostCanRun } from "./host-capacity.mjs";
|
|
5
|
+
import { fleetScriptPaths as fleet_scripts_fleetScriptPaths } from "./src_fleet-scripts_mjs.mjs";
|
|
6
|
+
import { inventoryWorkerUrls } from "./fleet-env.mjs";
|
|
7
|
+
import { warnUtmDeprecated } from "./src_utm-deprecated_mjs.mjs";
|
|
8
|
+
const execFileAsync = promisify(execFile);
|
|
9
|
+
const DEFAULT_WORKER = "http://localhost:8765";
|
|
10
|
+
const CTL = fleet_scripts_fleetScriptPaths().workerCtl;
|
|
11
|
+
const STATUS_TIMEOUT_MS = 30000;
|
|
12
|
+
const LIFECYCLE_TIMEOUT_MS = 300000;
|
|
13
|
+
const AFTER_RUN_VALUES = [
|
|
14
|
+
"restore",
|
|
15
|
+
"stop",
|
|
16
|
+
"pause",
|
|
17
|
+
"leave"
|
|
18
|
+
];
|
|
19
|
+
function isAfterRun(v) {
|
|
20
|
+
return AFTER_RUN_VALUES.includes(v);
|
|
21
|
+
}
|
|
22
|
+
const IPV4_OCTETS = 4;
|
|
23
|
+
function ipv4ToInt(ip) {
|
|
24
|
+
const parts = ip.split(".").map(Number);
|
|
25
|
+
if (parts.length !== IPV4_OCTETS || parts.some((n)=>!Number.isInteger(n) || n < 0 || n > 255)) return null;
|
|
26
|
+
return parts.reduce((acc, octet)=>256 * acc + octet, 0);
|
|
27
|
+
}
|
|
28
|
+
function hostAddressFor(guestIp) {
|
|
29
|
+
const guest = ipv4ToInt(guestIp);
|
|
30
|
+
if (null === guest) return;
|
|
31
|
+
for (const addresses of Object.values(networkInterfaces()))for (const a of addresses ?? []){
|
|
32
|
+
if ("IPv4" !== a.family || a.internal) continue;
|
|
33
|
+
const host = ipv4ToInt(a.address);
|
|
34
|
+
const mask = ipv4ToInt(a.netmask);
|
|
35
|
+
if (null !== host && null !== mask) {
|
|
36
|
+
if ((host & mask) === (guest & mask)) return a.address;
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
async function ctl(args, timeoutMs, vmName) {
|
|
41
|
+
const env = vmName ? {
|
|
42
|
+
...process.env,
|
|
43
|
+
A11Y_VM_NAME: vmName
|
|
44
|
+
} : process.env;
|
|
45
|
+
const { stdout } = await execFileAsync(CTL, args, {
|
|
46
|
+
timeout: timeoutMs,
|
|
47
|
+
encoding: "utf8",
|
|
48
|
+
env
|
|
49
|
+
});
|
|
50
|
+
return stdout;
|
|
51
|
+
}
|
|
52
|
+
async function findLocalVm() {
|
|
53
|
+
if ("darwin" !== process.platform) return null;
|
|
54
|
+
try {
|
|
55
|
+
return JSON.parse(await ctl([
|
|
56
|
+
"json"
|
|
57
|
+
], STATUS_TIMEOUT_MS));
|
|
58
|
+
} catch (e) {
|
|
59
|
+
if (process.env.A11Y_DEBUG_VM) process.stderr.write(`local VM lookup failed (continuing without one): ${e.message}\n`);
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
function restoreAction(stateBefore) {
|
|
64
|
+
if ("started" === stateBefore) return "leave";
|
|
65
|
+
if ("paused" === stateBefore) return "pause";
|
|
66
|
+
return "stop";
|
|
67
|
+
}
|
|
68
|
+
async function releaseVm(action, stateBefore) {
|
|
69
|
+
const resolved = "restore" === action ? restoreAction(stateBefore) : action;
|
|
70
|
+
if ("leave" === resolved) return;
|
|
71
|
+
const now = await findLocalVm();
|
|
72
|
+
if (now?.busy) return void process.stderr.write(`worker is busy with another capture; leaving the VM ${now.state}\n`);
|
|
73
|
+
process.stderr.write(`${"pause" === resolved ? "Pausing" : "Shutting down"} the local worker VM ...\n`);
|
|
74
|
+
await ctl([
|
|
75
|
+
resolved
|
|
76
|
+
], LIFECYCLE_TIMEOUT_MS);
|
|
77
|
+
}
|
|
78
|
+
async function acquireLocalWorker(vm, after) {
|
|
79
|
+
const stateBefore = vm.state;
|
|
80
|
+
if ("started" === stateBefore && vm.healthy) process.stderr.write(`Using the running local worker VM '${vm.name}' at ${vm.ip}\n`);
|
|
81
|
+
else {
|
|
82
|
+
process.stderr.write(`Local worker VM '${vm.name}' is ${stateBefore}; bringing it up ...\n`);
|
|
83
|
+
process.stderr.write(await ctl([
|
|
84
|
+
"up"
|
|
85
|
+
], LIFECYCLE_TIMEOUT_MS));
|
|
86
|
+
}
|
|
87
|
+
const ready = await findLocalVm();
|
|
88
|
+
if (!ready?.healthy || !ready.ip) throw new Error(`Local worker VM '${vm.name}' did not become healthy. Check it directly: ${CTL} status`);
|
|
89
|
+
return {
|
|
90
|
+
worker: `http://${ready.ip}:${ready.port}`,
|
|
91
|
+
source: "local-vm",
|
|
92
|
+
hostAddress: hostAddressFor(ready.ip),
|
|
93
|
+
release: ()=>releaseVm(after, stateBefore).catch((e)=>{
|
|
94
|
+
process.stderr.write(`WARNING: could not release the local worker VM: ${e.message}\n`);
|
|
95
|
+
})
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
async function leaseWorker({ worker, after }, deps = {}) {
|
|
99
|
+
const release = async ()=>{};
|
|
100
|
+
if (worker) return {
|
|
101
|
+
worker: worker.replace(/\/$/, ""),
|
|
102
|
+
source: "explicit",
|
|
103
|
+
release
|
|
104
|
+
};
|
|
105
|
+
const fleet = deps.inventory ? deps.inventory() : inventoryWorkerUrls();
|
|
106
|
+
if (fleet.length) return {
|
|
107
|
+
worker: fleet[0].replace(/\/$/, ""),
|
|
108
|
+
source: "inventory.yml",
|
|
109
|
+
release
|
|
110
|
+
};
|
|
111
|
+
if ("0" === process.env.A11Y_LOCAL_VM) return {
|
|
112
|
+
worker: DEFAULT_WORKER,
|
|
113
|
+
source: "default",
|
|
114
|
+
release
|
|
115
|
+
};
|
|
116
|
+
const vm = deps.findLocalVm ? await deps.findLocalVm() : await findLocalVm();
|
|
117
|
+
if (!vm) return {
|
|
118
|
+
worker: DEFAULT_WORKER,
|
|
119
|
+
source: "default",
|
|
120
|
+
release
|
|
121
|
+
};
|
|
122
|
+
warnUtmDeprecated("this run (no worker named, no fleet configured)");
|
|
123
|
+
return acquireLocalWorker(vm, after);
|
|
124
|
+
}
|
|
125
|
+
function hostAddressForWorker(workerUrl) {
|
|
126
|
+
try {
|
|
127
|
+
const { hostname } = new URL(workerUrl);
|
|
128
|
+
return null === ipv4ToInt(hostname) ? void 0 : hostAddressFor(hostname);
|
|
129
|
+
} catch {
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
function guestReachableUrl(baseUrl, lease) {
|
|
134
|
+
const hostAddress = lease.hostAddress ?? hostAddressForWorker(lease.worker);
|
|
135
|
+
if (!hostAddress) return baseUrl;
|
|
136
|
+
const parsed = new URL(baseUrl);
|
|
137
|
+
if ("localhost" !== parsed.hostname && "127.0.0.1" !== parsed.hostname) return baseUrl;
|
|
138
|
+
parsed.hostname = hostAddress;
|
|
139
|
+
return parsed.toString().replace(/\/$/, "");
|
|
140
|
+
}
|
|
141
|
+
async function findLocalPool() {
|
|
142
|
+
if ("darwin" !== process.platform) return [];
|
|
143
|
+
try {
|
|
144
|
+
return JSON.parse(await ctl([
|
|
145
|
+
"pool"
|
|
146
|
+
], STATUS_TIMEOUT_MS));
|
|
147
|
+
} catch {
|
|
148
|
+
return [];
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
function configuredWorkerLimit() {
|
|
152
|
+
const configured = Number(process.env.A11Y_MAX_WORKERS);
|
|
153
|
+
return Number.isFinite(configured) && configured > 0 ? configured : null;
|
|
154
|
+
}
|
|
155
|
+
function chooseRunnableWorkers(pool) {
|
|
156
|
+
const availableMb = availableHostMemoryMb();
|
|
157
|
+
const running = pool.filter((vm)=>"started" === vm.state);
|
|
158
|
+
const limit = configuredWorkerLimit() ?? workersHostCanRun({
|
|
159
|
+
availableMb,
|
|
160
|
+
alreadyRunning: running.length
|
|
161
|
+
});
|
|
162
|
+
if (limit >= pool.length) return {
|
|
163
|
+
chosen: pool,
|
|
164
|
+
note: null
|
|
165
|
+
};
|
|
166
|
+
const chosen = [
|
|
167
|
+
...running,
|
|
168
|
+
...pool.filter((vm)=>"started" !== vm.state)
|
|
169
|
+
].slice(0, limit);
|
|
170
|
+
return {
|
|
171
|
+
chosen,
|
|
172
|
+
note: capacityReason({
|
|
173
|
+
limit,
|
|
174
|
+
wanted: pool.length,
|
|
175
|
+
availableMb
|
|
176
|
+
})
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
async function leaseWorkerPool(after) {
|
|
180
|
+
warnUtmDeprecated("the local UTM worker pool");
|
|
181
|
+
const pool = await findLocalPool();
|
|
182
|
+
if (pool.length < 2) return null;
|
|
183
|
+
const { chosen, note } = chooseRunnableWorkers(pool);
|
|
184
|
+
if (note) process.stderr.write(note + "\n");
|
|
185
|
+
const chosenNames = new Set(chosen.map((vm)=>vm.name));
|
|
186
|
+
const before = new Map(pool.map((vm)=>[
|
|
187
|
+
vm.name,
|
|
188
|
+
vm.state
|
|
189
|
+
]));
|
|
190
|
+
for (const vm of chosen)if ("started" !== vm.state || !vm.healthy) {
|
|
191
|
+
process.stderr.write(`Local worker '${vm.name}' is ${vm.state}; bringing it up ...\n`);
|
|
192
|
+
try {
|
|
193
|
+
await ctl([
|
|
194
|
+
"up"
|
|
195
|
+
], LIFECYCLE_TIMEOUT_MS, vm.name);
|
|
196
|
+
} catch (error) {
|
|
197
|
+
process.stderr.write(`Local worker '${vm.name}' would not come up (${error.message.split("\n")[0]}); continuing without it\n`);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
const ready = (await findLocalPool()).filter((vm)=>vm.healthy && vm.ip && chosenNames.has(vm.name));
|
|
201
|
+
if (!ready.length) throw new Error("no local worker became healthy; check: worker-ctl.sh pool");
|
|
202
|
+
const missing = chosen.length - ready.length;
|
|
203
|
+
if (missing > 0) process.stderr.write(`${missing} of ${chosen.length} local workers are unavailable; running on the rest\n`);
|
|
204
|
+
process.stderr.write(`Pool of ${ready.length}: ${ready.map((v)=>v.name).join(", ")}\n`);
|
|
205
|
+
return {
|
|
206
|
+
workers: ready.map((vm)=>`http://${vm.ip}:${vm.port}`),
|
|
207
|
+
hostAddress: hostAddressFor(ready[0].ip),
|
|
208
|
+
release: async ()=>{
|
|
209
|
+
for (const vm of ready){
|
|
210
|
+
const action = "restore" === after ? restoreAction(before.get(vm.name) ?? "stopped") : after;
|
|
211
|
+
if ("leave" !== action) try {
|
|
212
|
+
const now = (await findLocalPool()).find((v)=>v.name === vm.name);
|
|
213
|
+
if (now?.busy) {
|
|
214
|
+
process.stderr.write(`'${vm.name}' is busy with another capture; leaving it ${now.state}\n`);
|
|
215
|
+
continue;
|
|
216
|
+
}
|
|
217
|
+
process.stderr.write(`${"pause" === action ? "Pausing" : "Shutting down"} '${vm.name}' ...\n`);
|
|
218
|
+
await ctl([
|
|
219
|
+
action
|
|
220
|
+
], LIFECYCLE_TIMEOUT_MS, vm.name);
|
|
221
|
+
} catch (e) {
|
|
222
|
+
process.stderr.write(`WARNING: could not ${action} '${vm.name}': ${e.message}\n`);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
function fleetScriptPaths() {
|
|
229
|
+
return fleet_scripts_fleetScriptPaths();
|
|
230
|
+
}
|
|
231
|
+
export { DEFAULT_WORKER, fleetScriptPaths, guestReachableUrl, hostAddressForWorker, isAfterRun, leaseWorker, leaseWorkerPool };
|
package/dist/local-vm.d.ts
CHANGED
package/dist/measure-guard.d.mts
CHANGED
|
@@ -31,4 +31,3 @@ export function sampleVitals(worker: string, { timeoutMs, request, warn }?: {
|
|
|
31
31
|
* both. A vitals sample that times out costs the run one column, so there is no reason to give it less.
|
|
32
32
|
*/
|
|
33
33
|
export const VITALS_TIMEOUT_MS: 20000;
|
|
34
|
-
//# sourceMappingURL=measure-guard.d.mts.map
|
package/dist/probe-outcome.d.mts
CHANGED