@smoothbricks/nx-plugin 0.4.28 → 0.4.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +277 -89
- package/dist/bin/smoo-nx-test-clocks.d.ts +3 -0
- package/dist/bin/smoo-nx-test-clocks.d.ts.map +1 -0
- package/dist/bin/smoo-nx-test-clocks.js +28 -0
- package/dist/bounded-test-policy.d.ts +8 -8
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +18 -30
- package/dist/cargo-closure-input.d.ts +23 -0
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +41 -9
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +42 -19
- package/dist/ensure-built-receipt.d.ts +40 -6
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +297 -159
- package/dist/ensure-built-witness.d.ts +65 -4
- package/dist/ensure-built-witness.d.ts.map +1 -1
- package/dist/ensure-built-witness.js +2237 -23
- package/dist/ensure-built.d.ts +99 -1
- package/dist/ensure-built.d.ts.map +1 -1
- package/dist/ensure-built.js +45 -56
- package/dist/executors/bounded-exec/cpu-pool.d.ts +22 -7
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +51 -22
- package/dist/executors/bounded-exec/executor.d.ts +12 -3
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +174 -51
- package/dist/executors/bounded-exec/no-space.d.ts +8 -0
- package/dist/executors/bounded-exec/no-space.d.ts.map +1 -0
- package/dist/executors/bounded-exec/no-space.js +31 -0
- package/dist/executors/bounded-exec/ram-temp.d.ts +20 -11
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +42 -17
- package/dist/executors/bounded-exec/verdict.d.ts +34 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +54 -3
- package/dist/generators/bun-test-tracing/generator.js +2 -2
- package/dist/graph-input-contract.d.ts +102 -0
- package/dist/graph-input-contract.d.ts.map +1 -0
- package/dist/graph-input-contract.js +637 -0
- package/dist/index.d.ts +12 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -3
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +5 -2
- package/dist/managed-files/publish-workflow.d.ts +11 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/owned-installation.d.ts +14 -0
- package/dist/owned-installation.d.ts.map +1 -0
- package/dist/owned-installation.js +324 -0
- package/dist/producer-receipts.d.ts +19 -0
- package/dist/producer-receipts.d.ts.map +1 -0
- package/dist/producer-receipts.js +926 -0
- package/dist/test-backstop.d.ts +9 -0
- package/dist/test-backstop.d.ts.map +1 -0
- package/dist/test-backstop.js +10 -0
- package/dist/test-clock-policy.d.ts +30 -0
- package/dist/test-clock-policy.d.ts.map +1 -0
- package/dist/test-clock-policy.js +686 -0
- package/dist/testing.d.ts +8 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +66 -1
- package/managed/raw/patches/nx@23.2.1.patch +328 -50
- package/managed/raw/tooling/direnv/developer-links.ts +148 -29
- package/managed/raw/tooling/direnv/devenv.smoo.nix +7 -7
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +14 -5
- package/managed/raw/tooling/direnv/linked-snapshots.ts +206 -40
- package/managed/raw/tooling/direnv/repo-path +15 -2
- package/managed/raw/tooling/direnv/setup-environment.ts +179 -115
- package/managed/templates/github/actions/cache-node-modules/action.yml +9 -0
- package/managed/templates/github/actions/setup-devenv/action.yml +63 -7
- package/nextest.toml +1 -1
- package/package.json +25 -9
- package/src/__tests__/cargo-closure-fixture.ts +113 -0
- package/src/__tests__/counted-cargo.ts +8 -25
- package/src/__tests__/directory-change.ts +27 -0
- package/src/__tests__/fixture-nx-env.ts +7 -0
- package/src/__tests__/owned-installation.ts +14 -0
- package/src/__tests__/past-tick.ts +12 -0
- package/src/bin/smoo-nx-bound-failures.test.ts +1 -1
- package/src/bin/smoo-nx-test-clocks.ts +29 -0
- package/src/bounded-test-policy.test.ts +36 -22
- package/src/bounded-test-policy.ts +18 -32
- package/src/bun-bootstrap.test.ts +1 -1
- package/src/cargo-closure-input.property.test.ts +50 -0
- package/src/cargo-closure-input.test.ts +73 -121
- package/src/cargo-closure-input.ts +47 -8
- package/src/cargo-closure-key.test.ts +239 -0
- package/src/cargo-resolve-cache.test.ts +168 -31
- package/src/cargo-source-hash.test.ts +78 -3
- package/src/cargo-source-hash.ts +48 -30
- package/src/cross-check-policy.test.ts +2 -2
- package/src/ensure-built-receipt.test.ts +111 -16
- package/src/ensure-built-receipt.ts +294 -149
- package/src/ensure-built-witness.test.ts +306 -13
- package/src/ensure-built-witness.ts +255 -24
- package/src/ensure-built.test.ts +228 -102
- package/src/ensure-built.ts +127 -66
- package/src/executors/bounded-exec/cpu-pool.test.ts +134 -32
- package/src/executors/bounded-exec/cpu-pool.ts +64 -20
- package/src/executors/bounded-exec/cpu-tokens.test.ts +2 -2
- package/src/executors/bounded-exec/disk-lease.test.ts +47 -13
- package/src/executors/bounded-exec/executor-host.test.ts +74 -52
- package/src/executors/bounded-exec/executor.test.ts +343 -97
- package/src/executors/bounded-exec/executor.ts +192 -56
- package/src/executors/bounded-exec/hang-diagnostics.test.ts +14 -8
- package/src/executors/bounded-exec/no-space.ts +40 -0
- package/src/executors/bounded-exec/progress.test.ts +1 -1
- package/src/executors/bounded-exec/ram-temp-host.test.ts +51 -21
- package/src/executors/bounded-exec/ram-temp.test.ts +92 -16
- package/src/executors/bounded-exec/ram-temp.ts +49 -20
- package/src/executors/bounded-exec/verdict.test.ts +127 -22
- package/src/executors/bounded-exec/verdict.ts +78 -3
- package/src/generators/bounded-test-targets/generator.test.ts +1 -3
- package/src/generators/bun-test-tracing/generator.test.ts +2 -2
- package/src/generators/bun-test-tracing/generator.ts +2 -2
- package/src/graph-input-contract.test.ts +379 -0
- package/src/graph-input-contract.ts +627 -0
- package/src/index.test.ts +3 -3
- package/src/index.ts +24 -3
- package/src/linked-snapshots.nx.test.ts +112 -30
- package/src/managed-files/ci-workflow.ts +5 -2
- package/src/managed-files/publish-workflow.ts +12 -1
- package/src/managed-files/repo-path.test.ts +44 -1
- package/src/managed-files/setup-devenv-free-disk.test.ts +95 -0
- package/src/nextest-code-bound.test.ts +23 -17
- package/src/nextest-extraction.test.ts +114 -4
- package/src/nextest-partitions.test.ts +7 -1
- package/src/nx-cache-restore-mtime.test.ts +2 -2
- package/src/nx-daemon-claim.test.ts +10 -8
- package/src/nx-daemon-idle-cpu.test.ts +0 -1
- package/src/nx-daemon-idle-policy.test.ts +61 -1
- package/src/nx-daemon-ignore-files.test.ts +2 -2
- package/src/nx-daemon-validity.test.ts +36 -22
- package/src/nx-dot-input-policy.test.ts +5 -5
- package/src/nx-files-archive-racy.test.ts +1 -1
- package/src/nx-incomplete-run-summary.test.ts +1 -1
- package/src/nx-plugin-load-errors.test.ts +2 -2
- package/src/nx-project-graph-errors.test.ts +70 -0
- package/src/nx-skipped-task-exit.test.ts +2 -2
- package/src/nx-store-resolution.test.ts +7 -25
- package/src/nx-task-output-lease.test.ts +3 -3
- package/src/nx-use-local.test.ts +7 -5
- package/src/owned-installation.ts +174 -0
- package/src/producer-receipts.test.ts +164 -0
- package/src/producer-receipts.ts +636 -0
- package/src/runtime-input.test.ts +1 -1
- package/src/serial-group-wiring.test.ts +4 -4
- package/src/task-results.test.ts +18 -16
- package/src/test-backstop.ts +11 -0
- package/src/test-clock-policy.test.ts +338 -0
- package/src/test-clock-policy.ts +708 -0
- package/src/test-timings.test.ts +2 -2
- package/src/testing.test.ts +5 -5
- package/src/testing.ts +67 -1
- package/src/typecheck-test-policy.test.ts +1 -1
- package/src/typescript-resolution.test.ts +76 -0
- package/test-timings.json +8 -0
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AAMA,OAAO,EAKL,KAAK,gBAAgB,EAGtB,MAAM,eAAe,CAAC;AACvB,OAAO,EAEL,KAAK,SAAS,EAKf,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAGhF,OAAO,EAML,aAAa,EAEd,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAEL,KAAK,eAAe,EAarB,MAAM,cAAc,CAAC;AAiBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;CACxB;AAcD,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC;AAQpG;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED,mGAAmG;AACnG,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CA2B5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,MAAM,EAAE,SAAS,GAAG,IAAI,EACxB,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,GACzE,OAAO,CAAC,iBAAiB,CAAC,CA6iB5B;AAED,wBAAgB,uBAAuB,IAAI,iBAAiB,CAM3D"}
|
|
@@ -1,13 +1,15 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
|
+
import { accessSync, constants } from 'node:fs';
|
|
2
3
|
import { availableParallelism, userInfo } from 'node:os';
|
|
3
|
-
import { isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
4
|
+
import { delimiter, isAbsolute, join, resolve as resolvePath } from 'node:path';
|
|
4
5
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
5
6
|
import { CPU_POOL_ENV, createCpuPool, readInheritedCpuPool, takeParentCpuTokens, } from './cpu-pool.js';
|
|
6
7
|
import { CPU_TOKENS_ENV, gatewayCpuBudget, runnerDemand, sizeRunner, } from './cpu-tokens.js';
|
|
7
8
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
9
|
+
import { createNoSpaceWatch } from './no-space.js';
|
|
8
10
|
import { createRunProgress, withProgressReport } from './progress.js';
|
|
9
11
|
import { COWSHED_EXECUTABLE_ENV, createHostCommands, describeRamTempError, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
|
|
10
|
-
import { clearTaskDirectory, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
12
|
+
import { clearTaskDirectory, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
11
13
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
12
14
|
const REAP_AFTER_FORCE_KILL_MS = 2_000;
|
|
13
15
|
const EXIT_CODE_BY_SIGNAL = {
|
|
@@ -17,6 +19,48 @@ const EXIT_CODE_BY_SIGNAL = {
|
|
|
17
19
|
};
|
|
18
20
|
/** The variables the CPU budget sets for the command it grants: no target's environment may set them. */
|
|
19
21
|
const BUDGET_ENV = [CPU_TOKENS_ENV, CPU_POOL_ENV];
|
|
22
|
+
/** Workspace-provided cowshed launchers, in the order a workspace that provides one names them. */
|
|
23
|
+
const WORKSPACE_COWSHED_LAUNCHERS = [
|
|
24
|
+
'packages/cowshed/bin/cowshed',
|
|
25
|
+
'node_modules/@smoothbricks/cowshed/bin/cowshed',
|
|
26
|
+
];
|
|
27
|
+
/**
|
|
28
|
+
* The cowshed a run's disk commands use, in precedence order: the target's named one, an
|
|
29
|
+
* explicit one inherited from the parent process, the installed one on PATH, and — only when
|
|
30
|
+
* PATH provides none — the launcher this workspace itself provides (its own cowshed package,
|
|
31
|
+
* or a linked one's bin). An environment with no cowshed anywhere keeps the bare name, so
|
|
32
|
+
* its spawn failure names the missing installed tool.
|
|
33
|
+
*/
|
|
34
|
+
export function resolveCowshedExecutable(root, named, inherited, pathValue, cwd, exists = (path) => {
|
|
35
|
+
try {
|
|
36
|
+
accessSync(path, constants.X_OK);
|
|
37
|
+
return true;
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return false;
|
|
41
|
+
}
|
|
42
|
+
}) {
|
|
43
|
+
// A target-named executable is workspace-relative or absolute against the workspace root.
|
|
44
|
+
// An inherited explicit keeps the coordinate the parent gave it, exactly as the host
|
|
45
|
+
// commands always spawned it: an absolute one verbatim, a relative or bare one for
|
|
46
|
+
// spawn's own resolution — retargeting it to the root would silently change the tool.
|
|
47
|
+
if (named !== undefined)
|
|
48
|
+
return resolvePath(root, named);
|
|
49
|
+
if (inherited !== undefined)
|
|
50
|
+
return inherited;
|
|
51
|
+
// An empty PATH component names the spawning process's current directory, as POSIX
|
|
52
|
+
// spawn resolution does — the executor never chdirs, so that is the caller's cwd,
|
|
53
|
+
// not the workspace root.
|
|
54
|
+
const onPath = (pathValue ?? '').split(delimiter).some((dir) => exists(join(dir === '' ? cwd : dir, 'cowshed')));
|
|
55
|
+
if (onPath)
|
|
56
|
+
return undefined;
|
|
57
|
+
for (const candidate of WORKSPACE_COWSHED_LAUNCHERS) {
|
|
58
|
+
const launcher = join(root, candidate);
|
|
59
|
+
if (exists(launcher))
|
|
60
|
+
return launcher;
|
|
61
|
+
}
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
20
64
|
export default function boundedExecExecutor(options, context) {
|
|
21
65
|
return runDefaultBoundedExec(options, recordContext(context));
|
|
22
66
|
}
|
|
@@ -26,7 +70,7 @@ export function runDefaultBoundedExec(options, context) {
|
|
|
26
70
|
// for its command alike. Nx resolves `{workspaceRoot}/…` in an option to a workspace-relative path, which the
|
|
27
71
|
// command, starting in another directory, could not find: it gets the absolute one.
|
|
28
72
|
const named = options.env?.[COWSHED_EXECUTABLE_ENV];
|
|
29
|
-
const cowshed =
|
|
73
|
+
const cowshed = resolveCowshedExecutable(context.root, named, process.env[COWSHED_EXECUTABLE_ENV], process.env.PATH, process.cwd());
|
|
30
74
|
const run = cowshed === undefined ? options : { ...options, env: { ...options.env, [COWSHED_EXECUTABLE_ENV]: cowshed } };
|
|
31
75
|
// Only macOS has `hdiutil ram://`; elsewhere the command keeps the inherited TMPDIR.
|
|
32
76
|
const tempVolume = process.platform === 'darwin'
|
|
@@ -66,8 +110,9 @@ export function recordContext(context) {
|
|
|
66
110
|
*
|
|
67
111
|
* `inherited` is the parent's CPU pool (`cpu-pool.ts`) that lends this run its tokens. The run asks that pool first
|
|
68
112
|
* and the host budget only when the pool refuses. Whatever it is granted, the run serves its own sub-pool to its
|
|
69
|
-
* descendants, closed before the grant returns. A
|
|
70
|
-
*
|
|
113
|
+
* descendants, closed before the grant returns. A grant whose connection closes while the command runs (a gateway
|
|
114
|
+
* restart, or the lending parent's exit) ends its accounting, never the command: the command runs on over budget to
|
|
115
|
+
* its own exit, and the run says so.
|
|
71
116
|
*/
|
|
72
117
|
export async function runBoundedExec(options, context, killer, tempVolume, budget, inherited = null, captureDiagnostics = captureHangDiagnostics) {
|
|
73
118
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
@@ -85,8 +130,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
85
130
|
settled: false,
|
|
86
131
|
expiry: null,
|
|
87
132
|
forceKillNeeded: false,
|
|
88
|
-
grantLost: false,
|
|
89
133
|
poolCloseError: null,
|
|
134
|
+
noSpace: null,
|
|
90
135
|
};
|
|
91
136
|
const appendStdout = (chunk) => {
|
|
92
137
|
const text = chunk.toString();
|
|
@@ -151,13 +196,13 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
151
196
|
const cores = availableParallelism();
|
|
152
197
|
const demand = runnerDemand(requested, options.parallelism, cores, process.env, options.env);
|
|
153
198
|
const asked = Date.now();
|
|
154
|
-
// A run a parent's CPU pool
|
|
155
|
-
// A pool that answered and refused
|
|
156
|
-
//
|
|
199
|
+
// A run a parent's CPU pool serves takes its tokens there. Only an absent pool leaves the host budget to ask.
|
|
200
|
+
// A pool that answered and refused is a live budget of its own -- a lent grant, or a gate's or a resident's
|
|
201
|
+
// relay to the host -- so asking the host beside it would count the run twice; the run fails instead.
|
|
157
202
|
let grant = null;
|
|
158
203
|
let parentRefusal = null;
|
|
159
204
|
if (inherited !== null) {
|
|
160
|
-
const fromParent = await takeParentCpuTokens(inherited, demand.want
|
|
205
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want);
|
|
161
206
|
if (fromParent.granted) {
|
|
162
207
|
grant = fromParent;
|
|
163
208
|
}
|
|
@@ -209,9 +254,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
209
254
|
return cpuReleased;
|
|
210
255
|
};
|
|
211
256
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
212
|
-
// An `operational` run failed on its CPU budget (a
|
|
213
|
-
//
|
|
214
|
-
const recordRun = async (exitCode, expiry, elapsedMs, operational) => {
|
|
257
|
+
// An `operational` run failed on its CPU budget (a sub-pool that did not close): an ordinary failure, whatever
|
|
258
|
+
// bound also fired, so it is never judged as a bound.
|
|
259
|
+
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace) => {
|
|
215
260
|
if (context.record === undefined) {
|
|
216
261
|
return;
|
|
217
262
|
}
|
|
@@ -219,7 +264,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
219
264
|
const report = await readJunitReport(recordDirectory);
|
|
220
265
|
const verdict = operational
|
|
221
266
|
? judgeOperationalFailure(exitCode, report)
|
|
222
|
-
: judgeRun({ exitCode, elapsedMs, expiry, report });
|
|
267
|
+
: judgeRun({ exitCode, elapsedMs, expiry, report, noSpace });
|
|
223
268
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
224
269
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
225
270
|
}
|
|
@@ -230,16 +275,33 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
230
275
|
appendStderr(message);
|
|
231
276
|
await releaseCpu();
|
|
232
277
|
await releaseLease();
|
|
233
|
-
await recordRun(1, null, Date.now() - asked, false);
|
|
278
|
+
await recordRun(1, null, Date.now() - asked, false, null);
|
|
234
279
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
235
280
|
};
|
|
236
281
|
if (parentRefusal !== null) {
|
|
237
282
|
return refuse(`cowshed: cpu-tokens parent pool refused: ${parentRefusal}; refusing to run the command without asking the host budget\n`);
|
|
238
283
|
}
|
|
284
|
+
// Descendant work queued for host CPU tokens, as its pool hears it (`host-wait`): while any is held the idle clock
|
|
285
|
+
// is stopped, because a queued job is not an idle one (specs/cowshed/05_gateway.md, "The bounds start at the
|
|
286
|
+
// grant"). The clock it stops is made with the command below; a wait held before then is counted and honoured there.
|
|
287
|
+
let heldWaits = 0;
|
|
288
|
+
let idleClock = null;
|
|
289
|
+
const hostWaits = {
|
|
290
|
+
begin: () => {
|
|
291
|
+
heldWaits += 1;
|
|
292
|
+
if (heldWaits === 1)
|
|
293
|
+
idleClock?.stop();
|
|
294
|
+
},
|
|
295
|
+
end: () => {
|
|
296
|
+
heldWaits -= 1;
|
|
297
|
+
if (heldWaits === 0)
|
|
298
|
+
idleClock?.restart();
|
|
299
|
+
},
|
|
300
|
+
};
|
|
239
301
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
240
302
|
// A pool that cannot be set up refuses the run: no command runs unbudgeted.
|
|
241
303
|
if (grant !== null) {
|
|
242
|
-
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
304
|
+
const created = await createCpuPool(grant.tokens, grant.closed, hostWaits);
|
|
243
305
|
if (!created.ok) {
|
|
244
306
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
245
307
|
}
|
|
@@ -275,6 +337,13 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
275
337
|
}
|
|
276
338
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
277
339
|
const progress = createRunProgress();
|
|
340
|
+
// A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
|
|
341
|
+
// since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
|
|
342
|
+
const noSpace = createNoSpaceWatch();
|
|
343
|
+
const sight = async (line) => {
|
|
344
|
+
const held = lease;
|
|
345
|
+
return { line, volume: held === null ? null : ((await tempVolume?.space(held)) ?? null) };
|
|
346
|
+
};
|
|
278
347
|
let idleTimer;
|
|
279
348
|
let captureTimer;
|
|
280
349
|
let idleCapture;
|
|
@@ -282,14 +351,17 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
282
351
|
clearTimeout(captureTimer);
|
|
283
352
|
idleCapture?.controller.abort();
|
|
284
353
|
};
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
354
|
+
// What is left of the current no-progress interval. A finished test sets it back to `idleTimeoutMs`; a held host
|
|
355
|
+
// wait keeps it unspent: the clock is stopped while descendant work is queued for CPU tokens, and runs on from where
|
|
356
|
+
// it stopped once the last wait ends. `queuedMs` is the queued time this interval has not counted, for the report.
|
|
357
|
+
let idleLeftMs = idleTimeoutMs ?? 0;
|
|
358
|
+
let queuedMs = 0;
|
|
359
|
+
let queuedSinceMs = null;
|
|
360
|
+
const scheduleIdle = (limitMs, leftMs) => {
|
|
289
361
|
clearTimeout(idleTimer);
|
|
290
362
|
cancelCapture();
|
|
291
363
|
const capture = {
|
|
292
|
-
expiresAtMs: Date.now() +
|
|
364
|
+
expiresAtMs: Date.now() + leftMs,
|
|
293
365
|
controller: new AbortController(),
|
|
294
366
|
result: { kind: 'pending', startedAtMs: null },
|
|
295
367
|
};
|
|
@@ -322,8 +394,46 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
322
394
|
};
|
|
323
395
|
}
|
|
324
396
|
});
|
|
325
|
-
}, Math.max(0,
|
|
326
|
-
idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs
|
|
397
|
+
}, Math.max(0, leftMs - HANG_CAPTURE_MS));
|
|
398
|
+
idleTimer = setTimeout(() => expire({ kind: 'idle', limitMs, tests: progress.inFlight() }, queuedMs), leftMs);
|
|
399
|
+
};
|
|
400
|
+
const armIdleTimer = () => {
|
|
401
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null) {
|
|
402
|
+
return;
|
|
403
|
+
}
|
|
404
|
+
idleLeftMs = idleTimeoutMs;
|
|
405
|
+
queuedMs = 0;
|
|
406
|
+
if (queuedSinceMs !== null) {
|
|
407
|
+
queuedSinceMs = Date.now();
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
411
|
+
};
|
|
412
|
+
idleClock = {
|
|
413
|
+
stop: () => {
|
|
414
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs !== null) {
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
const now = Date.now();
|
|
418
|
+
queuedSinceMs = now;
|
|
419
|
+
if (idleCapture !== undefined) {
|
|
420
|
+
idleLeftMs = Math.max(0, idleCapture.expiresAtMs - now);
|
|
421
|
+
}
|
|
422
|
+
clearTimeout(idleTimer);
|
|
423
|
+
cancelCapture();
|
|
424
|
+
idleCapture = undefined;
|
|
425
|
+
appendStderr(`bounded-exec: idle clock stopped: a descendant's work is queued for host CPU tokens; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`);
|
|
426
|
+
},
|
|
427
|
+
restart: () => {
|
|
428
|
+
if (idleTimeoutMs === undefined || state.settled || state.expiry !== null || queuedSinceMs === null) {
|
|
429
|
+
return;
|
|
430
|
+
}
|
|
431
|
+
const waitedMs = Date.now() - queuedSinceMs;
|
|
432
|
+
queuedMs += waitedMs;
|
|
433
|
+
queuedSinceMs = null;
|
|
434
|
+
appendStderr(`bounded-exec: idle clock restarted: no descendant's work is queued for host CPU tokens after ${waitedMs}ms; ${idleLeftMs}ms of idleTimeoutMs=${idleTimeoutMs} left\n`);
|
|
435
|
+
scheduleIdle(idleTimeoutMs, idleLeftMs);
|
|
436
|
+
},
|
|
327
437
|
};
|
|
328
438
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
329
439
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
@@ -332,12 +442,18 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
332
442
|
const text = chunk.toString();
|
|
333
443
|
if (progress.scan('stdout', text))
|
|
334
444
|
armIdleTimer();
|
|
445
|
+
const said = noSpace.scan('stdout', text);
|
|
446
|
+
if (said !== null)
|
|
447
|
+
state.noSpace = sight(said);
|
|
335
448
|
appendStdout(text);
|
|
336
449
|
});
|
|
337
450
|
child.stderr?.on('data', (chunk) => {
|
|
338
451
|
const text = chunk.toString();
|
|
339
452
|
if (progress.scan('stderr', text))
|
|
340
453
|
armIdleTimer();
|
|
454
|
+
const said = noSpace.scan('stderr', text);
|
|
455
|
+
if (said !== null)
|
|
456
|
+
state.noSpace = sight(said);
|
|
341
457
|
appendStderr(text);
|
|
342
458
|
});
|
|
343
459
|
const killChildTree = async (force) => {
|
|
@@ -391,8 +507,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
391
507
|
// the reader unable to tell a wedged toolchain from a loaded machine — the
|
|
392
508
|
// two have opposite fixes.
|
|
393
509
|
let termination = Promise.resolve();
|
|
394
|
-
|
|
395
|
-
|
|
510
|
+
// `queuedMs` is the time the idle interval did not count because descendant work was queued for host CPU tokens.
|
|
511
|
+
const expire = (bound, queuedMs = 0) => {
|
|
512
|
+
if (state.settled || state.expiry !== null) {
|
|
396
513
|
return;
|
|
397
514
|
}
|
|
398
515
|
state.expiry = bound;
|
|
@@ -400,7 +517,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
400
517
|
clearTimeout(idleTimer);
|
|
401
518
|
const elapsedMs = Date.now() - startedAt;
|
|
402
519
|
appendStderr(bound.kind === 'idle'
|
|
403
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
520
|
+
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs})${queuedMs > 0 ? `, not counting ${queuedMs}ms queued for host CPU tokens,` : ''} after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
404
521
|
? `still running: ${bound.tests.join(', ')}`
|
|
405
522
|
: 'the runner named no test still running'} (cwd=${cwd}): ${command}\n`
|
|
406
523
|
: `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`);
|
|
@@ -448,23 +565,21 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
448
565
|
};
|
|
449
566
|
const totalTimer = timeoutMs === undefined ? undefined : setTimeout(() => expire({ kind: 'total', limitMs: timeoutMs }), timeoutMs);
|
|
450
567
|
armIdleTimer();
|
|
451
|
-
// A
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
if (grant?.inherited) {
|
|
467
|
-
void grant.closed.then(onGrantLost);
|
|
568
|
+
// A wait its pool heard before the command started stops the clock it was not yet there to stop.
|
|
569
|
+
if (heldWaits > 0)
|
|
570
|
+
idleClock.stop();
|
|
571
|
+
// A grant's connection is a ledger entry, not the command's owner (05_gateway.md "Host CPU budget": a gateway
|
|
572
|
+
// restart forgets every grant, and what runs under it runs on over budget until it finishes). Its close, while the
|
|
573
|
+
// command runs, is said and changes nothing else; only a bound or a signal stops the command. Our own release after
|
|
574
|
+
// the command settled is not a loss; `cpuReleased` marks it.
|
|
575
|
+
if (grant !== null) {
|
|
576
|
+
const held = grant;
|
|
577
|
+
void held.closed.then(() => {
|
|
578
|
+
if (state.settled || cpuReleased !== null) {
|
|
579
|
+
return;
|
|
580
|
+
}
|
|
581
|
+
appendStderr(`cowshed: cpu-tokens grant ended: the ${held.inherited ? 'parent pool' : 'gateway'} closed its ${held.tokens}-token grant while the command ran; it runs on over budget until it ends: ${command}\n`);
|
|
582
|
+
});
|
|
468
583
|
}
|
|
469
584
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
470
585
|
await termination;
|
|
@@ -482,16 +597,24 @@ export async function runBoundedExec(options, context, killer, tempVolume, budge
|
|
|
482
597
|
if (state.expiry !== null && !state.forceKillNeeded) {
|
|
483
598
|
appendStderr(`Timed out command exited after graceful termination: ${command}\n`);
|
|
484
599
|
}
|
|
485
|
-
//
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
600
|
+
// ENOSPC said in the output names the writer's volume as it stood then. Without it, a volume still (nearly) full now
|
|
601
|
+
// is the likely cause, and the volume is named.
|
|
602
|
+
const seen = state.noSpace === null ? null : await state.noSpace;
|
|
603
|
+
if (seen !== null && (code !== 0 || state.expiry !== null)) {
|
|
604
|
+
const writer = context.record?.task.id;
|
|
605
|
+
appendStderr(`${writer === undefined ? '' : `${writer}: `}${describeNoSpace(seen)}\n`);
|
|
606
|
+
}
|
|
607
|
+
else {
|
|
608
|
+
const full = code !== 0 && lease !== null ? await tempVolume?.fullness(lease) : null;
|
|
609
|
+
if (full) {
|
|
610
|
+
appendStderr(`${describeRamTempError(full)}\n`);
|
|
611
|
+
}
|
|
489
612
|
}
|
|
490
613
|
await releaseLease();
|
|
491
|
-
// A run whose
|
|
492
|
-
//
|
|
493
|
-
const operationalFailure = state.
|
|
494
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure);
|
|
614
|
+
// A run whose sub-pool did not close cleanly fails even when the command exited 0, and whatever bound also fired:
|
|
615
|
+
// its verdict is an ordinary failure, never a bound.
|
|
616
|
+
const operationalFailure = state.poolCloseError !== null;
|
|
617
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen);
|
|
495
618
|
return {
|
|
496
619
|
success: !operationalFailure && state.expiry === null && code === 0,
|
|
497
620
|
terminalOutput: outputChunks.join(''),
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { OutputStream } from './progress.js';
|
|
2
|
+
/** Watches a command's output for the first line that says a write found no space. */
|
|
3
|
+
export interface NoSpaceWatch {
|
|
4
|
+
/** One chunk of one output stream: the first matching line, once, as printed; `null` before and after it. */
|
|
5
|
+
scan(stream: OutputStream, chunk: string): string | null;
|
|
6
|
+
}
|
|
7
|
+
export declare function createNoSpaceWatch(): NoSpaceWatch;
|
|
8
|
+
//# sourceMappingURL=no-space.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"no-space.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/no-space.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAWlD,sFAAsF;AACtF,MAAM,WAAW,YAAY;IAC3B,6GAA6G;IAC7G,IAAI,CAAC,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;CAC1D;AAED,wBAAgB,kBAAkB,IAAI,YAAY,CAsBjD"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The operating system's own words for ENOSPC: the errno name Node and Bun print (`ENOSPC: no space left on device,
|
|
3
|
+
* write`), and `strerror(28)`, which Rust (`No space left on device (os error 28)`), Go and C print. Nothing else in a
|
|
4
|
+
* command's output says the disk was full, and a write that failed on it is otherwise reported as whatever its caller
|
|
5
|
+
* was doing: a failed test, an extraction error.
|
|
6
|
+
*/
|
|
7
|
+
const NO_SPACE = /\bENOSPC\b|No space left on device/;
|
|
8
|
+
const ANSI = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*[A-Za-z]`, 'g');
|
|
9
|
+
export function createNoSpaceWatch() {
|
|
10
|
+
const partial = { stdout: '', stderr: '' };
|
|
11
|
+
let seen = false;
|
|
12
|
+
return {
|
|
13
|
+
scan(stream, chunk) {
|
|
14
|
+
if (seen) {
|
|
15
|
+
return null;
|
|
16
|
+
}
|
|
17
|
+
const lines = (partial[stream] + chunk).split('\n');
|
|
18
|
+
// The unfinished last line is kept for the next chunk, and read now too: a command that dies mid-line still
|
|
19
|
+
// printed the words.
|
|
20
|
+
partial[stream] = lines.at(-1) ?? '';
|
|
21
|
+
for (const line of lines) {
|
|
22
|
+
const text = line.replace(ANSI, '').trim();
|
|
23
|
+
if (NO_SPACE.test(text)) {
|
|
24
|
+
seen = true;
|
|
25
|
+
return text;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return null;
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type ProcessEntry } from '../../testing.js';
|
|
2
2
|
import { type PhaseShare } from './disk-lease.js';
|
|
3
|
+
import type { TempVolumeSpace } from './verdict.js';
|
|
3
4
|
/**
|
|
4
5
|
* The volume's ceiling, plus the revision's sectors. Pages are allocated as they are written and
|
|
5
6
|
* returned when the volume detaches, so this bounds what a runaway suite can take, not what a gate
|
|
@@ -24,15 +25,15 @@ export interface HostCommands {
|
|
|
24
25
|
*/
|
|
25
26
|
export interface LeaseProcesses {
|
|
26
27
|
/**
|
|
27
|
-
* End every process
|
|
28
|
-
* and answer each as `<pid> (<command>)`.
|
|
29
|
-
* would not rejects with its name.
|
|
28
|
+
* End every process that works in `directory` — its working directory or the `TMPDIR` it was
|
|
29
|
+
* started with is there — and everything those started, and answer each as `<pid> (<command>)`.
|
|
30
|
+
* Every one has exited when this settles; one that would not rejects with its name.
|
|
30
31
|
*/
|
|
31
32
|
stopWorkingIn(directory: string): Promise<string[]>;
|
|
32
33
|
}
|
|
33
34
|
/** The host reads and signals cleanup uses; a seam for process ordering and signal failures. */
|
|
34
35
|
export interface ProcessHost {
|
|
35
|
-
/** The processes whose working directory is `directory` or lies beneath it. */
|
|
36
|
+
/** The processes whose working directory, or the `TMPDIR` they were started with, is `directory` or lies beneath it. */
|
|
36
37
|
workingIn(directory: string): Promise<number[]>;
|
|
37
38
|
/** Every process on the host. */
|
|
38
39
|
table(): Promise<ProcessEntry[]>;
|
|
@@ -40,13 +41,17 @@ export interface ProcessHost {
|
|
|
40
41
|
}
|
|
41
42
|
/**
|
|
42
43
|
* The processes `host` reads. A lease belongs to one task and its command is gone by the time the
|
|
43
|
-
* lease ends, so what still works in it is what the command left behind.
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
44
|
+
* lease ends, so what still works in it is what the command left behind. Most of that works in the
|
|
45
|
+
* lease only through `TMPDIR`, never by working directory: a test's ttsc plugin build copies the
|
|
46
|
+
* typescript-go sources into `$TMPDIR` from a process working in its project, and a Bun or Node
|
|
47
|
+
* process writes its compile cache there as it exits. Neither is known to the task's pid, and
|
|
48
|
+
* whichever outlived that pid would write into a lease being removed (measured 2026-10-18: `rm`
|
|
49
|
+
* failing ENOTEMPTY under a ttsc copy whose task had died). A fixture's Nx daemon is another: Nx
|
|
50
|
+
* detaches it into a session of its own, so the process-group kill that ends a timed-out command
|
|
51
|
+
* never reaches it, and a test process that is killed runs no teardown of its own. Deleting the
|
|
52
|
+
* lease asks such a daemon to leave only by deleting its record, which it notices within a second
|
|
53
|
+
* when it is idle and not at all while it is wedged; a signal does not depend on the daemon
|
|
54
|
+
* noticing, and a process left on the volume is what keeps it from detaching.
|
|
50
55
|
*
|
|
51
56
|
* A leftover keeps running while it is looked for, and a child it starts after the process table
|
|
52
57
|
* was read is in no table that was. So each one found is held with SIGSTOP and the table is read
|
|
@@ -58,6 +63,8 @@ export interface ProcessHost {
|
|
|
58
63
|
* again what works in it, including when that table names nothing, to recover reparented children.
|
|
59
64
|
*/
|
|
60
65
|
export declare function leaseProcessesOn(host: ProcessHost): LeaseProcesses;
|
|
66
|
+
/** What works in `directory` on this host: by working directory, and by the `TMPDIR` it was started with. */
|
|
67
|
+
export declare function pidsInLease(directory: string): Promise<number[]>;
|
|
61
68
|
export declare const hostLeaseProcesses: LeaseProcesses;
|
|
62
69
|
export interface RamTempPaths {
|
|
63
70
|
/** Where DiskArbitration mounts the volume: `/Volumes/<volumeName>`. Short for sun_path (104 bytes). */
|
|
@@ -160,6 +167,8 @@ export declare class RamTempVolume {
|
|
|
160
167
|
private waitOut;
|
|
161
168
|
/** After a failed task: whether the volume ran out of space, which is then the error to report. */
|
|
162
169
|
fullness(lease: RamTempLease): Promise<RamTempError | null>;
|
|
170
|
+
/** The lease's volume and the space left on it now; `null` when the volume cannot be read. */
|
|
171
|
+
space(lease: RamTempLease): Promise<TempVolumeSpace | null>;
|
|
163
172
|
private lock;
|
|
164
173
|
private end;
|
|
165
174
|
private locked;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ram-temp.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/ram-temp.ts"],"names":[],"mappings":"AAqBA,OAAO,EAEL,KAAK,YAAY,
|
|
1
|
+
{"version":3,"file":"ram-temp.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/ram-temp.ts"],"names":[],"mappings":"AAqBA,OAAO,EAEL,KAAK,YAAY,EAMlB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAe,KAAK,UAAU,EAAgC,MAAM,iBAAiB,CAAC;AAU7F,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AA4GpD;;;;;;;GAOG;AACH,eAAO,MAAM,uBAAuB,QAA6C,CAAC;AAOlF,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oHAAoH;AACpH,MAAM,WAAW,YAAY;IAC3B,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;CACzF;AASD;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,aAAa,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACrD;AAED,gGAAgG;AAChG,MAAM,WAAW,WAAW;IAC1B,wHAAwH;IACxH,SAAS,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAChD,iCAAiC;IACjC,KAAK,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;CACnD;AAKD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,WAAW,GAAG,cAAc,CAsElE;AAED,6GAA6G;AAC7G,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAGtE;AAED,eAAO,MAAM,kBAAkB,EAAE,cAI/B,CAAC;AAEH,MAAM,WAAW,YAAY;IAC3B,wGAAwG;IACxG,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,kGAAkG;IAClG,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,IAAI,EAAE,MAAM,CAAC;CACd;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,YAAY,CASlF;AAED,wDAAwD;AACxD,MAAM,WAAW,OAAO;IACtB,qFAAqF;IACrF,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,+CAA+C;IAC/C,IAAI,EAAE,MAAM,CAAC;IACb,kDAAkD;IAClD,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,6EAA6E;IAC7E,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,kBAAkB;AAC5B;;;GAGG;AACD;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,KAAK,EAAE,YAAY,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE;AAC3E,sFAAsF;GACpF;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1C,wGAAwG;AACxG,KAAK,eAAe,GAAG;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAElF,MAAM,MAAM,YAAY,GACpB,eAAe;AACjB,oFAAoF;GAClF;IAAE,IAAI,EAAE,aAAa,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1F,MAAM,MAAM,MAAM,CAAC,CAAC,EAAE,CAAC,IAAI;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,CAAC,CAAA;CAAE,CAAC;AAE5E,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,YAAY,GAAG,MAAM,CAOhE;AAwID,qBAAa,aAAa;IAOtB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,GAAG;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,SAAS;IAX5B,qGAAqG;IACrG,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,qFAAqF;IACrF,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IAEpC,YACmB,KAAK,EAAE,YAAY,EACnB,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,YAAY,EACtB,GAAG,GAAE,MAAoB,EACzB,OAAO,GAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAwB,EAClD,SAAS,GAAE,cAAmC,EAIhE;IAED,2FAA2F;IACrF,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC,kBAAkB,EAAE,YAAY,CAAC,CAAC,CA2BjE;YAGa,IAAI;YAYJ,IAAI;YAsCJ,MAAM;YAoBN,KAAK;YA8DL,MAAM;IA6CpB;;;;;OAKG;IACG,IAAI,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAavE;IAED,0EAA0E;IACpE,OAAO,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC,CAiBtE;YAEa,YAAY;YA2DZ,OAAO;IAcrB,mGAAmG;IAC7F,QAAQ,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAOhE;IAED,8FAA8F;IACxF,KAAK,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAYhE;YAEa,IAAI;YAsBJ,GAAG;YAqBH,MAAM;YAkCN,UAAU;YAqCV,OAAO;YA+BP,KAAK;YA+CL,MAAM;YAgEN,SAAS;YAmBT,eAAe;YAgDf,SAAS;YA4DT,aAAa;YAYb,QAAQ;YAwBR,aAAa;YA6Cb,MAAM;YAkDN,aAAa;IAY3B;;;;;;OAMG;IACH,OAAO,CAAC,WAAW;CAUpB;AAmQD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAI5E;AAED,iGAAiG;AACjG,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAEnE;AAED,wFAAwF;AACxF,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAE3D;AAED;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,EAAE,CAiBlD;AAkBD,8FAA8F;AAC9F,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,UAAU,EAAE,MAAM,GACjB;IAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAAC,QAAQ,EAAE,WAAW,GAAG,cAAc,CAAA;CAAE,GAAG,IAAI,CAanE;AAED;;;;GAIG;AACH,eAAO,MAAM,sBAAsB,uBAAuB,CAAC;AAe3D;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,iBAAiB,GAAE,MAAiC,EACpD,aAAa,GAAE,MAAuB,EACtC,WAAW,GAAE,UAA4B,EACzC,OAAO,SAAmD,GACzD,YAAY,CAwCd"}
|
|
@@ -5,7 +5,7 @@ import { lstat, mkdir, mkdtemp, open, readdir, readFile, rename, rm, rmdir, stat
|
|
|
5
5
|
import { Socket } from 'node:net';
|
|
6
6
|
import { basename, join } from 'node:path';
|
|
7
7
|
import { setTimeout as sleep } from 'node:timers/promises';
|
|
8
|
-
import { awaitExit, pidsWorkingIn, processTable, terminate, withDescendants, } from '../../testing.js';
|
|
8
|
+
import { awaitExit, pidsWithTempIn, pidsWorkingIn, processTable, terminate, withDescendants, } from '../../testing.js';
|
|
9
9
|
import { isRecord } from '../../type-guards.js';
|
|
10
10
|
import { diskClassOf, recordedShare, takeDiskLease } from './disk-lease.js';
|
|
11
11
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
@@ -51,7 +51,11 @@ import { captureHeldVerdict, parseHeldVerdict, validateHeldVerdict, } from './ra
|
|
|
51
51
|
* checks that the volume, the lease directory and its own claim are still as
|
|
52
52
|
* it claimed them. A lease is a directory named after the task process's pid;
|
|
53
53
|
* a lease whose pid is gone is reclaimed by the next task, and the volume is
|
|
54
|
-
* detached when the last lease ends.
|
|
54
|
+
* detached when the last lease ends. The pid's death proves only that the task
|
|
55
|
+
* process is gone, not what it started: its command runs in a process group of
|
|
56
|
+
* its own, and every process started with the lease as its `TMPDIR` writes
|
|
57
|
+
* there wherever it works. So a dead lease is removed only once every process
|
|
58
|
+
* working in it, by working directory or by `TMPDIR`, has been stopped.
|
|
55
59
|
*
|
|
56
60
|
* The volume is created under its final name and never renamed. It is mounted by `mount_apfs` at a
|
|
57
61
|
* private staging directory and marked there; the kernel then unmounts it (`umount`) and `diskutil mount`, the
|
|
@@ -135,13 +139,17 @@ const LEFTOVER_GRACE_MS = 5_000;
|
|
|
135
139
|
const STARTS_NOTHING = /^[TtZ]/;
|
|
136
140
|
/**
|
|
137
141
|
* The processes `host` reads. A lease belongs to one task and its command is gone by the time the
|
|
138
|
-
* lease ends, so what still works in it is what the command left behind.
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
142
|
+
* lease ends, so what still works in it is what the command left behind. Most of that works in the
|
|
143
|
+
* lease only through `TMPDIR`, never by working directory: a test's ttsc plugin build copies the
|
|
144
|
+
* typescript-go sources into `$TMPDIR` from a process working in its project, and a Bun or Node
|
|
145
|
+
* process writes its compile cache there as it exits. Neither is known to the task's pid, and
|
|
146
|
+
* whichever outlived that pid would write into a lease being removed (measured 2026-10-18: `rm`
|
|
147
|
+
* failing ENOTEMPTY under a ttsc copy whose task had died). A fixture's Nx daemon is another: Nx
|
|
148
|
+
* detaches it into a session of its own, so the process-group kill that ends a timed-out command
|
|
149
|
+
* never reaches it, and a test process that is killed runs no teardown of its own. Deleting the
|
|
150
|
+
* lease asks such a daemon to leave only by deleting its record, which it notices within a second
|
|
151
|
+
* when it is idle and not at all while it is wedged; a signal does not depend on the daemon
|
|
152
|
+
* noticing, and a process left on the volume is what keeps it from detaching.
|
|
145
153
|
*
|
|
146
154
|
* A leftover keeps running while it is looked for, and a child it starts after the process table
|
|
147
155
|
* was read is in no table that was. So each one found is held with SIGSTOP and the table is read
|
|
@@ -219,8 +227,13 @@ export function leaseProcessesOn(host) {
|
|
|
219
227
|
},
|
|
220
228
|
};
|
|
221
229
|
}
|
|
230
|
+
/** What works in `directory` on this host: by working directory, and by the `TMPDIR` it was started with. */
|
|
231
|
+
export async function pidsInLease(directory) {
|
|
232
|
+
const [working, temp] = await Promise.all([pidsWorkingIn(directory), pidsWithTempIn(directory)]);
|
|
233
|
+
return [...new Set([...working, ...temp])];
|
|
234
|
+
}
|
|
222
235
|
export const hostLeaseProcesses = leaseProcessesOn({
|
|
223
|
-
workingIn:
|
|
236
|
+
workingIn: pidsInLease,
|
|
224
237
|
table: processTable,
|
|
225
238
|
signal: terminate,
|
|
226
239
|
});
|
|
@@ -566,12 +579,23 @@ export class RamTempVolume {
|
|
|
566
579
|
}
|
|
567
580
|
/** After a failed task: whether the volume ran out of space, which is then the error to report. */
|
|
568
581
|
async fullness(lease) {
|
|
582
|
+
const space = await this.space(lease);
|
|
583
|
+
if (space === null || space.freeBytes >= FULL_FREE_BYTES) {
|
|
584
|
+
return null;
|
|
585
|
+
}
|
|
586
|
+
const { mountpoint, capacityBytes, freeBytes } = space;
|
|
587
|
+
return { kind: 'volume-full', mountpoint, capacityBytes, freeBytes };
|
|
588
|
+
}
|
|
589
|
+
/** The lease's volume and the space left on it now; `null` when the volume cannot be read. */
|
|
590
|
+
async space(lease) {
|
|
569
591
|
try {
|
|
570
592
|
const fs = await statfs(lease.mountpoint);
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
:
|
|
593
|
+
return {
|
|
594
|
+
lease: lease.directory,
|
|
595
|
+
mountpoint: lease.mountpoint,
|
|
596
|
+
capacityBytes: lease.capacityBytes,
|
|
597
|
+
freeBytes: fs.bavail * fs.bsize,
|
|
598
|
+
};
|
|
575
599
|
}
|
|
576
600
|
catch {
|
|
577
601
|
return null;
|
|
@@ -760,9 +784,10 @@ export class RamTempVolume {
|
|
|
760
784
|
}
|
|
761
785
|
const reaped = [];
|
|
762
786
|
for (const candidate of candidates) {
|
|
763
|
-
// Its task is gone, so what works in it is what that task left: stopped before the lease
|
|
764
|
-
//
|
|
765
|
-
|
|
787
|
+
// Its task is gone, so what works in it is what that task left: stopped before the lease is
|
|
788
|
+
// removed, even where the removal is refused below. An empty lease is asked too: a leftover
|
|
789
|
+
// that has not written yet writes once it runs again. A task reaps its own (`reap`).
|
|
790
|
+
if (!candidate.own) {
|
|
766
791
|
reaped.push(...(await this.processes.stopWorkingIn(candidate.directory)));
|
|
767
792
|
}
|
|
768
793
|
}
|