@smoothbricks/nx-plugin 0.6.1 → 0.6.3-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +264 -55
- package/dist/.build-inputs.json +21 -15
- package/dist/bin/smoo-nx-publish.d.ts +3 -0
- package/dist/bin/smoo-nx-publish.d.ts.map +1 -0
- package/dist/bin/smoo-nx-publish.js +3 -0
- package/dist/cargo-retention-hook.d.ts +14 -3
- package/dist/cargo-retention-hook.d.ts.map +1 -1
- package/dist/cargo-retention-hook.js +25 -8
- package/dist/eslint-file-set.d.ts +4 -1
- package/dist/eslint-file-set.d.ts.map +1 -1
- package/dist/eslint-sources.d.ts +25 -0
- package/dist/eslint-sources.d.ts.map +1 -0
- package/dist/eslint-sources.js +99 -0
- package/dist/executors/all-verdicts/executor.d.ts +2 -1
- package/dist/executors/all-verdicts/executor.d.ts.map +1 -1
- package/dist/executors/all-verdicts/executor.js +48 -2
- package/dist/executors/all-verdicts/schema.d.ts +13 -0
- package/dist/executors/all-verdicts/schema.d.ts.map +1 -1
- package/dist/executors/all-verdicts/schema.js +7 -0
- package/dist/executors/bounded-exec/bounds.d.ts +57 -8
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +100 -11
- package/dist/executors/bounded-exec/executor.d.ts +2 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +270 -27
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +2 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +27 -3
- package/dist/executors/bounded-exec/mbx-pool.d.ts +20 -3
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +84 -4
- package/dist/executors/bounded-exec/mbx-queue.d.ts +30 -0
- package/dist/executors/bounded-exec/mbx-queue.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-queue.js +229 -0
- package/dist/executors/bounded-exec/progress.d.ts +9 -2
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +148 -22
- package/dist/executors/bounded-exec/test-result-writer.d.ts +30 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -0
- package/dist/executors/bounded-exec/test-result-writer.js +126 -0
- package/dist/executors/bounded-exec/test-results.d.ts +33 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -0
- package/dist/executors/bounded-exec/test-results.js +33 -0
- package/dist/executors/bounded-exec/verdict.d.ts +20 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +102 -4
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +67 -65
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +6 -0
- package/dist/nx-publish.d.ts +24 -0
- package/dist/nx-publish.d.ts.map +1 -0
- package/dist/nx-publish.js +150 -0
- package/dist/runtime-input-policy.d.ts +14 -0
- package/dist/runtime-input-policy.d.ts.map +1 -1
- package/dist/runtime-input-policy.js +1 -1
- package/dist/tool-bins.d.ts +6 -8
- package/dist/tool-bins.d.ts.map +1 -1
- package/dist/tool-bins.js +7 -5
- package/dist/workspace-index.d.ts +7 -0
- package/dist/workspace-index.d.ts.map +1 -1
- package/dist/workspace-index.js +11 -0
- package/executors.json +1 -1
- package/managed/raw/envrc +15 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +75 -0
- package/managed/raw/tooling/devenv +1 -2
- package/managed/raw/tooling/direnv/developer-links.ts +15 -1
- package/managed/raw/tooling/direnv/devenv.smoo.nix +45 -13
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +20 -10
- package/managed/raw/tooling/direnv/linked-snapshots.ts +50 -13
- package/managed/raw/tooling/direnv/setup-environment.ts +36 -9
- package/managed/raw/tooling/direnv/ttsc-env.sh +8 -6
- package/managed/raw/tooling/git-hooks/pre-push.sh +4 -1
- package/managed/raw/tooling/git-reword-commit +151 -0
- package/managed/templates/github/actions/cache-node-modules/action.yml +1 -1
- package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +12 -11
- package/package.json +226 -10
- package/runtime/nx-publish/cache_adapter.js +45 -0
- package/runtime/nx-publish/nx_publish.js +60 -0
- package/runtime-input.sh +12 -3
- package/src/bin/smoo-nx-publish.ts +4 -0
- package/src/cargo-retention-hook.ts +28 -11
- package/src/eslint-file-set.ts +4 -1
- package/src/eslint-sources.ts +134 -0
- package/src/executors/all-verdicts/executor.ts +58 -3
- package/src/executors/all-verdicts/schema.json +4 -0
- package/src/executors/all-verdicts/schema.ts +14 -0
- package/src/executors/bounded-exec/bounds.ts +132 -15
- package/src/executors/bounded-exec/executor.ts +303 -27
- package/src/executors/bounded-exec/hang-diagnostics.ts +31 -3
- package/src/executors/bounded-exec/mbx-pool.ts +82 -5
- package/src/executors/bounded-exec/mbx-queue.ts +170 -0
- package/src/executors/bounded-exec/progress.ts +168 -15
- package/src/executors/bounded-exec/test-result-writer.ts +152 -0
- package/src/executors/bounded-exec/test-results.ts +39 -0
- package/src/executors/bounded-exec/verdict.ts +108 -5
- package/src/index.ts +74 -70
- package/src/managed-files/files.ts +6 -0
- package/src/nx-publish.ts +157 -0
- package/src/runtime-input-policy.ts +1 -1
- package/src/tool-bins.ts +13 -9
- package/src/workspace-index.ts +12 -0
- package/test-timings.json +8 -2
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type InheritedCpuPool } from './cpu-pool.js';
|
|
2
2
|
import { captureHangDiagnostics } from './hang-diagnostics.js';
|
|
3
|
+
import { snapshotMbxQueue } from './mbx-queue.js';
|
|
3
4
|
import { type SubtreeOwner } from './owned-subtree.js';
|
|
4
5
|
import { RamTempVolume } from './ram-temp.js';
|
|
5
6
|
import type { BoundedExecOptions } from './schema.js';
|
|
@@ -69,6 +70,6 @@ export declare function recordContext(context: NxTaskContext): BoundedExecContex
|
|
|
69
70
|
* connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
|
|
70
71
|
* command runs on over budget to its own exit, and the run says so.
|
|
71
72
|
*/
|
|
72
|
-
export declare function runBoundedExec(options: BoundedExecOptions, context: BoundedExecContext, killer: ProcessTreeKiller, tempVolume: TempVolume | null, inherited?: InheritedCpuPool | null, captureDiagnostics?: typeof captureHangDiagnostics, owner?: SubtreeOwner | null): Promise<BoundedExecResult>;
|
|
73
|
+
export declare function runBoundedExec(options: BoundedExecOptions, context: BoundedExecContext, killer: ProcessTreeKiller, tempVolume: TempVolume | null, inherited?: InheritedCpuPool | null, captureDiagnostics?: typeof captureHangDiagnostics, owner?: SubtreeOwner | null, queueHost?: Parameters<typeof snapshotMbxQueue>[3]): Promise<BoundedExecResult>;
|
|
73
74
|
export declare function createProcessTreeKiller(): ProcessTreeKiller;
|
|
74
75
|
//# sourceMappingURL=executor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AA0BA,OAAO,EAIL,KAAK,gBAAgB,EAItB,MAAM,eAAe,CAAC;AAGvB,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAIhF,OAAO,EAA4C,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAE5F,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EASL,aAAa,EAEd,MAAM,eAAe,CAAC;AAIvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAGL,KAAK,eAAe,EAerB,MAAM,cAAc,CAAC;AAuBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;CACxB;AAkBD,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,aAAa,EACb,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,CAC3E,CAAC;AAQF;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAc5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,KAAK,UAAU,GAAG,IAAI,GACzD,OAAO,CAAC,iBAAiB,CAAC,CAsB5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,EAC1E,KAAK,GAAE,YAAY,GAAG,IAAW,EACjC,SAAS,CAAC,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC,CAAC,CAAC,GACjD,OAAO,CAAC,iBAAiB,CAAC,CAihC5B;AAED,wBAAgB,uBAAuB,IAAI,iBAAiB,CAM3D"}
|
|
@@ -7,19 +7,21 @@ import { createInterface } from 'node:readline';
|
|
|
7
7
|
import { Duplex, Readable } from 'node:stream';
|
|
8
8
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
9
9
|
import { DEADLINE_PATIENCE, describeMembersLeft, killProcessGroup, processGroupHasMembers, signalProcessGroup, } from '../../process-group.js';
|
|
10
|
-
import { earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
10
|
+
import { admissionWaitMs, earliestBound, NO_ADMISSION_WAIT, stepAdmissionWait, stepBounds, UNARMED, } from './bounds.js';
|
|
11
11
|
import { CPU_POOL_ENV, createCpuPool, readInheritedCpuPool, takeParentCpuTokens, } from './cpu-pool.js';
|
|
12
12
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
13
13
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
14
|
-
import { mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
|
|
14
|
+
import { MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
|
|
15
|
+
import { MBX_QUEUE_POLL_MS, snapshotMbxQueue } from './mbx-queue.js';
|
|
15
16
|
import { createNoSpaceWatch } from './no-space.js';
|
|
16
17
|
import { COWSHED_EXECUTABLE_ENV, cowshedSubtreeOwner, HANDLE_FD, readAdmission, SUPERVISOR_SOCKET_ENV, } from './owned-subtree.js';
|
|
17
|
-
import { createRunProgress, withProgressReport } from './progress.js';
|
|
18
|
+
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
18
19
|
import { createHostCommands, describeCoverageGap, describeRamTempError, describeSignalled, LEFTOVER_GRACE_MS, PRODUCTION, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
|
|
19
20
|
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
20
21
|
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
|
|
21
22
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
22
|
-
import {
|
|
23
|
+
import { createTestResultWriter } from './test-result-writer.js';
|
|
24
|
+
import { BUN_TEST, clearTaskDirectory, describeAdmissionQueue, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
23
25
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
24
26
|
const REAP_AFTER_FORCE_KILL_MS = 2_000;
|
|
25
27
|
const EXIT_CODE_BY_SIGNAL = {
|
|
@@ -142,7 +144,7 @@ export function recordContext(context) {
|
|
|
142
144
|
* connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
|
|
143
145
|
* command runs on over budget to its own exit, and the run says so.
|
|
144
146
|
*/
|
|
145
|
-
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null) {
|
|
147
|
+
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null, queueHost) {
|
|
146
148
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
147
149
|
const directory = context.record?.directory ?? null;
|
|
148
150
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
@@ -157,6 +159,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
157
159
|
? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
|
|
158
160
|
: null;
|
|
159
161
|
const outputChunks = [];
|
|
162
|
+
let resultWriter = null;
|
|
163
|
+
let resultFailure = null;
|
|
160
164
|
const state = {
|
|
161
165
|
settled: false,
|
|
162
166
|
expiry: null,
|
|
@@ -404,18 +408,58 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
404
408
|
})();
|
|
405
409
|
return cpuReleased;
|
|
406
410
|
};
|
|
411
|
+
let mbxAdmitted = false;
|
|
412
|
+
let admissionWait = NO_ADMISSION_WAIT;
|
|
413
|
+
let lastQueueRequests = [];
|
|
414
|
+
let queueFailure = null;
|
|
415
|
+
const queueWait = () => mbxAdmitted || lastQueueRequests.length === 0
|
|
416
|
+
? null
|
|
417
|
+
: {
|
|
418
|
+
waitedMs: admissionWaitMs(admissionWait, performance.now()),
|
|
419
|
+
requests: lastQueueRequests,
|
|
420
|
+
};
|
|
421
|
+
const queueAbort = new AbortController();
|
|
422
|
+
let queueTimer;
|
|
423
|
+
let queueSnapshot = null;
|
|
424
|
+
const stopQueueObservation = () => {
|
|
425
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
|
|
426
|
+
clearInterval(queueTimer);
|
|
427
|
+
queueAbort.abort();
|
|
428
|
+
};
|
|
407
429
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
408
|
-
//
|
|
409
|
-
// fired, so it is never judged as a bound.
|
|
430
|
+
// Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
|
|
410
431
|
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits) => {
|
|
411
432
|
if (context.record === undefined) {
|
|
412
433
|
return;
|
|
413
434
|
}
|
|
414
435
|
const { task, directory: recordDirectory } = context.record;
|
|
415
|
-
const report = await readJunitReport(recordDirectory)
|
|
416
|
-
|
|
436
|
+
const report = await readJunitReport(recordDirectory, requested.search(BUN_TEST) !== -1 && resultWriter !== null
|
|
437
|
+
? (results) => {
|
|
438
|
+
const error = resultWriter?.reconcile(results) ?? null;
|
|
439
|
+
if (error !== null) {
|
|
440
|
+
resultFailure ??= error;
|
|
441
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
: undefined).finally(async () => {
|
|
445
|
+
const error = (await resultWriter?.close()) ?? null;
|
|
446
|
+
if (error !== null) {
|
|
447
|
+
resultFailure ??= error;
|
|
448
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
449
|
+
}
|
|
450
|
+
});
|
|
451
|
+
const admissionQueue = queueWait();
|
|
452
|
+
const verdict = operational || resultFailure !== null
|
|
417
453
|
? judgeOperationalFailure(exitCode, report)
|
|
418
|
-
: judgeRun({
|
|
454
|
+
: judgeRun({
|
|
455
|
+
exitCode,
|
|
456
|
+
elapsedMs,
|
|
457
|
+
expiry,
|
|
458
|
+
report,
|
|
459
|
+
noSpace,
|
|
460
|
+
lockWaits,
|
|
461
|
+
admissionQueue,
|
|
462
|
+
});
|
|
419
463
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
420
464
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
421
465
|
}
|
|
@@ -448,9 +492,10 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
448
492
|
}
|
|
449
493
|
pool = created.pool;
|
|
450
494
|
}
|
|
495
|
+
const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
|
|
451
496
|
const command = directory === null
|
|
452
|
-
? withProgressReport(
|
|
453
|
-
: await withJunitReport(withProgressReport(
|
|
497
|
+
? withProgressReport(native.command)
|
|
498
|
+
: await withJunitReport(withProgressReport(native.command), directory);
|
|
454
499
|
// A runner bounded-exec cannot see in the command (a script that spawns `bun test`) writes its JUnit report to this
|
|
455
500
|
// path when it is set. The grant's sizing outranks the target's own environment: the runner may use what it was
|
|
456
501
|
// granted, not what it was configured with. The sub-pool's environment comes last: a descendant reads only the pool
|
|
@@ -459,7 +504,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
459
504
|
...(directory === null ? {} : { [JUNIT_REPORT_ENV]: join(directory, JUNIT_FILE) }),
|
|
460
505
|
...targetEnv(options.env, appendStderr),
|
|
461
506
|
...sized.env,
|
|
507
|
+
...native.env,
|
|
462
508
|
...(pool?.environment ?? {}),
|
|
509
|
+
...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
|
|
463
510
|
};
|
|
464
511
|
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
465
512
|
// of the command: the progress bound runs during it and its own ceiling does not.
|
|
@@ -476,25 +523,36 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
476
523
|
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
477
524
|
const unadmitted = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
|
|
478
525
|
const shellCommand = leaf === null ? unadmitted : withMbxLeaf(unadmitted, leaf);
|
|
526
|
+
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
527
|
+
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
528
|
+
// fd 4 is unused without a subtree admission, so the grant takes fd 5 either way.
|
|
529
|
+
const grantSlots = leaf !== null ? ['ignore', 'pipe'] : [];
|
|
530
|
+
const rootGrantSlot = leaf !== null ? ['pipe'] : [];
|
|
479
531
|
const root = owner?.rootArgv(shellCommand);
|
|
480
532
|
let child;
|
|
533
|
+
const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
|
|
534
|
+
if (results !== null && !results.ok)
|
|
535
|
+
return refuse(`bounded-exec: ${results.error}\n`);
|
|
536
|
+
resultWriter = results?.ok ? results.writer : null;
|
|
481
537
|
try {
|
|
482
538
|
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
483
|
-
//
|
|
539
|
+
// cowshed closes fd 4 before exec; fd 5 carries mbx's admission/release stream, including Cargo descendants.
|
|
484
540
|
child =
|
|
485
541
|
root === undefined
|
|
486
542
|
? spawn(shellCommand, [], {
|
|
487
543
|
cwd,
|
|
488
544
|
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
489
545
|
shell: true,
|
|
490
|
-
stdio: hooked === null
|
|
546
|
+
stdio: hooked === null && leaf === null
|
|
547
|
+
? 'pipe'
|
|
548
|
+
: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
|
|
491
549
|
detached: process.platform !== 'win32',
|
|
492
550
|
windowsHide: true,
|
|
493
551
|
})
|
|
494
552
|
: spawn(root.file, root.args, {
|
|
495
553
|
cwd,
|
|
496
554
|
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
497
|
-
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
|
|
555
|
+
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot],
|
|
498
556
|
detached: process.platform !== 'win32',
|
|
499
557
|
windowsHide: true,
|
|
500
558
|
});
|
|
@@ -518,7 +576,17 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
518
576
|
});
|
|
519
577
|
}
|
|
520
578
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
521
|
-
const progress = createRunProgress(command
|
|
579
|
+
const progress = createRunProgress(command, resultWriter === null
|
|
580
|
+
? undefined
|
|
581
|
+
: (result) => {
|
|
582
|
+
const error = resultWriter?.append(result) ?? null;
|
|
583
|
+
if (error !== null && resultFailure === null) {
|
|
584
|
+
resultFailure = error;
|
|
585
|
+
lifecycleFailure ??= error;
|
|
586
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
587
|
+
termination = terminate();
|
|
588
|
+
}
|
|
589
|
+
});
|
|
522
590
|
// A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
|
|
523
591
|
// since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
|
|
524
592
|
const noSpace = createNoSpaceWatch();
|
|
@@ -581,8 +649,48 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
581
649
|
: kind === 'fixture'
|
|
582
650
|
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
583
651
|
: { kind, limitMs: timeoutMs ?? 0 };
|
|
652
|
+
let checkingIdle = false;
|
|
653
|
+
const armBoundTimer = () => {
|
|
654
|
+
clearTimeout(boundTimer);
|
|
655
|
+
if (state.settled || state.expiry !== null || queueFailure !== null)
|
|
656
|
+
return;
|
|
657
|
+
const next = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
|
|
658
|
+
if (next !== null) {
|
|
659
|
+
boundTimer = setTimeout(() => {
|
|
660
|
+
void confirmExpiry(next.kind, next.at);
|
|
661
|
+
}, Math.max(0, next.at - Date.now()));
|
|
662
|
+
}
|
|
663
|
+
};
|
|
664
|
+
const confirmExpiry = async (kind, at) => {
|
|
665
|
+
if (state.settled || state.expiry !== null || queueFailure !== null)
|
|
666
|
+
return;
|
|
667
|
+
const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
|
|
668
|
+
if (scheduled?.kind !== kind || scheduled.at !== at)
|
|
669
|
+
return;
|
|
670
|
+
if (kind === 'idle' && leaf !== null) {
|
|
671
|
+
if (checkingIdle)
|
|
672
|
+
return;
|
|
673
|
+
checkingIdle = true;
|
|
674
|
+
armBoundTimer();
|
|
675
|
+
try {
|
|
676
|
+
// Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
|
|
677
|
+
// ceilings remain armed while this observation runs.
|
|
678
|
+
await refreshQueue(true);
|
|
679
|
+
}
|
|
680
|
+
finally {
|
|
681
|
+
checkingIdle = false;
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
if (state.settled || state.expiry !== null || queueFailure !== null)
|
|
685
|
+
return;
|
|
686
|
+
const next = earliestBound(bounds);
|
|
687
|
+
if (next !== null && next.at <= Date.now())
|
|
688
|
+
expire(expiredAs(next.kind));
|
|
689
|
+
else
|
|
690
|
+
armBoundTimer();
|
|
691
|
+
};
|
|
584
692
|
const stepTo = (event) => {
|
|
585
|
-
if (state.settled || state.expiry !== null)
|
|
693
|
+
if (state.settled || state.expiry !== null || queueFailure !== null)
|
|
586
694
|
return;
|
|
587
695
|
const idleBefore = bounds.idle;
|
|
588
696
|
bounds = stepBounds(bounds, {
|
|
@@ -596,10 +704,67 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
596
704
|
if (bounds.idle !== null)
|
|
597
705
|
scheduleCapture(bounds.idle);
|
|
598
706
|
}
|
|
707
|
+
armBoundTimer();
|
|
708
|
+
};
|
|
709
|
+
let lastQueueDiagnostic = '';
|
|
710
|
+
let lastQueueDiagnosticAt = null;
|
|
711
|
+
let lastSnapshotDiagnostics = '';
|
|
712
|
+
const failQueueObservation = (detail) => {
|
|
713
|
+
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || queueFailure !== null)
|
|
714
|
+
return;
|
|
715
|
+
queueFailure = detail;
|
|
716
|
+
tell(`bounded-exec: mbx admission snapshot failed: ${detail}; queue/admission state unconfirmed, not an idle wedge\n`);
|
|
599
717
|
clearTimeout(boundTimer);
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
718
|
+
cancelCapture();
|
|
719
|
+
stopQueueObservation();
|
|
720
|
+
termination = terminate();
|
|
721
|
+
};
|
|
722
|
+
const refreshQueue = async (fresh) => {
|
|
723
|
+
if (fresh && queueSnapshot !== null)
|
|
724
|
+
await queueSnapshot;
|
|
725
|
+
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined)
|
|
726
|
+
return;
|
|
727
|
+
const pid = child.pid;
|
|
728
|
+
queueSnapshot ??= (async () => {
|
|
729
|
+
const snapshot = await snapshotMbxQueue(pid, mergeEnv(lease?.directory ?? ordinaryTemp, env), queueAbort.signal, queueHost);
|
|
730
|
+
if (state.settled || state.expiry !== null || queueAbort.signal.aborted)
|
|
731
|
+
return;
|
|
732
|
+
if (!snapshot.ok) {
|
|
733
|
+
failQueueObservation(snapshot.detail);
|
|
734
|
+
return;
|
|
735
|
+
}
|
|
736
|
+
const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
|
|
737
|
+
if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics)
|
|
738
|
+
tell(`bounded-exec: ${diagnostics}\n`);
|
|
739
|
+
lastSnapshotDiagnostics = diagnostics;
|
|
740
|
+
const waiting = snapshot.requests.length > 0;
|
|
741
|
+
const observedAt = Date.now();
|
|
742
|
+
const observedWaitAt = performance.now();
|
|
743
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
|
|
744
|
+
if (waiting) {
|
|
745
|
+
lastQueueRequests = snapshot.requests;
|
|
746
|
+
const positions = JSON.stringify(snapshot.requests);
|
|
747
|
+
if (positions !== lastQueueDiagnostic ||
|
|
748
|
+
lastQueueDiagnosticAt === null ||
|
|
749
|
+
Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS) {
|
|
750
|
+
const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
|
|
751
|
+
tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
|
|
752
|
+
lastQueueDiagnostic = positions;
|
|
753
|
+
lastQueueDiagnosticAt = Date.now();
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
else {
|
|
757
|
+
lastQueueDiagnostic = '';
|
|
758
|
+
}
|
|
759
|
+
stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
|
|
760
|
+
})()
|
|
761
|
+
.catch((error) => {
|
|
762
|
+
failQueueObservation(error instanceof Error ? error.message : String(error));
|
|
763
|
+
})
|
|
764
|
+
.finally(() => {
|
|
765
|
+
queueSnapshot = null;
|
|
766
|
+
});
|
|
767
|
+
await queueSnapshot;
|
|
603
768
|
};
|
|
604
769
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
605
770
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
@@ -611,6 +776,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
611
776
|
if (!fixtureDone)
|
|
612
777
|
return;
|
|
613
778
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
779
|
+
const error = progress.error();
|
|
780
|
+
if (error !== null && lifecycleFailure === null) {
|
|
781
|
+
lifecycleFailure = error;
|
|
782
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
783
|
+
termination = terminate();
|
|
784
|
+
}
|
|
614
785
|
};
|
|
615
786
|
if (lifecycle !== null) {
|
|
616
787
|
const channel = child.stdio[3];
|
|
@@ -652,7 +823,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
652
823
|
}
|
|
653
824
|
child.stdout?.on('data', (chunk) => {
|
|
654
825
|
const text = chunk.toString();
|
|
655
|
-
scanProgress('stdout',
|
|
826
|
+
scanProgress('stdout', chunk);
|
|
656
827
|
const said = noSpace.scan('stdout', text);
|
|
657
828
|
if (said !== null)
|
|
658
829
|
state.noSpace = sight(said);
|
|
@@ -660,7 +831,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
660
831
|
});
|
|
661
832
|
child.stderr?.on('data', (chunk) => {
|
|
662
833
|
const text = chunk.toString();
|
|
663
|
-
scanProgress('stderr',
|
|
834
|
+
scanProgress('stderr', chunk);
|
|
664
835
|
const said = noSpace.scan('stderr', text);
|
|
665
836
|
if (said !== null)
|
|
666
837
|
state.noSpace = sight(said);
|
|
@@ -684,12 +855,24 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
684
855
|
const onProcessExit = () => {
|
|
685
856
|
void killChildTree();
|
|
686
857
|
};
|
|
858
|
+
let signalCleanup = null;
|
|
687
859
|
const onTerminationSignal = (signal) => {
|
|
688
860
|
removeSignalHandlers();
|
|
689
861
|
cancelCapture();
|
|
690
|
-
|
|
862
|
+
clearTimeout(boundTimer);
|
|
863
|
+
// Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
|
|
864
|
+
// reports an operational failure, never a manufactured queue or admission.
|
|
865
|
+
signalCleanup = (leaf === null ? Promise.resolve() : refreshQueue(false))
|
|
866
|
+
.finally(() => {
|
|
867
|
+
stopQueueObservation();
|
|
868
|
+
return killChildTree();
|
|
869
|
+
})
|
|
691
870
|
.finally(releaseCpu)
|
|
692
871
|
.finally(() => releaseLease(true))
|
|
872
|
+
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null ||
|
|
873
|
+
lifecycleFailure !== null ||
|
|
874
|
+
state.retireError !== null ||
|
|
875
|
+
queueFailure !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits))
|
|
693
876
|
.finally(() => process.kill(process.pid, signal));
|
|
694
877
|
};
|
|
695
878
|
const onSigint = () => onTerminationSignal('SIGINT');
|
|
@@ -713,10 +896,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
713
896
|
child.once('error', (error) => {
|
|
714
897
|
appendStderr(`${error.message}\n`);
|
|
715
898
|
state.settled = true;
|
|
899
|
+
stopQueueObservation();
|
|
716
900
|
resolve({ code: 1, signal: null });
|
|
717
901
|
});
|
|
718
902
|
child.once(hooked === null ? 'exit' : 'close', (code, signal) => {
|
|
719
903
|
state.settled = true;
|
|
904
|
+
stopQueueObservation();
|
|
720
905
|
resolve({ code, signal });
|
|
721
906
|
});
|
|
722
907
|
});
|
|
@@ -815,7 +1000,55 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
815
1000
|
};
|
|
816
1001
|
// The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
|
|
817
1002
|
// fire on a command the fixture already handed over to.
|
|
818
|
-
stepTo({
|
|
1003
|
+
stepTo({
|
|
1004
|
+
kind: 'spawn',
|
|
1005
|
+
at: Date.now(),
|
|
1006
|
+
testing: isTesting(),
|
|
1007
|
+
fixture: fixture !== null && lifecycle !== null,
|
|
1008
|
+
queued,
|
|
1009
|
+
});
|
|
1010
|
+
// Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
|
|
1011
|
+
if (leaf !== null) {
|
|
1012
|
+
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1013
|
+
if (!(channel instanceof Readable))
|
|
1014
|
+
throw new Error('an mbx leaf requires its grant descriptor');
|
|
1015
|
+
let said = false;
|
|
1016
|
+
const failGrant = (detail) => {
|
|
1017
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
1018
|
+
return;
|
|
1019
|
+
lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
|
|
1020
|
+
appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
|
|
1021
|
+
termination = terminate();
|
|
1022
|
+
};
|
|
1023
|
+
channel.on('data', (chunk) => {
|
|
1024
|
+
for (const byte of chunk) {
|
|
1025
|
+
if (byte === 103) {
|
|
1026
|
+
if (!said) {
|
|
1027
|
+
mbxAdmitted = true;
|
|
1028
|
+
said = true;
|
|
1029
|
+
tell(leaf.kind === 'reserve'
|
|
1030
|
+
? `cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms cpus=${leaf.cpus} priority=${leaf.priority}\n`
|
|
1031
|
+
: `cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1032
|
+
}
|
|
1033
|
+
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1034
|
+
}
|
|
1035
|
+
else if (byte === 114) {
|
|
1036
|
+
stepTo({ kind: 'released', at: Date.now() });
|
|
1037
|
+
}
|
|
1038
|
+
else {
|
|
1039
|
+
failGrant(`unexpected byte ${byte}`);
|
|
1040
|
+
return;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
});
|
|
1044
|
+
channel.on('error', (error) => failGrant(error.message));
|
|
1045
|
+
}
|
|
1046
|
+
if (leaf !== null) {
|
|
1047
|
+
queueTimer = setInterval(() => {
|
|
1048
|
+
void refreshQueue(false);
|
|
1049
|
+
}, MBX_QUEUE_POLL_MS);
|
|
1050
|
+
void refreshQueue(false);
|
|
1051
|
+
}
|
|
819
1052
|
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
820
1053
|
// under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
|
|
821
1054
|
// else; only a bound or a signal stops the command. Our own release after the command settled is not a loss;
|
|
@@ -830,6 +1063,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
830
1063
|
});
|
|
831
1064
|
}
|
|
832
1065
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
1066
|
+
if (signalCleanup !== null) {
|
|
1067
|
+
await signalCleanup;
|
|
1068
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
1069
|
+
}
|
|
1070
|
+
stopQueueObservation();
|
|
1071
|
+
await queueSnapshot;
|
|
833
1072
|
await termination;
|
|
834
1073
|
clearTimeout(boundTimer);
|
|
835
1074
|
cancelCapture();
|
|
@@ -859,13 +1098,17 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
859
1098
|
appendStderr(`${describeRamTempError(full)}\n`);
|
|
860
1099
|
}
|
|
861
1100
|
}
|
|
862
|
-
await releaseLease(code !== 0 ||
|
|
1101
|
+
await releaseLease(code !== 0 ||
|
|
1102
|
+
state.expiry !== null ||
|
|
1103
|
+
state.poolCloseError !== null ||
|
|
1104
|
+
lifecycleFailure !== null ||
|
|
1105
|
+
queueFailure !== null);
|
|
863
1106
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
864
1107
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
865
|
-
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1108
|
+
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null || queueFailure !== null;
|
|
866
1109
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
867
1110
|
return {
|
|
868
|
-
success: !operationalFailure && state.expiry === null && code === 0,
|
|
1111
|
+
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
869
1112
|
terminalOutput: outputChunks.join(''),
|
|
870
1113
|
};
|
|
871
1114
|
}
|
|
@@ -49,4 +49,6 @@ export declare function captureHangDiagnostics(request: HangCaptureRequest, sour
|
|
|
49
49
|
* waits here, and the kernel lists it as a member of its own group until then, so a group left listed is expected.
|
|
50
50
|
*/
|
|
51
51
|
export declare function stopProbe(file: string, pid: number): boolean;
|
|
52
|
+
/** A probe owns its group and output pipes; its deadline never waits for an unresponsive child to reap. */
|
|
53
|
+
export declare function runProbe(file: string, args: readonly string[], timeoutMs: number, signal: AbortSignal, environment?: NodeJS.ProcessEnv): Promise<HangProbeResult>;
|
|
52
54
|
//# sourceMappingURL=hang-diagnostics.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hang-diagnostics.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/hang-diagnostics.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,QAAQ,EAKd,MAAM,gBAAgB,CAAC;AAExB;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"hang-diagnostics.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/hang-diagnostics.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,QAAQ,EAKd,MAAM,gBAAgB,CAAC;AAExB;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAoCtC,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE,SAAS,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3C,0HAA0H;IAC1H,SAAS,EAAE,SAAS,QAAQ,EAAE,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,eAAe,GACvB;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAClE;IAAE,MAAM,EAAE,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvG,qGAAqG;AACrG,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC1B,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CAChH;AAUD,0FAA0F;AAC1F,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,kBAAkB,EAC3B,MAAM,GAAE,kBAAyB,GAChC,OAAO,CAAC,eAAe,CAAC,CAgR1B;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAM5D;AAED,2GAA2G;AAC3G,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,WAAW,EACnB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,GAC9B,OAAO,CAAC,eAAe,CAAC,CA6D1B"}
|
|
@@ -17,6 +17,8 @@ const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,lstart=,command=';
|
|
|
17
17
|
*/
|
|
18
18
|
const MAX_SAMPLED_LEAVES = 4;
|
|
19
19
|
const SAMPLE_SECONDS = 1;
|
|
20
|
+
/** How long a process in an uninterruptible kernel wait has its file-system calls traced. */
|
|
21
|
+
const FS_USAGE_SECONDS = 5;
|
|
20
22
|
/**
|
|
21
23
|
* A process census's limit. The stack samples run until the closing census is this far from the window's end.
|
|
22
24
|
*/
|
|
@@ -173,6 +175,28 @@ export function captureHangDiagnostics(request, source = host) {
|
|
|
173
175
|
lines.push(`${name}: no stack; sudo -n sample did not answer, so the call this process waits in is not observed.`);
|
|
174
176
|
}));
|
|
175
177
|
};
|
|
178
|
+
/**
|
|
179
|
+
* A process in an uninterruptible kernel wait (state U) is in a file-system or device call no signal ends, and its
|
|
180
|
+
* stack stops at the system call. Its file-system calls over a few seconds (fs_usage) name the volume and files it
|
|
181
|
+
* waits on, which tells a fixture doing its I/O on a contended shared volume from one serialised behind one APFS
|
|
182
|
+
* operation. Bounded like the stack samples; one fs_usage sudo refuses is named as no trace.
|
|
183
|
+
*/
|
|
184
|
+
const kernelWaits = async (tree) => {
|
|
185
|
+
const waiting = tree.filter((entry) => entry.stat.startsWith('U')).slice(0, MAX_SAMPLED_LEAVES);
|
|
186
|
+
const limitMs = endsAtMs - Date.now() - CENSUS_LIMIT_MS;
|
|
187
|
+
await Promise.all(waiting.map(async ({ pid }) => {
|
|
188
|
+
const name = `fs_usage pid ${pid}`;
|
|
189
|
+
const args = ['-n', 'fs_usage', '-w', '-t', String(FS_USAGE_SECONDS), '-f', 'filesys', String(pid)];
|
|
190
|
+
const result = await probe(name, 'sudo', args, limitMs);
|
|
191
|
+
if (finished)
|
|
192
|
+
return;
|
|
193
|
+
if (result.stdout.trim() !== '') {
|
|
194
|
+
lines.push(`${name} (${FS_USAGE_SECONDS}s of file-system calls):\n${result.stdout.trimEnd()}`);
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
lines.push(`${name}: no trace; sudo -n fs_usage gave none, so the files this process waits on are not observed.`);
|
|
198
|
+
}));
|
|
199
|
+
};
|
|
176
200
|
const collect = async () => {
|
|
177
201
|
const before = await census('ps before');
|
|
178
202
|
if (finished)
|
|
@@ -227,7 +251,7 @@ export function captureHangDiagnostics(request, source = host) {
|
|
|
227
251
|
}
|
|
228
252
|
}
|
|
229
253
|
};
|
|
230
|
-
await Promise.all([openFiles(), stackSamples(before)]);
|
|
254
|
+
await Promise.all([openFiles(), stackSamples(before), kernelWaits(before)]);
|
|
231
255
|
if (finished)
|
|
232
256
|
return;
|
|
233
257
|
}
|
|
@@ -282,14 +306,14 @@ export function stopProbe(file, pid) {
|
|
|
282
306
|
return killProcessGroupNow(pid, TEARDOWN_PATIENCE);
|
|
283
307
|
}
|
|
284
308
|
/** A probe owns its group and output pipes; its deadline never waits for an unresponsive child to reap. */
|
|
285
|
-
function runProbe(file, args, timeoutMs, signal) {
|
|
309
|
+
export function runProbe(file, args, timeoutMs, signal, environment) {
|
|
286
310
|
// Keep the package's es2022-compatible event-completion convention.
|
|
287
311
|
return new Promise((resolve) => {
|
|
288
312
|
if (signal.aborted) {
|
|
289
313
|
resolve({ status: 'timed-out', detail: 'capture deadline already reached', stdout: '', stderr: '' });
|
|
290
314
|
return;
|
|
291
315
|
}
|
|
292
|
-
const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
316
|
+
const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: environment });
|
|
293
317
|
let stdout = '';
|
|
294
318
|
let stderr = '';
|
|
295
319
|
let bytes = 0;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type RunnerKind } from './runner-sizing.js';
|
|
2
2
|
/**
|
|
3
3
|
* mbx's machine-wide pool as a leaf runner meets it (specs/cowshed/05_gateway.md, "Host scheduling", "Parallelism inside
|
|
4
4
|
* a job"). With the mbx capability the canonical environment names the one pool (`MBX_SCHEDULER=1` beside the shared
|
|
@@ -8,7 +8,13 @@ import type { RunnerKind } from './runner-sizing.js';
|
|
|
8
8
|
export interface MbxPool {
|
|
9
9
|
readonly priority: 'normal' | 'low';
|
|
10
10
|
}
|
|
11
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged, or
|
|
13
|
+
* where this run is already inside an mbx reservation: `mbx reserve` hands its command `MBX_SCHED_DISABLE=1`, and nested
|
|
14
|
+
* non-Cargo work shares the enclosing grant instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism
|
|
15
|
+
* inside a job"). A second reservation would wait in the pool while its parent holds a grant, so under a full pool the
|
|
16
|
+
* nested leaf queued behind the very work it belongs to.
|
|
17
|
+
*/
|
|
12
18
|
export declare function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | null;
|
|
13
19
|
/**
|
|
14
20
|
* How a leaf enters the pool once: a Cargo leaf runs through mbx, which admits its compiler and supported test work
|
|
@@ -23,9 +29,20 @@ export type MbxLeaf = {
|
|
|
23
29
|
readonly priority: 'normal' | 'low';
|
|
24
30
|
};
|
|
25
31
|
export declare function mbxLeaf(pool: MbxPool, kind: RunnerKind, want: number): MbxLeaf;
|
|
32
|
+
/**
|
|
33
|
+
* Only a known scheduled execution may stop its idle clock at zero holds. Plain Cargo tests can run doctests;
|
|
34
|
+
* archive-file targets, direct binaries and other runners are not proven scheduled at spawn (11_shell.md,
|
|
35
|
+
* "Cargo test-runner boundaries"). The inferred extracted native nextest command is a supported Cargo invocation.
|
|
36
|
+
*/
|
|
37
|
+
export declare function mbxIdleGated(leaf: MbxLeaf | null, command: string, environment: Readonly<NodeJS.ProcessEnv>): boolean;
|
|
38
|
+
/**
|
|
39
|
+
* mbx writes one byte per admission (g) and permit drop (r). Cargo descendants inherit this descriptor through
|
|
40
|
+
* MBX_GRANT_FD; reserve names it with --grant-fd and closes it on exec of the external command.
|
|
41
|
+
*/
|
|
42
|
+
export declare const MBX_GRANT_FD = 5;
|
|
26
43
|
/**
|
|
27
44
|
* `command` as the shell runs it under its leaf's admission. `mbx` is the canonical environment's, whose realpath
|
|
28
|
-
* cowshed's admission proved to be the host's pinned mbx.
|
|
45
|
+
* cowshed's admission proved to be the host's pinned mbx; it writes the grant on this descriptor itself.
|
|
29
46
|
*/
|
|
30
47
|
export declare function withMbxLeaf(command: string, leaf: MbxLeaf): string;
|
|
31
48
|
//# sourceMappingURL=mbx-pool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mbx-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/mbx-pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"mbx-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/mbx-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,KAAK,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAGrE;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,CAAC;CACrC;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,GAAG,IAAI,CAKhF;AAED;;;;GAIG;AACH,MAAM,MAAM,OAAO,GACf;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAC1B;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,CAAA;CAAE,CAAC;AAI7F,wBAAgB,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAI9E;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,CAyDrH;AAED;;;GAGG;AACH,eAAO,MAAM,YAAY,IAAI,CAAC;AAE9B;;;GAGG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,MAAM,CAKlE"}
|