@smoothbricks/nx-plugin 0.7.17 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +315 -114
- package/dist/.build-inputs.json +47 -38
- package/dist/built-output-packages.js +1 -1
- package/dist/bun-lock-scope.d.ts +128 -20
- package/dist/bun-lock-scope.d.ts.map +1 -1
- package/dist/bun-lock-scope.js +267 -44
- package/dist/cargo-closure-input.d.ts +8 -6
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +15 -12
- package/dist/cargo-source-hash.d.ts +26 -25
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +84 -105
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts +20 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +43 -22
- package/dist/ensure-built-receipt.d.ts +3 -2
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +3 -2
- package/dist/eslint-config-closure.d.ts +8 -3
- package/dist/eslint-config-closure.d.ts.map +1 -1
- package/dist/eslint-config-closure.js +24 -5
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +19 -6
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +22 -15
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/cpu-rise.d.ts +73 -0
- package/dist/executors/bounded-exec/cpu-rise.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-rise.js +111 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts +1 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +256 -86
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +11 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +125 -47
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +42 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +45 -8
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +12 -6
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +190 -284
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress-channel.d.ts +38 -0
- package/dist/executors/bounded-exec/progress-channel.d.ts.map +1 -0
- package/dist/executors/bounded-exec/progress-channel.js +113 -0
- package/dist/executors/bounded-exec/progress.d.ts +5 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts +24 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +123 -16
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +7 -2
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/schema.js +17 -1
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +39 -10
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +60 -14
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +294 -152
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/managed-files/generator.d.ts +5 -1
- package/dist/managed-files/generator.d.ts.map +1 -1
- package/dist/managed-files/generator.js +22 -7
- package/dist/managed-files/render-memo.d.ts +13 -0
- package/dist/managed-files/render-memo.d.ts.map +1 -0
- package/dist/managed-files/render-memo.js +94 -0
- package/dist/task-results.js +2 -1
- package/dist/test-program.d.ts +11 -6
- package/dist/test-program.d.ts.map +1 -1
- package/dist/test-program.js +39 -35
- package/dist/tool-bins.d.ts +12 -0
- package/dist/tool-bins.d.ts.map +1 -1
- package/dist/tool-bins.js +31 -6
- package/dist/typecheck-test-policy.d.ts +10 -1
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +40 -24
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/dist/workspace-index.d.ts +14 -1
- package/dist/workspace-index.d.ts.map +1 -1
- package/dist/workspace-index.js +36 -4
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1851 -221
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +122 -14
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/managed/templates/github/actions/save-nix-devenv/action.yml +19 -1
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +68 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/bun-lock-scope.ts +293 -40
- package/src/cargo-closure-input.ts +20 -12
- package/src/cargo-source-hash.ts +119 -145
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +59 -21
- package/src/ensure-built-receipt.ts +3 -2
- package/src/eslint-config-closure.ts +27 -6
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +41 -19
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/cpu-rise.ts +156 -0
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +341 -123
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +150 -50
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +57 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +216 -183
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress-channel.ts +129 -0
- package/src/executors/bounded-exec/progress.ts +39 -3
- package/src/executors/bounded-exec/ram-temp.ts +137 -17
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +21 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +83 -22
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +357 -166
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/managed-files/generator.ts +21 -6
- package/src/managed-files/render-memo.ts +135 -0
- package/src/test-program.ts +36 -41
- package/src/tool-bins.ts +35 -6
- package/src/typecheck-test-policy.ts +49 -24
- package/src/workspace-config-policy.ts +161 -43
- package/src/workspace-index.ts +40 -5
- package/test-timings.json +157 -141
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -14,10 +14,22 @@ import {
|
|
|
14
14
|
signalProcessGroup,
|
|
15
15
|
} from '../../process-group.js';
|
|
16
16
|
import { AdmissionObservation, type QueueHost } from './admission-queue.js';
|
|
17
|
+
import { atClockDeadline, openAwakeClock } from './awake-clock.js';
|
|
17
18
|
import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
19
|
+
import {
|
|
20
|
+
type CaseGateServer,
|
|
21
|
+
type GatedCases,
|
|
22
|
+
gateCases,
|
|
23
|
+
runWallMs,
|
|
24
|
+
serveCaseGate,
|
|
25
|
+
TEST_GATE_ENV,
|
|
26
|
+
TEST_GATE_SERVED_ENV,
|
|
27
|
+
} from './case-gate.js';
|
|
18
28
|
import { DIAGNOSTIC_CONTEXT_ENV, DIAGNOSTIC_MODULE_ENV, grantMemoryEnvironment } from './child-exit-diagnostics.js';
|
|
19
|
-
import { type CpuPool, createCpuPool } from './cpu-pool.js';
|
|
29
|
+
import { type CpuPool, createCpuPool, lendParentToken } from './cpu-pool.js';
|
|
30
|
+
import type { CpuRise } from './cpu-rise.js';
|
|
20
31
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
32
|
+
import { declaredGoTests, goRunSummaryText } from './go-run-summary.js';
|
|
21
33
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
22
34
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
23
35
|
import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
@@ -30,6 +42,7 @@ import {
|
|
|
30
42
|
mbxPool,
|
|
31
43
|
withCountedReservation,
|
|
32
44
|
withMbxLeaf,
|
|
45
|
+
withReservationAfterFixture,
|
|
33
46
|
} from './mbx-pool.js';
|
|
34
47
|
import { describeUnserved, RESERVATIONS_ENV, type Reservation, reserveCores, runEndpoint } from './mbx-reservations.js';
|
|
35
48
|
import { killedFromOutside, type RunOutcome, runUntilNotKilledForMemory } from './memory-retry.js';
|
|
@@ -47,6 +60,7 @@ import { askParentPool, describeGrant } from './parent-grant.js';
|
|
|
47
60
|
import { type LastPeak, readLastPeak } from './peak-charge.js';
|
|
48
61
|
import { CPU_POOL_ENV, type InheritedCpuPool, readInheritedCpuPool } from './pool-environment.js';
|
|
49
62
|
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
63
|
+
import { PROGRESS_SOCKET_ENV, type ProgressChannel, serveProgressChannel } from './progress-channel.js';
|
|
50
64
|
import {
|
|
51
65
|
createHostCommands,
|
|
52
66
|
describeCoverageGap,
|
|
@@ -62,7 +76,8 @@ import {
|
|
|
62
76
|
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
63
77
|
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
|
|
64
78
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
65
|
-
import type
|
|
79
|
+
import { type BoundedExecOptions, boundedCommand } from './schema.js';
|
|
80
|
+
import { readLastWalls, suiteMinimum } from './suite-minimum.js';
|
|
66
81
|
import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
|
|
67
82
|
import {
|
|
68
83
|
type BoundedExecRecord,
|
|
@@ -76,6 +91,7 @@ import {
|
|
|
76
91
|
judgeOperationalFailure,
|
|
77
92
|
judgeRun,
|
|
78
93
|
type NoSpaceSighting,
|
|
94
|
+
type OperationalCause,
|
|
79
95
|
readJunitReport,
|
|
80
96
|
taskDirectory,
|
|
81
97
|
withJunitReport,
|
|
@@ -137,6 +153,8 @@ interface RunState {
|
|
|
137
153
|
noSpace: Promise<NoSpaceSighting> | null;
|
|
138
154
|
/** The lock waits the expiry's snapshot named: empty unless that capture completed. Judged with the expiry. */
|
|
139
155
|
lockWaits: readonly LockWait[];
|
|
156
|
+
/** The processes the expiry's snapshot saw gain CPU time: empty unless that capture completed. Judged with the expiry. */
|
|
157
|
+
cpuRising: readonly CpuRise[];
|
|
140
158
|
}
|
|
141
159
|
|
|
142
160
|
export interface ProcessTreeKiller {
|
|
@@ -305,10 +323,14 @@ export async function runBoundedExec(
|
|
|
305
323
|
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
306
324
|
// killed run replaced, not by the peak it reached before it was killed.
|
|
307
325
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
326
|
+
// The walls its last run recorded size the minimum its reservation names (suite-minimum.ts), read before the same
|
|
327
|
+
// clear removes them.
|
|
328
|
+
const lastWalls = directory !== null ? await readLastWalls(directory) : new Map<string, number>();
|
|
308
329
|
return runUntilNotKilledForMemory(
|
|
309
330
|
(alone) =>
|
|
310
331
|
runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
311
332
|
lastPeak,
|
|
333
|
+
lastWalls,
|
|
312
334
|
alone,
|
|
313
335
|
}),
|
|
314
336
|
inherited !== null,
|
|
@@ -316,9 +338,13 @@ export async function runBoundedExec(
|
|
|
316
338
|
);
|
|
317
339
|
}
|
|
318
340
|
|
|
319
|
-
/**
|
|
341
|
+
/**
|
|
342
|
+
* What a run of the command is told that the run before it was not: the task's charge, the walls its tests last took,
|
|
343
|
+
* and whether it runs alone.
|
|
344
|
+
*/
|
|
320
345
|
interface Attempt {
|
|
321
346
|
readonly lastPeak: LastPeak | null;
|
|
347
|
+
readonly lastWalls: ReadonlyMap<string, number>;
|
|
322
348
|
readonly alone: boolean;
|
|
323
349
|
}
|
|
324
350
|
|
|
@@ -336,12 +362,12 @@ async function runCommand(
|
|
|
336
362
|
): Promise<BoundedExecResult> {
|
|
337
363
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
338
364
|
const directory = context.record?.directory ?? null;
|
|
339
|
-
const { lastPeak } = attempt;
|
|
365
|
+
const { lastPeak, lastWalls } = attempt;
|
|
340
366
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
341
367
|
if (directory !== null) {
|
|
342
368
|
await clearTaskDirectory(directory);
|
|
343
369
|
}
|
|
344
|
-
const requested =
|
|
370
|
+
const requested = boundedCommand(options);
|
|
345
371
|
const timeoutMs = options.timeoutMs;
|
|
346
372
|
const idleTimeoutMs = options.idleTimeoutMs;
|
|
347
373
|
const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
|
|
@@ -361,6 +387,7 @@ async function runCommand(
|
|
|
361
387
|
retireError: null,
|
|
362
388
|
noSpace: null,
|
|
363
389
|
lockWaits: [],
|
|
390
|
+
cpuRising: [],
|
|
364
391
|
};
|
|
365
392
|
const appendStdout = (chunk: Buffer | string): void => {
|
|
366
393
|
const text = chunk.toString();
|
|
@@ -393,13 +420,21 @@ async function runCommand(
|
|
|
393
420
|
await writeRecord(recordDirectory, {
|
|
394
421
|
task: task.id,
|
|
395
422
|
hash: task.hash,
|
|
396
|
-
verdict: judgeOperationalFailure(1, null),
|
|
423
|
+
verdict: judgeOperationalFailure(1, null, { kind: 'executor', message }),
|
|
397
424
|
peakBytes: null,
|
|
398
425
|
});
|
|
399
426
|
}
|
|
400
427
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
401
428
|
};
|
|
402
429
|
|
|
430
|
+
let awakeNow = (): number => performance.now();
|
|
431
|
+
if (idleTimeoutMs !== undefined) {
|
|
432
|
+
try {
|
|
433
|
+
awakeNow = await openAwakeClock();
|
|
434
|
+
} catch (error) {
|
|
435
|
+
return refuseUnstarted(`bounded-exec: cannot read its awake idle clock: ${String(error)}\n`);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
403
438
|
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
404
439
|
// target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
|
|
405
440
|
if (
|
|
@@ -582,6 +617,7 @@ async function runCommand(
|
|
|
582
617
|
parallelism: chargeParallelism,
|
|
583
618
|
alone: attempt.alone,
|
|
584
619
|
task: context.record?.task.id,
|
|
620
|
+
gated: demand.kind === 'nextest' && process.env[TEST_GATE_SERVED_ENV] === '1',
|
|
585
621
|
});
|
|
586
622
|
for (const said of parent.said) appendStderr(said);
|
|
587
623
|
const { grant, refusal: parentRefusal, wait: parentWait } = parent;
|
|
@@ -591,15 +627,26 @@ async function runCommand(
|
|
|
591
627
|
// With mbx the leaf's admission is mbx's one pool, not a parent's tokens. A leaf mbx does not admit itself asks for
|
|
592
628
|
// its runner's count, a maximum, and its runner runs on the cores it is granted, which only the command learns
|
|
593
629
|
// (specs/cowshed/05_gateway.md, "Parallelism inside a job"). No grant and no sub-pool beside it.
|
|
594
|
-
const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested);
|
|
630
|
+
const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested, process.env[TEST_GATE_SERVED_ENV] === '1');
|
|
631
|
+
// A nextest leaf granted fewer of its parent pool's tokens than it asked takes the case gate where the run's mbx
|
|
632
|
+
// gates cases: its runner starts every thread of its ask, and a case that finds no token of the grant free asks the
|
|
633
|
+
// pool for one of its own (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold).
|
|
634
|
+
const lender =
|
|
635
|
+
grant !== null &&
|
|
636
|
+
inherited !== null &&
|
|
637
|
+
demand.kind === 'nextest' &&
|
|
638
|
+
grant.tokens < demand.want &&
|
|
639
|
+
process.env[TEST_GATE_SERVED_ENV] === '1'
|
|
640
|
+
? { own: grant.tokens, pool: inherited }
|
|
641
|
+
: null;
|
|
595
642
|
if (grant !== null) {
|
|
596
|
-
sized = sizeRunner(demand, requested, grant.tokens);
|
|
643
|
+
sized = sizeRunner(demand, requested, grant.tokens, lender !== null);
|
|
597
644
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
598
645
|
for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
|
|
599
646
|
appendStderr(said);
|
|
600
647
|
}
|
|
601
648
|
} else if (leaf?.kind === 'reserve') {
|
|
602
|
-
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
649
|
+
sized = { command: sizeRunnerToGrant(leaf.demand, requested, leaf.gated), env: {} };
|
|
603
650
|
} else if (leaf !== null) {
|
|
604
651
|
sized = sizeRunner(demand, requested, demand.want);
|
|
605
652
|
}
|
|
@@ -609,6 +656,37 @@ async function runCommand(
|
|
|
609
656
|
let pool: CpuPool | null = null;
|
|
610
657
|
let reservation: Reservation | null = null;
|
|
611
658
|
const releaseReservation = (): void => reservation?.release();
|
|
659
|
+
// A gated nextest leaf's case gate (case-gate.ts; specs/cowshed/05_gateway.md, "A case starts on a core"): served
|
|
660
|
+
// before the command starts, decided from the leaf's grant on, and closed once the command is gone, when every core a
|
|
661
|
+
// case still holds goes back to the pool and the leaf says what its cases ran on.
|
|
662
|
+
let caseGate: CaseGateServer | null = null;
|
|
663
|
+
let gated: GatedCases | null = null;
|
|
664
|
+
let caseGateClosed: Promise<void> | null = null;
|
|
665
|
+
const closeCaseGate = (): Promise<void> => {
|
|
666
|
+
caseGateClosed ??= (async () => {
|
|
667
|
+
const left = (await caseGate?.close()) ?? null;
|
|
668
|
+
if (left !== null) appendStderr(`bounded-exec: ${left}\n`);
|
|
669
|
+
const decided = gated;
|
|
670
|
+
if (decided === null) return;
|
|
671
|
+
decided.releaseAll();
|
|
672
|
+
const { own, lent, lentCoreMs, waitedCaseMs } = decided.cases.summary();
|
|
673
|
+
tell(
|
|
674
|
+
`cowshed: ${grant !== null ? 'cpu-pool' : 'mbx reserve'} cases own=${own} lent=${lent} lent_core_s=${Math.round(lentCoreMs / 1000)} waited_case_s=${Math.round(waitedCaseMs / 1000)}\n`,
|
|
675
|
+
);
|
|
676
|
+
})();
|
|
677
|
+
return caseGateClosed;
|
|
678
|
+
};
|
|
679
|
+
// The run's progress channel (progress-channel.ts), served when the run has an idle bound for its records to restart,
|
|
680
|
+
// and closed once its command is gone. A channel that leaves its socket behind says so; the run's verdict stands.
|
|
681
|
+
let progressChannel: ProgressChannel | null = null;
|
|
682
|
+
let progressClosed: Promise<void> | null = null;
|
|
683
|
+
const closeProgress = (): Promise<void> => {
|
|
684
|
+
progressClosed ??= (async () => {
|
|
685
|
+
const left = (await progressChannel?.close()) ?? null;
|
|
686
|
+
if (left !== null) appendStderr(`bounded-exec: ${left}\n`);
|
|
687
|
+
})();
|
|
688
|
+
return progressClosed;
|
|
689
|
+
};
|
|
612
690
|
let cpuReleased: Promise<void> | null = null;
|
|
613
691
|
const releaseCpu = (): Promise<void> => {
|
|
614
692
|
cpuReleased ??= (async () => {
|
|
@@ -631,6 +709,7 @@ async function runCommand(
|
|
|
631
709
|
tell,
|
|
632
710
|
step: (event) => stepTo(event),
|
|
633
711
|
testing: () => isTesting(),
|
|
712
|
+
awakeNow,
|
|
634
713
|
queueHost,
|
|
635
714
|
});
|
|
636
715
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
@@ -639,9 +718,10 @@ async function runCommand(
|
|
|
639
718
|
exitCode: number,
|
|
640
719
|
expiry: ExpiredBound | null,
|
|
641
720
|
elapsedMs: number,
|
|
642
|
-
operational:
|
|
721
|
+
operational: OperationalCause | null,
|
|
643
722
|
noSpace: NoSpaceSighting | null,
|
|
644
723
|
lockWaits: readonly LockWait[],
|
|
724
|
+
cpuRising: readonly CpuRise[],
|
|
645
725
|
peakBytes: number | null,
|
|
646
726
|
): Promise<void> => {
|
|
647
727
|
if (context.record === undefined) {
|
|
@@ -667,9 +747,11 @@ async function runCommand(
|
|
|
667
747
|
}
|
|
668
748
|
});
|
|
669
749
|
const admissionQueue = queue.queueWait(reservation !== null);
|
|
750
|
+
const cause: OperationalCause | null =
|
|
751
|
+
operational ?? (resultFailure === null ? null : { kind: 'executor', message: resultFailure });
|
|
670
752
|
const verdict =
|
|
671
|
-
|
|
672
|
-
? judgeOperationalFailure(exitCode, report)
|
|
753
|
+
cause !== null
|
|
754
|
+
? judgeOperationalFailure(exitCode, report, cause)
|
|
673
755
|
: judgeRun({
|
|
674
756
|
exitCode,
|
|
675
757
|
elapsedMs,
|
|
@@ -677,6 +759,7 @@ async function runCommand(
|
|
|
677
759
|
report,
|
|
678
760
|
noSpace,
|
|
679
761
|
lockWaits,
|
|
762
|
+
cpuRising,
|
|
680
763
|
admissionQueue,
|
|
681
764
|
});
|
|
682
765
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
@@ -695,9 +778,11 @@ async function runCommand(
|
|
|
695
778
|
const refuse = async (message: string): Promise<BoundedExecResult> => {
|
|
696
779
|
appendStderr(message);
|
|
697
780
|
await releaseCpu();
|
|
781
|
+
await closeProgress();
|
|
782
|
+
await closeCaseGate();
|
|
698
783
|
// The command never started, so its lease holds nothing of its own to keep.
|
|
699
784
|
await releaseLease(false);
|
|
700
|
-
await recordRun(1, null, Date.now() - asked,
|
|
785
|
+
await recordRun(1, null, Date.now() - asked, { kind: 'executor', message: message.trim() }, null, [], [], null);
|
|
701
786
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
702
787
|
};
|
|
703
788
|
if (parentRefusal !== null) {
|
|
@@ -723,6 +808,58 @@ async function runCommand(
|
|
|
723
808
|
}
|
|
724
809
|
pool = created.pool;
|
|
725
810
|
}
|
|
811
|
+
// A running test reports its own progress on the run's progress channel, and each record restarts the idle bound as a
|
|
812
|
+
// finished test does; a record that is not one fails the run as a broken reporter. A run with no idle bound has
|
|
813
|
+
// nothing for a record to restart and is served none. Only the command learns the socket, from its environment, so
|
|
814
|
+
// these handlers run after it is spawned and its bounds are wired below.
|
|
815
|
+
if (idleTimeoutMs !== undefined) {
|
|
816
|
+
const served = await serveProgressChannel({
|
|
817
|
+
moved: () => {
|
|
818
|
+
if (fixtureDone) stepTo({ kind: 'reported', at: awakeNow(), testing: isTesting() });
|
|
819
|
+
},
|
|
820
|
+
invalid: (detail) => {
|
|
821
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
|
|
822
|
+
lifecycleFailure = { kind: 'executor', message: `progress channel: ${detail}` };
|
|
823
|
+
appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
|
|
824
|
+
termination = terminate();
|
|
825
|
+
},
|
|
826
|
+
});
|
|
827
|
+
if (!served.ok) {
|
|
828
|
+
return refuse(
|
|
829
|
+
`bounded-exec: the run's progress channel could not be served (${served.error}); refusing to run the command\n`,
|
|
830
|
+
);
|
|
831
|
+
}
|
|
832
|
+
progressChannel = served.channel;
|
|
833
|
+
}
|
|
834
|
+
// A case that arrives before the leaf's grant has no core to take and no leaf to wait for: its connection is refused,
|
|
835
|
+
// and its shim fails it by name.
|
|
836
|
+
if (reserved?.leaf.gated === true || lender !== null) {
|
|
837
|
+
const served = await serveCaseGate({
|
|
838
|
+
arrived: (id) => gated?.cases.arrived(id) ?? false,
|
|
839
|
+
ended: (id) => gated?.cases.ended(id),
|
|
840
|
+
invalid: (detail) => {
|
|
841
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
|
|
842
|
+
lifecycleFailure = { kind: 'executor', message: `case gate: ${detail}` };
|
|
843
|
+
appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
|
|
844
|
+
termination = terminate();
|
|
845
|
+
},
|
|
846
|
+
});
|
|
847
|
+
if (!served.ok) {
|
|
848
|
+
return refuse(
|
|
849
|
+
`bounded-exec: the leaf's case gate could not be served (${served.error}); refusing to run the command\n`,
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
caseGate = served.gate;
|
|
853
|
+
if (lender !== null) {
|
|
854
|
+
const gate = served.gate;
|
|
855
|
+
gated = gateCases(
|
|
856
|
+
lender.own,
|
|
857
|
+
lastWalls,
|
|
858
|
+
(id) => gate.start(id),
|
|
859
|
+
(id, left) => lendParentToken(lender.pool, `nextest:${id}`, left),
|
|
860
|
+
);
|
|
861
|
+
}
|
|
862
|
+
}
|
|
726
863
|
const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
|
|
727
864
|
const command =
|
|
728
865
|
directory === null
|
|
@@ -740,11 +877,13 @@ async function runCommand(
|
|
|
740
877
|
...(pool?.environment ?? {}),
|
|
741
878
|
...(await grantMemoryEnvironment(grant?.charged ?? null, process.env)),
|
|
742
879
|
...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
|
|
880
|
+
...(progressChannel === null ? {} : { [PROGRESS_SOCKET_ENV]: progressChannel.socket }),
|
|
881
|
+
...(caseGate === null ? {} : { [TEST_GATE_ENV]: caseGate.socket }),
|
|
743
882
|
};
|
|
744
883
|
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
745
|
-
// of the command: the progress bound runs during it and its own ceiling does not. Under mbx a leaf
|
|
746
|
-
//
|
|
747
|
-
//
|
|
884
|
+
// of the command: the progress bound runs during it and its own ceiling does not. Under mbx a reserved leaf's
|
|
885
|
+
// fixture runs before its reservation, its ceiling from spawn (`withReservationAfterFixture`); a leaf that names its
|
|
886
|
+
// units counts them at the end of that segment, its fixture's own output on stderr (`withCountedReservation`).
|
|
748
887
|
const counted =
|
|
749
888
|
process.platform !== 'win32' && leaf?.kind === 'reserve' && fixture !== null && options.unitsCommand !== undefined;
|
|
750
889
|
const segment =
|
|
@@ -753,15 +892,29 @@ async function runCommand(
|
|
|
753
892
|
process.platform === 'win32' || segment === null ? null : withFixtureLifecycle(segment, command);
|
|
754
893
|
const hooked = process.platform === 'win32' ? null : (fixtureHooked ?? withRunnerLifecycle(command));
|
|
755
894
|
const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
|
|
756
|
-
let lifecycleFailure:
|
|
895
|
+
let lifecycleFailure: OperationalCause | null = null;
|
|
896
|
+
/** Why the executor ended or failed the run whatever its command did, if it did: the verdict's operational cause. */
|
|
897
|
+
/** The cause the executor ended the command for, if it ended it (closures set `lifecycleFailure`). */
|
|
898
|
+
const endedFor = (): OperationalCause | null => lifecycleFailure;
|
|
899
|
+
const operationalCause = (): OperationalCause | null =>
|
|
900
|
+
endedFor() ??
|
|
901
|
+
(state.poolCloseError !== null
|
|
902
|
+
? { kind: 'executor', message: `cpu pool did not close cleanly: ${state.poolCloseError}` }
|
|
903
|
+
: state.retireError !== null
|
|
904
|
+
? { kind: 'executor', message: `could not retire the command's processes: ${state.retireError}` }
|
|
905
|
+
: null);
|
|
757
906
|
const isTesting = (): boolean => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
|
|
758
907
|
const startedAt = Date.now();
|
|
759
908
|
// With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
|
|
760
909
|
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
761
910
|
const unadmitted = hooked?.command ?? (segment === null ? command : `${segment} && ${command}`);
|
|
762
911
|
const shellCommand =
|
|
763
|
-
leaf?.kind === 'reserve' &&
|
|
764
|
-
?
|
|
912
|
+
leaf?.kind === 'reserve' && fixtureHooked !== null
|
|
913
|
+
? (counted ? withCountedReservation : withReservationAfterFixture)(
|
|
914
|
+
fixtureHooked.fixture,
|
|
915
|
+
fixtureHooked.main,
|
|
916
|
+
leaf,
|
|
917
|
+
)
|
|
765
918
|
: withMbxLeaf(unadmitted, leaf);
|
|
766
919
|
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
767
920
|
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
@@ -845,16 +998,23 @@ async function runCommand(
|
|
|
845
998
|
});
|
|
846
999
|
}
|
|
847
1000
|
|
|
848
|
-
// Progress is a finished test,
|
|
1001
|
+
// Progress in the output is a finished test, never output itself: a runner's SLOW warnings and a hung test's heartbeat
|
|
1002
|
+
// keep printing. A running test reports its own progress on the progress channel instead.
|
|
849
1003
|
const progress = createRunProgress(
|
|
850
1004
|
command,
|
|
851
1005
|
resultWriter === null
|
|
852
1006
|
? undefined
|
|
853
|
-
: (
|
|
1007
|
+
: (closed) => {
|
|
1008
|
+
// nextest times a case from its shim's start: the wait at the gate is taken off its wall and named beside it.
|
|
1009
|
+
const waitMs = gated?.cases.waitedMs(closed.id) ?? 0;
|
|
1010
|
+
const result =
|
|
1011
|
+
closed.durationMs === null || waitMs === 0
|
|
1012
|
+
? closed
|
|
1013
|
+
: { ...closed, durationMs: runWallMs(closed.durationMs, waitMs), waitMs };
|
|
854
1014
|
const error = resultWriter?.append(result) ?? null;
|
|
855
1015
|
if (error !== null && resultFailure === null) {
|
|
856
1016
|
resultFailure = error;
|
|
857
|
-
lifecycleFailure ??= error;
|
|
1017
|
+
lifecycleFailure ??= { kind: 'executor', message: error };
|
|
858
1018
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
859
1019
|
termination = terminate();
|
|
860
1020
|
}
|
|
@@ -874,56 +1034,57 @@ async function runCommand(
|
|
|
874
1034
|
let fixtureDone = fixture === null || lifecycle === null;
|
|
875
1035
|
// Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
|
|
876
1036
|
let mainStarted = fixture === null || lifecycle === null;
|
|
877
|
-
let
|
|
878
|
-
let idleCapture:
|
|
1037
|
+
let cancelCaptureTimer: (() => void) | undefined;
|
|
1038
|
+
let idleCapture:
|
|
1039
|
+
| { scheduledExpiryAtMs: number; controller: AbortController; result: IdleDiagnosticResult }
|
|
1040
|
+
| undefined;
|
|
879
1041
|
const cancelCapture = (): void => {
|
|
880
|
-
|
|
1042
|
+
cancelCaptureTimer?.();
|
|
881
1043
|
idleCapture?.controller.abort();
|
|
882
1044
|
};
|
|
883
|
-
const scheduleCapture = (
|
|
1045
|
+
const scheduleCapture = (awakeExpiresAtMs: number): void => {
|
|
884
1046
|
const capture: NonNullable<typeof idleCapture> = {
|
|
885
|
-
|
|
1047
|
+
scheduledExpiryAtMs: Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow())),
|
|
886
1048
|
controller: new AbortController(),
|
|
887
1049
|
result: { kind: 'pending', startedAtMs: null },
|
|
888
1050
|
};
|
|
889
1051
|
idleCapture = capture;
|
|
890
|
-
|
|
891
|
-
()
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
})
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
}
|
|
922
|
-
}
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
);
|
|
1052
|
+
cancelCaptureTimer = atClockDeadline(awakeExpiresAtMs - HANG_CAPTURE_MS, awakeNow, () => {
|
|
1053
|
+
if (state.settled || state.expiry !== null || idleCapture !== capture) return;
|
|
1054
|
+
const group = child.pid;
|
|
1055
|
+
if (group === undefined) {
|
|
1056
|
+
idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
|
|
1057
|
+
return;
|
|
1058
|
+
}
|
|
1059
|
+
idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
|
|
1060
|
+
capture.scheduledExpiryAtMs = Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow()));
|
|
1061
|
+
void runnerPid
|
|
1062
|
+
.then((pid) => {
|
|
1063
|
+
if (capture.controller.signal.aborted || awakeNow() >= awakeExpiresAtMs) return;
|
|
1064
|
+
if (pid === null)
|
|
1065
|
+
throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
|
|
1066
|
+
return captureDiagnostics({
|
|
1067
|
+
pid,
|
|
1068
|
+
pgid: process.platform === 'win32' ? null : group,
|
|
1069
|
+
awakeExpiresAtMs,
|
|
1070
|
+
awakeNow,
|
|
1071
|
+
signal: capture.controller.signal,
|
|
1072
|
+
});
|
|
1073
|
+
})
|
|
1074
|
+
.then(
|
|
1075
|
+
(snapshot) => {
|
|
1076
|
+
if (snapshot !== undefined && idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
|
|
1077
|
+
},
|
|
1078
|
+
(error: unknown) => {
|
|
1079
|
+
if (idleCapture === capture) {
|
|
1080
|
+
idleCapture.result = {
|
|
1081
|
+
kind: 'failed',
|
|
1082
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
1083
|
+
};
|
|
1084
|
+
}
|
|
1085
|
+
},
|
|
1086
|
+
);
|
|
1087
|
+
});
|
|
927
1088
|
};
|
|
928
1089
|
const expiredAs = (kind: BoundKind): ExpiredBound =>
|
|
929
1090
|
kind === 'idle'
|
|
@@ -932,25 +1093,27 @@ async function runCommand(
|
|
|
932
1093
|
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
933
1094
|
: { kind, limitMs: timeoutMs ?? 0 };
|
|
934
1095
|
let checkingIdle = false;
|
|
1096
|
+
const nextBound = () => earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds, awakeNow(), Date.now());
|
|
935
1097
|
const armBoundTimer = (): void => {
|
|
936
1098
|
clearTimeout(boundTimer);
|
|
937
1099
|
if (state.settled || state.expiry !== null) return;
|
|
938
|
-
const next =
|
|
1100
|
+
const next = nextBound();
|
|
939
1101
|
if (next !== null) {
|
|
940
|
-
|
|
941
|
-
()
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
Math.max(0, next.at - Date.now()),
|
|
945
|
-
);
|
|
1102
|
+
if (next.kind === 'idle' && idleCapture !== undefined && next.remainingMs > 0) {
|
|
1103
|
+
idleCapture.scheduledExpiryAtMs = Date.now() + Math.ceil(next.remainingMs);
|
|
1104
|
+
}
|
|
1105
|
+
boundTimer = setTimeout(() => void confirmExpiry(), Math.ceil(Math.max(0, next.remainingMs)));
|
|
946
1106
|
}
|
|
947
1107
|
};
|
|
948
|
-
const confirmExpiry = async (
|
|
1108
|
+
const confirmExpiry = async (): Promise<void> => {
|
|
949
1109
|
if (state.settled || state.expiry !== null) return;
|
|
950
|
-
const scheduled =
|
|
951
|
-
if (scheduled
|
|
952
|
-
if (
|
|
953
|
-
|
|
1110
|
+
const scheduled = nextBound();
|
|
1111
|
+
if (scheduled === null) return;
|
|
1112
|
+
if (scheduled.remainingMs > 0) {
|
|
1113
|
+
armBoundTimer();
|
|
1114
|
+
return;
|
|
1115
|
+
}
|
|
1116
|
+
if (scheduled.kind === 'idle' && leaf?.kind === 'cargo') {
|
|
954
1117
|
checkingIdle = true;
|
|
955
1118
|
armBoundTimer();
|
|
956
1119
|
try {
|
|
@@ -962,14 +1125,14 @@ async function runCommand(
|
|
|
962
1125
|
}
|
|
963
1126
|
}
|
|
964
1127
|
if (state.settled || state.expiry !== null) return;
|
|
965
|
-
const next =
|
|
966
|
-
if (next !== null && next.
|
|
1128
|
+
const next = nextBound();
|
|
1129
|
+
if (next !== null && next.remainingMs <= 0) expire(expiredAs(next.kind));
|
|
967
1130
|
else armBoundTimer();
|
|
968
1131
|
};
|
|
969
1132
|
const stepTo = (event: BoundEvent): void => {
|
|
970
1133
|
if (state.settled || state.expiry !== null) return;
|
|
971
1134
|
const idleBefore = bounds.idle;
|
|
972
|
-
|
|
1135
|
+
const next = stepBounds(
|
|
973
1136
|
bounds,
|
|
974
1137
|
{
|
|
975
1138
|
...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
|
|
@@ -978,6 +1141,8 @@ async function runCommand(
|
|
|
978
1141
|
},
|
|
979
1142
|
event,
|
|
980
1143
|
);
|
|
1144
|
+
if (next === bounds) return;
|
|
1145
|
+
bounds = next;
|
|
981
1146
|
if (bounds.idle !== idleBefore) {
|
|
982
1147
|
cancelCapture();
|
|
983
1148
|
idleCapture = undefined;
|
|
@@ -990,14 +1155,14 @@ async function runCommand(
|
|
|
990
1155
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
991
1156
|
// message extend the very bound that just fired.
|
|
992
1157
|
const updateIdle = (finished: boolean, wasTesting: boolean): void => {
|
|
993
|
-
stepTo({ kind: 'progress', at:
|
|
1158
|
+
stepTo({ kind: 'progress', at: awakeNow(), testing: isTesting(), wasTesting, finished });
|
|
994
1159
|
};
|
|
995
1160
|
const scanProgress = (stream: 'stdout' | 'stderr', text: Buffer | string): void => {
|
|
996
1161
|
if (!fixtureDone) return;
|
|
997
1162
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
998
1163
|
const error = progress.error();
|
|
999
1164
|
if (error !== null && lifecycleFailure === null) {
|
|
1000
|
-
lifecycleFailure = error;
|
|
1165
|
+
lifecycleFailure = { kind: 'executor', message: error };
|
|
1001
1166
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
1002
1167
|
termination = terminate();
|
|
1003
1168
|
}
|
|
@@ -1007,7 +1172,7 @@ async function runCommand(
|
|
|
1007
1172
|
if (!(channel instanceof Duplex)) throw new Error('runner lifecycle requires a duplex child descriptor');
|
|
1008
1173
|
const failLifecycle = (detail: string, wasTesting = isTesting()): void => {
|
|
1009
1174
|
if (lifecycleFailure !== null || state.expiry !== null) return;
|
|
1010
|
-
lifecycleFailure = detail;
|
|
1175
|
+
lifecycleFailure = { kind: 'executor', message: `runner lifecycle failed: ${detail}` };
|
|
1011
1176
|
appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
|
|
1012
1177
|
updateIdle(false, wasTesting);
|
|
1013
1178
|
};
|
|
@@ -1082,6 +1247,8 @@ async function runCommand(
|
|
|
1082
1247
|
queue.stop();
|
|
1083
1248
|
return killChildTree();
|
|
1084
1249
|
})
|
|
1250
|
+
.finally(closeProgress)
|
|
1251
|
+
.finally(closeCaseGate)
|
|
1085
1252
|
.finally(releaseCpu)
|
|
1086
1253
|
.finally(() => releaseLease(true))
|
|
1087
1254
|
.finally(releaseReservation)
|
|
@@ -1090,9 +1257,10 @@ async function runCommand(
|
|
|
1090
1257
|
signalToExitCode(signal),
|
|
1091
1258
|
state.expiry,
|
|
1092
1259
|
Date.now() - startedAt,
|
|
1093
|
-
|
|
1260
|
+
operationalCause(),
|
|
1094
1261
|
state.noSpace === null ? null : await state.noSpace,
|
|
1095
1262
|
state.lockWaits,
|
|
1263
|
+
state.cpuRising,
|
|
1096
1264
|
null,
|
|
1097
1265
|
),
|
|
1098
1266
|
)
|
|
@@ -1145,7 +1313,7 @@ async function runCommand(
|
|
|
1145
1313
|
const elapsedMs = Date.now() - startedAt;
|
|
1146
1314
|
appendStderr(
|
|
1147
1315
|
bound.kind === 'idle'
|
|
1148
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
|
|
1316
|
+
? `\nCommand made no progress: no test finished or reported progress for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
|
|
1149
1317
|
bound.tests.length > 0
|
|
1150
1318
|
? `still running: ${bound.tests.join(', ')}`
|
|
1151
1319
|
: 'the runner named no test still running'
|
|
@@ -1161,10 +1329,11 @@ async function runCommand(
|
|
|
1161
1329
|
if (bound.kind === 'idle') {
|
|
1162
1330
|
const observed = idleCapture?.result;
|
|
1163
1331
|
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
1332
|
+
state.cpuRising = observed?.kind === 'captured' ? observed.snapshot.cpuRising : [];
|
|
1164
1333
|
appendStderr(
|
|
1165
1334
|
`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${
|
|
1166
1335
|
process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
|
|
1167
|
-
} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.
|
|
1336
|
+
} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.scheduledExpiryAtMs ?? 'unavailable'}\n`,
|
|
1168
1337
|
);
|
|
1169
1338
|
if (observed?.kind === 'captured') {
|
|
1170
1339
|
appendStderr(
|
|
@@ -1252,22 +1421,27 @@ async function runCommand(
|
|
|
1252
1421
|
// fire on a command the fixture already handed over to.
|
|
1253
1422
|
stepTo({
|
|
1254
1423
|
kind: 'spawn',
|
|
1255
|
-
at:
|
|
1424
|
+
at: awakeNow(),
|
|
1425
|
+
wallAt: Date.now(),
|
|
1256
1426
|
testing: isTesting(),
|
|
1257
1427
|
fixture: fixture !== null && lifecycle !== null,
|
|
1258
|
-
fixtureFirst:
|
|
1428
|
+
fixtureFirst: leaf?.kind === 'reserve' && fixtureHooked !== null,
|
|
1259
1429
|
queued,
|
|
1260
1430
|
});
|
|
1261
1431
|
if (leaf !== null) {
|
|
1262
1432
|
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1263
1433
|
if (!(channel instanceof Duplex)) throw new Error('an mbx leaf requires its mbx descriptor');
|
|
1264
|
-
const
|
|
1434
|
+
const descriptorFailure = (detail: string): OperationalCause => ({
|
|
1435
|
+
kind: 'executor',
|
|
1436
|
+
message: `mbx grant descriptor: ${detail}`,
|
|
1437
|
+
});
|
|
1438
|
+
const failGrant = (cause: OperationalCause): void => {
|
|
1265
1439
|
if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
|
|
1266
|
-
lifecycleFailure
|
|
1267
|
-
appendStderr(`bounded-exec: ${
|
|
1440
|
+
lifecycleFailure = cause;
|
|
1441
|
+
appendStderr(`bounded-exec: ${cause.message}\n`);
|
|
1268
1442
|
termination = terminate();
|
|
1269
1443
|
};
|
|
1270
|
-
channel.on('error', (error) => failGrant(error.message));
|
|
1444
|
+
channel.on('error', (error) => failGrant(descriptorFailure(error.message)));
|
|
1271
1445
|
if (reserved !== null) {
|
|
1272
1446
|
// The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
|
|
1273
1447
|
// ends the descriptor unanswered: the shell's read fails, and nothing runs.
|
|
@@ -1276,7 +1450,7 @@ async function runCommand(
|
|
|
1276
1450
|
channel.setEncoding('utf8');
|
|
1277
1451
|
channel.on('data', (text: string) => {
|
|
1278
1452
|
if (reservation !== null) {
|
|
1279
|
-
failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
|
|
1453
|
+
failGrant(descriptorFailure(`the command asked for cores a second time: ${JSON.stringify(text)}`));
|
|
1280
1454
|
return;
|
|
1281
1455
|
}
|
|
1282
1456
|
asked += text;
|
|
@@ -1285,17 +1459,30 @@ async function runCommand(
|
|
|
1285
1459
|
const line = asked.slice(0, end);
|
|
1286
1460
|
const cpus = askedCores(line, reservedLeaf.demand.want);
|
|
1287
1461
|
if (cpus === null) {
|
|
1288
|
-
failGrant(
|
|
1462
|
+
failGrant(
|
|
1463
|
+
descriptorFailure(
|
|
1464
|
+
`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`,
|
|
1465
|
+
),
|
|
1466
|
+
);
|
|
1289
1467
|
channel.end();
|
|
1290
1468
|
return;
|
|
1291
1469
|
}
|
|
1292
1470
|
queue.waiting();
|
|
1293
|
-
|
|
1294
|
-
|
|
1471
|
+
// A whole-suite test leaf names the fewest threads at which its recorded walls keep a test finishing within
|
|
1472
|
+
// its own no-progress bound (specs/cowshed/05_gateway.md, "A minimum for a whole suite").
|
|
1473
|
+
const min = idleTimeoutMs === undefined ? 1 : suiteMinimum([...lastWalls.values()], cpus, idleTimeoutMs);
|
|
1474
|
+
const named = min > 1 ? ` min=${min}` : '';
|
|
1475
|
+
tell(`cowshed: mbx reserve wait start cpus=${cpus}${named} priority=${reservedLeaf.priority}\n`);
|
|
1476
|
+
const held = reserveCores(endpoint, cpus, reservedLeaf.priority, min);
|
|
1295
1477
|
reservation = held;
|
|
1296
1478
|
void held.granted.then((granted) => {
|
|
1297
1479
|
if (!granted.ok) {
|
|
1298
|
-
failGrant(
|
|
1480
|
+
failGrant({
|
|
1481
|
+
kind: 'reservation-refused',
|
|
1482
|
+
endpoint: granted.refusal.endpoint,
|
|
1483
|
+
errno: granted.refusal.errno,
|
|
1484
|
+
message: `mbx did not grant the reservation: ${granted.refusal.message}`,
|
|
1485
|
+
});
|
|
1299
1486
|
channel.end();
|
|
1300
1487
|
return;
|
|
1301
1488
|
}
|
|
@@ -1303,8 +1490,20 @@ async function runCommand(
|
|
|
1303
1490
|
if (state.settled || state.expiry !== null) return;
|
|
1304
1491
|
queue.admitted = true;
|
|
1305
1492
|
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1493
|
+
if (caseGate !== null) {
|
|
1494
|
+
const gate = caseGate;
|
|
1495
|
+
gated = gateCases(
|
|
1496
|
+
granted.cpus,
|
|
1497
|
+
lastWalls,
|
|
1498
|
+
(id) => gate.start(id),
|
|
1499
|
+
(id) => {
|
|
1500
|
+
const held = reserveCores(endpoint, 1, 'low', 1, `nextest:${id}`);
|
|
1501
|
+
return { granted: held.granted.then((answer) => answer.ok), release: () => held.release() };
|
|
1502
|
+
},
|
|
1503
|
+
);
|
|
1504
|
+
}
|
|
1306
1505
|
channel.end(`${granted.cpus}\n`);
|
|
1307
|
-
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1506
|
+
stepTo({ kind: 'granted', at: awakeNow(), wallAt: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1308
1507
|
void granted.lost.then((why) => {
|
|
1309
1508
|
if (state.settled || cpuReleased !== null) return;
|
|
1310
1509
|
appendStderr(
|
|
@@ -1324,11 +1523,17 @@ async function runCommand(
|
|
|
1324
1523
|
said = true;
|
|
1325
1524
|
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1326
1525
|
}
|
|
1327
|
-
stepTo({
|
|
1526
|
+
stepTo({
|
|
1527
|
+
kind: 'granted',
|
|
1528
|
+
at: awakeNow(),
|
|
1529
|
+
wallAt: Date.now(),
|
|
1530
|
+
testing: isTesting(),
|
|
1531
|
+
fixture: !fixtureDone,
|
|
1532
|
+
});
|
|
1328
1533
|
} else if (byte === 114) {
|
|
1329
|
-
stepTo({ kind: 'released', at:
|
|
1534
|
+
stepTo({ kind: 'released', at: awakeNow() });
|
|
1330
1535
|
} else {
|
|
1331
|
-
failGrant(`unexpected byte ${byte}`);
|
|
1536
|
+
failGrant(descriptorFailure(`unexpected byte ${byte}`));
|
|
1332
1537
|
return;
|
|
1333
1538
|
}
|
|
1334
1539
|
}
|
|
@@ -1365,18 +1570,25 @@ async function runCommand(
|
|
|
1365
1570
|
clearTimeout(boundTimer);
|
|
1366
1571
|
cancelCapture();
|
|
1367
1572
|
removeSignalHandlers();
|
|
1368
|
-
// The runner is gone
|
|
1369
|
-
// then the grant goes back to its parent or the host.
|
|
1573
|
+
// The runner is gone, and with it every test that could report on the run's progress channel. Its sub-pool closes
|
|
1574
|
+
// first, so no descendant still draws on tokens the grant returns; then the grant goes back to its parent or the host.
|
|
1575
|
+
await closeProgress();
|
|
1576
|
+
await closeCaseGate();
|
|
1370
1577
|
await releaseCpu();
|
|
1371
1578
|
|
|
1372
1579
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
1373
1580
|
if (code !== 0 && state.expiry === null) {
|
|
1374
|
-
//
|
|
1375
|
-
// SIGKILL): the signal is named, never folded
|
|
1581
|
+
// The executor ended it for a cause of its own, which is said; otherwise no bound of this executor ended it, so a
|
|
1582
|
+
// signal it died of came from outside (the kernel's OOM killer sends SIGKILL): the signal is named, never folded
|
|
1583
|
+
// into a bare status.
|
|
1584
|
+
const signalled = exitCode === null && exitSignal !== null;
|
|
1585
|
+
const endedBy = endedFor();
|
|
1376
1586
|
const ended =
|
|
1377
|
-
|
|
1378
|
-
? `was
|
|
1379
|
-
:
|
|
1587
|
+
endedBy !== null
|
|
1588
|
+
? `was ended by bounded-exec because ${endedBy.message} (${signalled ? `${exitSignal}, ` : ''}status ${code})`
|
|
1589
|
+
: signalled
|
|
1590
|
+
? `was killed by ${exitSignal} (status ${code})`
|
|
1591
|
+
: `exited with status ${code}`;
|
|
1380
1592
|
appendStderr(
|
|
1381
1593
|
fixture !== null && !mainStarted
|
|
1382
1594
|
? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
|
|
@@ -1411,10 +1623,29 @@ async function runCommand(
|
|
|
1411
1623
|
}
|
|
1412
1624
|
const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
|
|
1413
1625
|
|
|
1626
|
+
// A Go run sums nothing itself, and a reader that keeps part of a long output cannot count it: the sum ends the output.
|
|
1627
|
+
const goRun = progress.goSummary();
|
|
1628
|
+
if (goRun !== null) {
|
|
1629
|
+
appendStdout(
|
|
1630
|
+
goRunSummaryText(
|
|
1631
|
+
goRun,
|
|
1632
|
+
declaredGoTests(goRun.failedTests, cwd, context.root, mergeEnv(lease?.directory ?? ordinaryTemp, env)),
|
|
1633
|
+
),
|
|
1634
|
+
);
|
|
1635
|
+
}
|
|
1414
1636
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1415
1637
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1416
|
-
const operationalFailure =
|
|
1417
|
-
await recordRun(
|
|
1638
|
+
const operationalFailure = operationalCause();
|
|
1639
|
+
await recordRun(
|
|
1640
|
+
code,
|
|
1641
|
+
state.expiry,
|
|
1642
|
+
Date.now() - startedAt,
|
|
1643
|
+
operationalFailure,
|
|
1644
|
+
seen,
|
|
1645
|
+
state.lockWaits,
|
|
1646
|
+
state.cpuRising,
|
|
1647
|
+
peakBytes,
|
|
1648
|
+
);
|
|
1418
1649
|
const killedForMemory = killedFromOutside({
|
|
1419
1650
|
sentSignal: state.sentSignal,
|
|
1420
1651
|
bounded: state.expiry !== null,
|
|
@@ -1422,7 +1653,7 @@ async function runCommand(
|
|
|
1422
1653
|
code,
|
|
1423
1654
|
});
|
|
1424
1655
|
return {
|
|
1425
|
-
success:
|
|
1656
|
+
success: operationalFailure === null && resultFailure === null && state.expiry === null && code === 0,
|
|
1426
1657
|
terminalOutput: outputChunks.join(''),
|
|
1427
1658
|
...(killedForMemory ? { killedForMemory: true } : {}),
|
|
1428
1659
|
};
|
|
@@ -1443,19 +1674,6 @@ function resolveCwd(cwd: string | undefined, root: string): string {
|
|
|
1443
1674
|
return isAbsolute(cwd) ? cwd : join(root, cwd);
|
|
1444
1675
|
}
|
|
1445
1676
|
|
|
1446
|
-
function buildCommand(options: BoundedExecOptions): string {
|
|
1447
|
-
const parts = [options.command];
|
|
1448
|
-
if (Array.isArray(options.args)) {
|
|
1449
|
-
parts.push(...options.args);
|
|
1450
|
-
} else if (options.args) {
|
|
1451
|
-
parts.push(options.args);
|
|
1452
|
-
}
|
|
1453
|
-
if (options.forwardAllArgs !== false && options.__unparsed__?.length) {
|
|
1454
|
-
parts.push(...options.__unparsed__);
|
|
1455
|
-
}
|
|
1456
|
-
return parts.join(' ');
|
|
1457
|
-
}
|
|
1458
|
-
|
|
1459
1677
|
/** Owned scratch is the command's TMPDIR unless the target names one itself. */
|
|
1460
1678
|
function mergeEnv(directory: string | null, env: Record<string, string>): NodeJS.ProcessEnv {
|
|
1461
1679
|
const inherited = withoutPoolEnv(process.env);
|