@smoothbricks/nx-plugin 0.7.18 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -99
- package/dist/.build-inputs.json +35 -29
- package/dist/built-output-packages.js +1 -1
- package/dist/cargo-closure-input.js +1 -1
- package/dist/cargo-source-hash.d.ts +3 -21
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +4 -91
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +2 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +7 -5
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +14 -11
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +207 -69
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +3 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +66 -48
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +28 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +30 -9
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +8 -5
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +187 -283
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress.d.ts +3 -0
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +2 -2
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +26 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +26 -5
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +105 -73
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/task-results.js +2 -1
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1179 -133
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +105 -11
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/package.json +67 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/cargo-closure-input.ts +1 -1
- package/src/cargo-source-hash.ts +16 -132
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +1 -2
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +28 -15
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +280 -106
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +70 -52
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +36 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +213 -182
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress.ts +37 -2
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +2 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +39 -6
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +112 -78
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/workspace-config-policy.ts +161 -43
- package/test-timings.json +157 -147
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -14,11 +14,22 @@ import {
|
|
|
14
14
|
signalProcessGroup,
|
|
15
15
|
} from '../../process-group.js';
|
|
16
16
|
import { AdmissionObservation, type QueueHost } from './admission-queue.js';
|
|
17
|
+
import { atClockDeadline, openAwakeClock } from './awake-clock.js';
|
|
17
18
|
import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
19
|
+
import {
|
|
20
|
+
type CaseGateServer,
|
|
21
|
+
type GatedCases,
|
|
22
|
+
gateCases,
|
|
23
|
+
runWallMs,
|
|
24
|
+
serveCaseGate,
|
|
25
|
+
TEST_GATE_ENV,
|
|
26
|
+
TEST_GATE_SERVED_ENV,
|
|
27
|
+
} from './case-gate.js';
|
|
18
28
|
import { DIAGNOSTIC_CONTEXT_ENV, DIAGNOSTIC_MODULE_ENV, grantMemoryEnvironment } from './child-exit-diagnostics.js';
|
|
19
|
-
import { type CpuPool, createCpuPool } from './cpu-pool.js';
|
|
29
|
+
import { type CpuPool, createCpuPool, lendParentToken } from './cpu-pool.js';
|
|
20
30
|
import type { CpuRise } from './cpu-rise.js';
|
|
21
31
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
32
|
+
import { declaredGoTests, goRunSummaryText } from './go-run-summary.js';
|
|
22
33
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
23
34
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
24
35
|
import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
@@ -31,6 +42,7 @@ import {
|
|
|
31
42
|
mbxPool,
|
|
32
43
|
withCountedReservation,
|
|
33
44
|
withMbxLeaf,
|
|
45
|
+
withReservationAfterFixture,
|
|
34
46
|
} from './mbx-pool.js';
|
|
35
47
|
import { describeUnserved, RESERVATIONS_ENV, type Reservation, reserveCores, runEndpoint } from './mbx-reservations.js';
|
|
36
48
|
import { killedFromOutside, type RunOutcome, runUntilNotKilledForMemory } from './memory-retry.js';
|
|
@@ -65,6 +77,7 @@ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from
|
|
|
65
77
|
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
|
|
66
78
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
67
79
|
import { type BoundedExecOptions, boundedCommand } from './schema.js';
|
|
80
|
+
import { readLastWalls, suiteMinimum } from './suite-minimum.js';
|
|
68
81
|
import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
|
|
69
82
|
import {
|
|
70
83
|
type BoundedExecRecord,
|
|
@@ -78,6 +91,7 @@ import {
|
|
|
78
91
|
judgeOperationalFailure,
|
|
79
92
|
judgeRun,
|
|
80
93
|
type NoSpaceSighting,
|
|
94
|
+
type OperationalCause,
|
|
81
95
|
readJunitReport,
|
|
82
96
|
taskDirectory,
|
|
83
97
|
withJunitReport,
|
|
@@ -309,10 +323,14 @@ export async function runBoundedExec(
|
|
|
309
323
|
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
310
324
|
// killed run replaced, not by the peak it reached before it was killed.
|
|
311
325
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
326
|
+
// The walls its last run recorded size the minimum its reservation names (suite-minimum.ts), read before the same
|
|
327
|
+
// clear removes them.
|
|
328
|
+
const lastWalls = directory !== null ? await readLastWalls(directory) : new Map<string, number>();
|
|
312
329
|
return runUntilNotKilledForMemory(
|
|
313
330
|
(alone) =>
|
|
314
331
|
runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
315
332
|
lastPeak,
|
|
333
|
+
lastWalls,
|
|
316
334
|
alone,
|
|
317
335
|
}),
|
|
318
336
|
inherited !== null,
|
|
@@ -320,9 +338,13 @@ export async function runBoundedExec(
|
|
|
320
338
|
);
|
|
321
339
|
}
|
|
322
340
|
|
|
323
|
-
/**
|
|
341
|
+
/**
|
|
342
|
+
* What a run of the command is told that the run before it was not: the task's charge, the walls its tests last took,
|
|
343
|
+
* and whether it runs alone.
|
|
344
|
+
*/
|
|
324
345
|
interface Attempt {
|
|
325
346
|
readonly lastPeak: LastPeak | null;
|
|
347
|
+
readonly lastWalls: ReadonlyMap<string, number>;
|
|
326
348
|
readonly alone: boolean;
|
|
327
349
|
}
|
|
328
350
|
|
|
@@ -340,7 +362,7 @@ async function runCommand(
|
|
|
340
362
|
): Promise<BoundedExecResult> {
|
|
341
363
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
342
364
|
const directory = context.record?.directory ?? null;
|
|
343
|
-
const { lastPeak } = attempt;
|
|
365
|
+
const { lastPeak, lastWalls } = attempt;
|
|
344
366
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
345
367
|
if (directory !== null) {
|
|
346
368
|
await clearTaskDirectory(directory);
|
|
@@ -398,13 +420,21 @@ async function runCommand(
|
|
|
398
420
|
await writeRecord(recordDirectory, {
|
|
399
421
|
task: task.id,
|
|
400
422
|
hash: task.hash,
|
|
401
|
-
verdict: judgeOperationalFailure(1, null),
|
|
423
|
+
verdict: judgeOperationalFailure(1, null, { kind: 'executor', message }),
|
|
402
424
|
peakBytes: null,
|
|
403
425
|
});
|
|
404
426
|
}
|
|
405
427
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
406
428
|
};
|
|
407
429
|
|
|
430
|
+
let awakeNow = (): number => performance.now();
|
|
431
|
+
if (idleTimeoutMs !== undefined) {
|
|
432
|
+
try {
|
|
433
|
+
awakeNow = await openAwakeClock();
|
|
434
|
+
} catch (error) {
|
|
435
|
+
return refuseUnstarted(`bounded-exec: cannot read its awake idle clock: ${String(error)}\n`);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
408
438
|
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
409
439
|
// target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
|
|
410
440
|
if (
|
|
@@ -587,6 +617,7 @@ async function runCommand(
|
|
|
587
617
|
parallelism: chargeParallelism,
|
|
588
618
|
alone: attempt.alone,
|
|
589
619
|
task: context.record?.task.id,
|
|
620
|
+
gated: demand.kind === 'nextest' && process.env[TEST_GATE_SERVED_ENV] === '1',
|
|
590
621
|
});
|
|
591
622
|
for (const said of parent.said) appendStderr(said);
|
|
592
623
|
const { grant, refusal: parentRefusal, wait: parentWait } = parent;
|
|
@@ -596,15 +627,26 @@ async function runCommand(
|
|
|
596
627
|
// With mbx the leaf's admission is mbx's one pool, not a parent's tokens. A leaf mbx does not admit itself asks for
|
|
597
628
|
// its runner's count, a maximum, and its runner runs on the cores it is granted, which only the command learns
|
|
598
629
|
// (specs/cowshed/05_gateway.md, "Parallelism inside a job"). No grant and no sub-pool beside it.
|
|
599
|
-
const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested);
|
|
630
|
+
const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested, process.env[TEST_GATE_SERVED_ENV] === '1');
|
|
631
|
+
// A nextest leaf granted fewer of its parent pool's tokens than it asked takes the case gate where the run's mbx
|
|
632
|
+
// gates cases: its runner starts every thread of its ask, and a case that finds no token of the grant free asks the
|
|
633
|
+
// pool for one of its own (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold).
|
|
634
|
+
const lender =
|
|
635
|
+
grant !== null &&
|
|
636
|
+
inherited !== null &&
|
|
637
|
+
demand.kind === 'nextest' &&
|
|
638
|
+
grant.tokens < demand.want &&
|
|
639
|
+
process.env[TEST_GATE_SERVED_ENV] === '1'
|
|
640
|
+
? { own: grant.tokens, pool: inherited }
|
|
641
|
+
: null;
|
|
600
642
|
if (grant !== null) {
|
|
601
|
-
sized = sizeRunner(demand, requested, grant.tokens);
|
|
643
|
+
sized = sizeRunner(demand, requested, grant.tokens, lender !== null);
|
|
602
644
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
603
645
|
for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
|
|
604
646
|
appendStderr(said);
|
|
605
647
|
}
|
|
606
648
|
} else if (leaf?.kind === 'reserve') {
|
|
607
|
-
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
649
|
+
sized = { command: sizeRunnerToGrant(leaf.demand, requested, leaf.gated), env: {} };
|
|
608
650
|
} else if (leaf !== null) {
|
|
609
651
|
sized = sizeRunner(demand, requested, demand.want);
|
|
610
652
|
}
|
|
@@ -614,6 +656,26 @@ async function runCommand(
|
|
|
614
656
|
let pool: CpuPool | null = null;
|
|
615
657
|
let reservation: Reservation | null = null;
|
|
616
658
|
const releaseReservation = (): void => reservation?.release();
|
|
659
|
+
// A gated nextest leaf's case gate (case-gate.ts; specs/cowshed/05_gateway.md, "A case starts on a core"): served
|
|
660
|
+
// before the command starts, decided from the leaf's grant on, and closed once the command is gone, when every core a
|
|
661
|
+
// case still holds goes back to the pool and the leaf says what its cases ran on.
|
|
662
|
+
let caseGate: CaseGateServer | null = null;
|
|
663
|
+
let gated: GatedCases | null = null;
|
|
664
|
+
let caseGateClosed: Promise<void> | null = null;
|
|
665
|
+
const closeCaseGate = (): Promise<void> => {
|
|
666
|
+
caseGateClosed ??= (async () => {
|
|
667
|
+
const left = (await caseGate?.close()) ?? null;
|
|
668
|
+
if (left !== null) appendStderr(`bounded-exec: ${left}\n`);
|
|
669
|
+
const decided = gated;
|
|
670
|
+
if (decided === null) return;
|
|
671
|
+
decided.releaseAll();
|
|
672
|
+
const { own, lent, lentCoreMs, waitedCaseMs } = decided.cases.summary();
|
|
673
|
+
tell(
|
|
674
|
+
`cowshed: ${grant !== null ? 'cpu-pool' : 'mbx reserve'} cases own=${own} lent=${lent} lent_core_s=${Math.round(lentCoreMs / 1000)} waited_case_s=${Math.round(waitedCaseMs / 1000)}\n`,
|
|
675
|
+
);
|
|
676
|
+
})();
|
|
677
|
+
return caseGateClosed;
|
|
678
|
+
};
|
|
617
679
|
// The run's progress channel (progress-channel.ts), served when the run has an idle bound for its records to restart,
|
|
618
680
|
// and closed once its command is gone. A channel that leaves its socket behind says so; the run's verdict stands.
|
|
619
681
|
let progressChannel: ProgressChannel | null = null;
|
|
@@ -647,6 +709,7 @@ async function runCommand(
|
|
|
647
709
|
tell,
|
|
648
710
|
step: (event) => stepTo(event),
|
|
649
711
|
testing: () => isTesting(),
|
|
712
|
+
awakeNow,
|
|
650
713
|
queueHost,
|
|
651
714
|
});
|
|
652
715
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
@@ -655,7 +718,7 @@ async function runCommand(
|
|
|
655
718
|
exitCode: number,
|
|
656
719
|
expiry: ExpiredBound | null,
|
|
657
720
|
elapsedMs: number,
|
|
658
|
-
operational:
|
|
721
|
+
operational: OperationalCause | null,
|
|
659
722
|
noSpace: NoSpaceSighting | null,
|
|
660
723
|
lockWaits: readonly LockWait[],
|
|
661
724
|
cpuRising: readonly CpuRise[],
|
|
@@ -684,9 +747,11 @@ async function runCommand(
|
|
|
684
747
|
}
|
|
685
748
|
});
|
|
686
749
|
const admissionQueue = queue.queueWait(reservation !== null);
|
|
750
|
+
const cause: OperationalCause | null =
|
|
751
|
+
operational ?? (resultFailure === null ? null : { kind: 'executor', message: resultFailure });
|
|
687
752
|
const verdict =
|
|
688
|
-
|
|
689
|
-
? judgeOperationalFailure(exitCode, report)
|
|
753
|
+
cause !== null
|
|
754
|
+
? judgeOperationalFailure(exitCode, report, cause)
|
|
690
755
|
: judgeRun({
|
|
691
756
|
exitCode,
|
|
692
757
|
elapsedMs,
|
|
@@ -714,9 +779,10 @@ async function runCommand(
|
|
|
714
779
|
appendStderr(message);
|
|
715
780
|
await releaseCpu();
|
|
716
781
|
await closeProgress();
|
|
782
|
+
await closeCaseGate();
|
|
717
783
|
// The command never started, so its lease holds nothing of its own to keep.
|
|
718
784
|
await releaseLease(false);
|
|
719
|
-
await recordRun(1, null, Date.now() - asked,
|
|
785
|
+
await recordRun(1, null, Date.now() - asked, { kind: 'executor', message: message.trim() }, null, [], [], null);
|
|
720
786
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
721
787
|
};
|
|
722
788
|
if (parentRefusal !== null) {
|
|
@@ -749,12 +815,12 @@ async function runCommand(
|
|
|
749
815
|
if (idleTimeoutMs !== undefined) {
|
|
750
816
|
const served = await serveProgressChannel({
|
|
751
817
|
moved: () => {
|
|
752
|
-
if (fixtureDone) stepTo({ kind: 'reported', at:
|
|
818
|
+
if (fixtureDone) stepTo({ kind: 'reported', at: awakeNow(), testing: isTesting() });
|
|
753
819
|
},
|
|
754
820
|
invalid: (detail) => {
|
|
755
821
|
if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
|
|
756
|
-
lifecycleFailure = `progress channel: ${detail}
|
|
757
|
-
appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
|
|
822
|
+
lifecycleFailure = { kind: 'executor', message: `progress channel: ${detail}` };
|
|
823
|
+
appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
|
|
758
824
|
termination = terminate();
|
|
759
825
|
},
|
|
760
826
|
});
|
|
@@ -765,6 +831,35 @@ async function runCommand(
|
|
|
765
831
|
}
|
|
766
832
|
progressChannel = served.channel;
|
|
767
833
|
}
|
|
834
|
+
// A case that arrives before the leaf's grant has no core to take and no leaf to wait for: its connection is refused,
|
|
835
|
+
// and its shim fails it by name.
|
|
836
|
+
if (reserved?.leaf.gated === true || lender !== null) {
|
|
837
|
+
const served = await serveCaseGate({
|
|
838
|
+
arrived: (id) => gated?.cases.arrived(id) ?? false,
|
|
839
|
+
ended: (id) => gated?.cases.ended(id),
|
|
840
|
+
invalid: (detail) => {
|
|
841
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
|
|
842
|
+
lifecycleFailure = { kind: 'executor', message: `case gate: ${detail}` };
|
|
843
|
+
appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
|
|
844
|
+
termination = terminate();
|
|
845
|
+
},
|
|
846
|
+
});
|
|
847
|
+
if (!served.ok) {
|
|
848
|
+
return refuse(
|
|
849
|
+
`bounded-exec: the leaf's case gate could not be served (${served.error}); refusing to run the command\n`,
|
|
850
|
+
);
|
|
851
|
+
}
|
|
852
|
+
caseGate = served.gate;
|
|
853
|
+
if (lender !== null) {
|
|
854
|
+
const gate = served.gate;
|
|
855
|
+
gated = gateCases(
|
|
856
|
+
lender.own,
|
|
857
|
+
lastWalls,
|
|
858
|
+
(id) => gate.start(id),
|
|
859
|
+
(id, left) => lendParentToken(lender.pool, `nextest:${id}`, left),
|
|
860
|
+
);
|
|
861
|
+
}
|
|
862
|
+
}
|
|
768
863
|
const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
|
|
769
864
|
const command =
|
|
770
865
|
directory === null
|
|
@@ -783,11 +878,12 @@ async function runCommand(
|
|
|
783
878
|
...(await grantMemoryEnvironment(grant?.charged ?? null, process.env)),
|
|
784
879
|
...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
|
|
785
880
|
...(progressChannel === null ? {} : { [PROGRESS_SOCKET_ENV]: progressChannel.socket }),
|
|
881
|
+
...(caseGate === null ? {} : { [TEST_GATE_ENV]: caseGate.socket }),
|
|
786
882
|
};
|
|
787
883
|
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
788
|
-
// of the command: the progress bound runs during it and its own ceiling does not. Under mbx a leaf
|
|
789
|
-
//
|
|
790
|
-
//
|
|
884
|
+
// of the command: the progress bound runs during it and its own ceiling does not. Under mbx a reserved leaf's
|
|
885
|
+
// fixture runs before its reservation, its ceiling from spawn (`withReservationAfterFixture`); a leaf that names its
|
|
886
|
+
// units counts them at the end of that segment, its fixture's own output on stderr (`withCountedReservation`).
|
|
791
887
|
const counted =
|
|
792
888
|
process.platform !== 'win32' && leaf?.kind === 'reserve' && fixture !== null && options.unitsCommand !== undefined;
|
|
793
889
|
const segment =
|
|
@@ -796,15 +892,29 @@ async function runCommand(
|
|
|
796
892
|
process.platform === 'win32' || segment === null ? null : withFixtureLifecycle(segment, command);
|
|
797
893
|
const hooked = process.platform === 'win32' ? null : (fixtureHooked ?? withRunnerLifecycle(command));
|
|
798
894
|
const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
|
|
799
|
-
let lifecycleFailure:
|
|
895
|
+
let lifecycleFailure: OperationalCause | null = null;
|
|
896
|
+
/** Why the executor ended or failed the run whatever its command did, if it did: the verdict's operational cause. */
|
|
897
|
+
/** The cause the executor ended the command for, if it ended it (closures set `lifecycleFailure`). */
|
|
898
|
+
const endedFor = (): OperationalCause | null => lifecycleFailure;
|
|
899
|
+
const operationalCause = (): OperationalCause | null =>
|
|
900
|
+
endedFor() ??
|
|
901
|
+
(state.poolCloseError !== null
|
|
902
|
+
? { kind: 'executor', message: `cpu pool did not close cleanly: ${state.poolCloseError}` }
|
|
903
|
+
: state.retireError !== null
|
|
904
|
+
? { kind: 'executor', message: `could not retire the command's processes: ${state.retireError}` }
|
|
905
|
+
: null);
|
|
800
906
|
const isTesting = (): boolean => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
|
|
801
907
|
const startedAt = Date.now();
|
|
802
908
|
// With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
|
|
803
909
|
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
804
910
|
const unadmitted = hooked?.command ?? (segment === null ? command : `${segment} && ${command}`);
|
|
805
911
|
const shellCommand =
|
|
806
|
-
leaf?.kind === 'reserve' &&
|
|
807
|
-
?
|
|
912
|
+
leaf?.kind === 'reserve' && fixtureHooked !== null
|
|
913
|
+
? (counted ? withCountedReservation : withReservationAfterFixture)(
|
|
914
|
+
fixtureHooked.fixture,
|
|
915
|
+
fixtureHooked.main,
|
|
916
|
+
leaf,
|
|
917
|
+
)
|
|
808
918
|
: withMbxLeaf(unadmitted, leaf);
|
|
809
919
|
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
810
920
|
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
@@ -894,11 +1004,17 @@ async function runCommand(
|
|
|
894
1004
|
command,
|
|
895
1005
|
resultWriter === null
|
|
896
1006
|
? undefined
|
|
897
|
-
: (
|
|
1007
|
+
: (closed) => {
|
|
1008
|
+
// nextest times a case from its shim's start: the wait at the gate is taken off its wall and named beside it.
|
|
1009
|
+
const waitMs = gated?.cases.waitedMs(closed.id) ?? 0;
|
|
1010
|
+
const result =
|
|
1011
|
+
closed.durationMs === null || waitMs === 0
|
|
1012
|
+
? closed
|
|
1013
|
+
: { ...closed, durationMs: runWallMs(closed.durationMs, waitMs), waitMs };
|
|
898
1014
|
const error = resultWriter?.append(result) ?? null;
|
|
899
1015
|
if (error !== null && resultFailure === null) {
|
|
900
1016
|
resultFailure = error;
|
|
901
|
-
lifecycleFailure ??= error;
|
|
1017
|
+
lifecycleFailure ??= { kind: 'executor', message: error };
|
|
902
1018
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
903
1019
|
termination = terminate();
|
|
904
1020
|
}
|
|
@@ -918,56 +1034,57 @@ async function runCommand(
|
|
|
918
1034
|
let fixtureDone = fixture === null || lifecycle === null;
|
|
919
1035
|
// Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
|
|
920
1036
|
let mainStarted = fixture === null || lifecycle === null;
|
|
921
|
-
let
|
|
922
|
-
let idleCapture:
|
|
1037
|
+
let cancelCaptureTimer: (() => void) | undefined;
|
|
1038
|
+
let idleCapture:
|
|
1039
|
+
| { scheduledExpiryAtMs: number; controller: AbortController; result: IdleDiagnosticResult }
|
|
1040
|
+
| undefined;
|
|
923
1041
|
const cancelCapture = (): void => {
|
|
924
|
-
|
|
1042
|
+
cancelCaptureTimer?.();
|
|
925
1043
|
idleCapture?.controller.abort();
|
|
926
1044
|
};
|
|
927
|
-
const scheduleCapture = (
|
|
1045
|
+
const scheduleCapture = (awakeExpiresAtMs: number): void => {
|
|
928
1046
|
const capture: NonNullable<typeof idleCapture> = {
|
|
929
|
-
|
|
1047
|
+
scheduledExpiryAtMs: Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow())),
|
|
930
1048
|
controller: new AbortController(),
|
|
931
1049
|
result: { kind: 'pending', startedAtMs: null },
|
|
932
1050
|
};
|
|
933
1051
|
idleCapture = capture;
|
|
934
|
-
|
|
935
|
-
()
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
})
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
}
|
|
966
|
-
}
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
);
|
|
1052
|
+
cancelCaptureTimer = atClockDeadline(awakeExpiresAtMs - HANG_CAPTURE_MS, awakeNow, () => {
|
|
1053
|
+
if (state.settled || state.expiry !== null || idleCapture !== capture) return;
|
|
1054
|
+
const group = child.pid;
|
|
1055
|
+
if (group === undefined) {
|
|
1056
|
+
idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
|
|
1057
|
+
return;
|
|
1058
|
+
}
|
|
1059
|
+
idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
|
|
1060
|
+
capture.scheduledExpiryAtMs = Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow()));
|
|
1061
|
+
void runnerPid
|
|
1062
|
+
.then((pid) => {
|
|
1063
|
+
if (capture.controller.signal.aborted || awakeNow() >= awakeExpiresAtMs) return;
|
|
1064
|
+
if (pid === null)
|
|
1065
|
+
throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
|
|
1066
|
+
return captureDiagnostics({
|
|
1067
|
+
pid,
|
|
1068
|
+
pgid: process.platform === 'win32' ? null : group,
|
|
1069
|
+
awakeExpiresAtMs,
|
|
1070
|
+
awakeNow,
|
|
1071
|
+
signal: capture.controller.signal,
|
|
1072
|
+
});
|
|
1073
|
+
})
|
|
1074
|
+
.then(
|
|
1075
|
+
(snapshot) => {
|
|
1076
|
+
if (snapshot !== undefined && idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
|
|
1077
|
+
},
|
|
1078
|
+
(error: unknown) => {
|
|
1079
|
+
if (idleCapture === capture) {
|
|
1080
|
+
idleCapture.result = {
|
|
1081
|
+
kind: 'failed',
|
|
1082
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
1083
|
+
};
|
|
1084
|
+
}
|
|
1085
|
+
},
|
|
1086
|
+
);
|
|
1087
|
+
});
|
|
971
1088
|
};
|
|
972
1089
|
const expiredAs = (kind: BoundKind): ExpiredBound =>
|
|
973
1090
|
kind === 'idle'
|
|
@@ -976,25 +1093,27 @@ async function runCommand(
|
|
|
976
1093
|
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
977
1094
|
: { kind, limitMs: timeoutMs ?? 0 };
|
|
978
1095
|
let checkingIdle = false;
|
|
1096
|
+
const nextBound = () => earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds, awakeNow(), Date.now());
|
|
979
1097
|
const armBoundTimer = (): void => {
|
|
980
1098
|
clearTimeout(boundTimer);
|
|
981
1099
|
if (state.settled || state.expiry !== null) return;
|
|
982
|
-
const next =
|
|
1100
|
+
const next = nextBound();
|
|
983
1101
|
if (next !== null) {
|
|
984
|
-
|
|
985
|
-
()
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
Math.max(0, next.at - Date.now()),
|
|
989
|
-
);
|
|
1102
|
+
if (next.kind === 'idle' && idleCapture !== undefined && next.remainingMs > 0) {
|
|
1103
|
+
idleCapture.scheduledExpiryAtMs = Date.now() + Math.ceil(next.remainingMs);
|
|
1104
|
+
}
|
|
1105
|
+
boundTimer = setTimeout(() => void confirmExpiry(), Math.ceil(Math.max(0, next.remainingMs)));
|
|
990
1106
|
}
|
|
991
1107
|
};
|
|
992
|
-
const confirmExpiry = async (
|
|
1108
|
+
const confirmExpiry = async (): Promise<void> => {
|
|
993
1109
|
if (state.settled || state.expiry !== null) return;
|
|
994
|
-
const scheduled =
|
|
995
|
-
if (scheduled
|
|
996
|
-
if (
|
|
997
|
-
|
|
1110
|
+
const scheduled = nextBound();
|
|
1111
|
+
if (scheduled === null) return;
|
|
1112
|
+
if (scheduled.remainingMs > 0) {
|
|
1113
|
+
armBoundTimer();
|
|
1114
|
+
return;
|
|
1115
|
+
}
|
|
1116
|
+
if (scheduled.kind === 'idle' && leaf?.kind === 'cargo') {
|
|
998
1117
|
checkingIdle = true;
|
|
999
1118
|
armBoundTimer();
|
|
1000
1119
|
try {
|
|
@@ -1006,14 +1125,14 @@ async function runCommand(
|
|
|
1006
1125
|
}
|
|
1007
1126
|
}
|
|
1008
1127
|
if (state.settled || state.expiry !== null) return;
|
|
1009
|
-
const next =
|
|
1010
|
-
if (next !== null && next.
|
|
1128
|
+
const next = nextBound();
|
|
1129
|
+
if (next !== null && next.remainingMs <= 0) expire(expiredAs(next.kind));
|
|
1011
1130
|
else armBoundTimer();
|
|
1012
1131
|
};
|
|
1013
1132
|
const stepTo = (event: BoundEvent): void => {
|
|
1014
1133
|
if (state.settled || state.expiry !== null) return;
|
|
1015
1134
|
const idleBefore = bounds.idle;
|
|
1016
|
-
|
|
1135
|
+
const next = stepBounds(
|
|
1017
1136
|
bounds,
|
|
1018
1137
|
{
|
|
1019
1138
|
...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
|
|
@@ -1022,6 +1141,8 @@ async function runCommand(
|
|
|
1022
1141
|
},
|
|
1023
1142
|
event,
|
|
1024
1143
|
);
|
|
1144
|
+
if (next === bounds) return;
|
|
1145
|
+
bounds = next;
|
|
1025
1146
|
if (bounds.idle !== idleBefore) {
|
|
1026
1147
|
cancelCapture();
|
|
1027
1148
|
idleCapture = undefined;
|
|
@@ -1034,14 +1155,14 @@ async function runCommand(
|
|
|
1034
1155
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
1035
1156
|
// message extend the very bound that just fired.
|
|
1036
1157
|
const updateIdle = (finished: boolean, wasTesting: boolean): void => {
|
|
1037
|
-
stepTo({ kind: 'progress', at:
|
|
1158
|
+
stepTo({ kind: 'progress', at: awakeNow(), testing: isTesting(), wasTesting, finished });
|
|
1038
1159
|
};
|
|
1039
1160
|
const scanProgress = (stream: 'stdout' | 'stderr', text: Buffer | string): void => {
|
|
1040
1161
|
if (!fixtureDone) return;
|
|
1041
1162
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
1042
1163
|
const error = progress.error();
|
|
1043
1164
|
if (error !== null && lifecycleFailure === null) {
|
|
1044
|
-
lifecycleFailure = error;
|
|
1165
|
+
lifecycleFailure = { kind: 'executor', message: error };
|
|
1045
1166
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
1046
1167
|
termination = terminate();
|
|
1047
1168
|
}
|
|
@@ -1051,7 +1172,7 @@ async function runCommand(
|
|
|
1051
1172
|
if (!(channel instanceof Duplex)) throw new Error('runner lifecycle requires a duplex child descriptor');
|
|
1052
1173
|
const failLifecycle = (detail: string, wasTesting = isTesting()): void => {
|
|
1053
1174
|
if (lifecycleFailure !== null || state.expiry !== null) return;
|
|
1054
|
-
lifecycleFailure = detail;
|
|
1175
|
+
lifecycleFailure = { kind: 'executor', message: `runner lifecycle failed: ${detail}` };
|
|
1055
1176
|
appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
|
|
1056
1177
|
updateIdle(false, wasTesting);
|
|
1057
1178
|
};
|
|
@@ -1127,6 +1248,7 @@ async function runCommand(
|
|
|
1127
1248
|
return killChildTree();
|
|
1128
1249
|
})
|
|
1129
1250
|
.finally(closeProgress)
|
|
1251
|
+
.finally(closeCaseGate)
|
|
1130
1252
|
.finally(releaseCpu)
|
|
1131
1253
|
.finally(() => releaseLease(true))
|
|
1132
1254
|
.finally(releaseReservation)
|
|
@@ -1135,7 +1257,7 @@ async function runCommand(
|
|
|
1135
1257
|
signalToExitCode(signal),
|
|
1136
1258
|
state.expiry,
|
|
1137
1259
|
Date.now() - startedAt,
|
|
1138
|
-
|
|
1260
|
+
operationalCause(),
|
|
1139
1261
|
state.noSpace === null ? null : await state.noSpace,
|
|
1140
1262
|
state.lockWaits,
|
|
1141
1263
|
state.cpuRising,
|
|
@@ -1211,7 +1333,7 @@ async function runCommand(
|
|
|
1211
1333
|
appendStderr(
|
|
1212
1334
|
`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${
|
|
1213
1335
|
process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
|
|
1214
|
-
} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.
|
|
1336
|
+
} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.scheduledExpiryAtMs ?? 'unavailable'}\n`,
|
|
1215
1337
|
);
|
|
1216
1338
|
if (observed?.kind === 'captured') {
|
|
1217
1339
|
appendStderr(
|
|
@@ -1299,22 +1421,27 @@ async function runCommand(
|
|
|
1299
1421
|
// fire on a command the fixture already handed over to.
|
|
1300
1422
|
stepTo({
|
|
1301
1423
|
kind: 'spawn',
|
|
1302
|
-
at:
|
|
1424
|
+
at: awakeNow(),
|
|
1425
|
+
wallAt: Date.now(),
|
|
1303
1426
|
testing: isTesting(),
|
|
1304
1427
|
fixture: fixture !== null && lifecycle !== null,
|
|
1305
|
-
fixtureFirst:
|
|
1428
|
+
fixtureFirst: leaf?.kind === 'reserve' && fixtureHooked !== null,
|
|
1306
1429
|
queued,
|
|
1307
1430
|
});
|
|
1308
1431
|
if (leaf !== null) {
|
|
1309
1432
|
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1310
1433
|
if (!(channel instanceof Duplex)) throw new Error('an mbx leaf requires its mbx descriptor');
|
|
1311
|
-
const
|
|
1434
|
+
const descriptorFailure = (detail: string): OperationalCause => ({
|
|
1435
|
+
kind: 'executor',
|
|
1436
|
+
message: `mbx grant descriptor: ${detail}`,
|
|
1437
|
+
});
|
|
1438
|
+
const failGrant = (cause: OperationalCause): void => {
|
|
1312
1439
|
if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
|
|
1313
|
-
lifecycleFailure
|
|
1314
|
-
appendStderr(`bounded-exec: ${
|
|
1440
|
+
lifecycleFailure = cause;
|
|
1441
|
+
appendStderr(`bounded-exec: ${cause.message}\n`);
|
|
1315
1442
|
termination = terminate();
|
|
1316
1443
|
};
|
|
1317
|
-
channel.on('error', (error) => failGrant(error.message));
|
|
1444
|
+
channel.on('error', (error) => failGrant(descriptorFailure(error.message)));
|
|
1318
1445
|
if (reserved !== null) {
|
|
1319
1446
|
// The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
|
|
1320
1447
|
// ends the descriptor unanswered: the shell's read fails, and nothing runs.
|
|
@@ -1323,7 +1450,7 @@ async function runCommand(
|
|
|
1323
1450
|
channel.setEncoding('utf8');
|
|
1324
1451
|
channel.on('data', (text: string) => {
|
|
1325
1452
|
if (reservation !== null) {
|
|
1326
|
-
failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
|
|
1453
|
+
failGrant(descriptorFailure(`the command asked for cores a second time: ${JSON.stringify(text)}`));
|
|
1327
1454
|
return;
|
|
1328
1455
|
}
|
|
1329
1456
|
asked += text;
|
|
@@ -1332,17 +1459,30 @@ async function runCommand(
|
|
|
1332
1459
|
const line = asked.slice(0, end);
|
|
1333
1460
|
const cpus = askedCores(line, reservedLeaf.demand.want);
|
|
1334
1461
|
if (cpus === null) {
|
|
1335
|
-
failGrant(
|
|
1462
|
+
failGrant(
|
|
1463
|
+
descriptorFailure(
|
|
1464
|
+
`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`,
|
|
1465
|
+
),
|
|
1466
|
+
);
|
|
1336
1467
|
channel.end();
|
|
1337
1468
|
return;
|
|
1338
1469
|
}
|
|
1339
1470
|
queue.waiting();
|
|
1340
|
-
|
|
1341
|
-
|
|
1471
|
+
// A whole-suite test leaf names the fewest threads at which its recorded walls keep a test finishing within
|
|
1472
|
+
// its own no-progress bound (specs/cowshed/05_gateway.md, "A minimum for a whole suite").
|
|
1473
|
+
const min = idleTimeoutMs === undefined ? 1 : suiteMinimum([...lastWalls.values()], cpus, idleTimeoutMs);
|
|
1474
|
+
const named = min > 1 ? ` min=${min}` : '';
|
|
1475
|
+
tell(`cowshed: mbx reserve wait start cpus=${cpus}${named} priority=${reservedLeaf.priority}\n`);
|
|
1476
|
+
const held = reserveCores(endpoint, cpus, reservedLeaf.priority, min);
|
|
1342
1477
|
reservation = held;
|
|
1343
1478
|
void held.granted.then((granted) => {
|
|
1344
1479
|
if (!granted.ok) {
|
|
1345
|
-
failGrant(
|
|
1480
|
+
failGrant({
|
|
1481
|
+
kind: 'reservation-refused',
|
|
1482
|
+
endpoint: granted.refusal.endpoint,
|
|
1483
|
+
errno: granted.refusal.errno,
|
|
1484
|
+
message: `mbx did not grant the reservation: ${granted.refusal.message}`,
|
|
1485
|
+
});
|
|
1346
1486
|
channel.end();
|
|
1347
1487
|
return;
|
|
1348
1488
|
}
|
|
@@ -1350,8 +1490,20 @@ async function runCommand(
|
|
|
1350
1490
|
if (state.settled || state.expiry !== null) return;
|
|
1351
1491
|
queue.admitted = true;
|
|
1352
1492
|
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1493
|
+
if (caseGate !== null) {
|
|
1494
|
+
const gate = caseGate;
|
|
1495
|
+
gated = gateCases(
|
|
1496
|
+
granted.cpus,
|
|
1497
|
+
lastWalls,
|
|
1498
|
+
(id) => gate.start(id),
|
|
1499
|
+
(id) => {
|
|
1500
|
+
const held = reserveCores(endpoint, 1, 'low', 1, `nextest:${id}`);
|
|
1501
|
+
return { granted: held.granted.then((answer) => answer.ok), release: () => held.release() };
|
|
1502
|
+
},
|
|
1503
|
+
);
|
|
1504
|
+
}
|
|
1353
1505
|
channel.end(`${granted.cpus}\n`);
|
|
1354
|
-
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1506
|
+
stepTo({ kind: 'granted', at: awakeNow(), wallAt: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1355
1507
|
void granted.lost.then((why) => {
|
|
1356
1508
|
if (state.settled || cpuReleased !== null) return;
|
|
1357
1509
|
appendStderr(
|
|
@@ -1371,11 +1523,17 @@ async function runCommand(
|
|
|
1371
1523
|
said = true;
|
|
1372
1524
|
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1373
1525
|
}
|
|
1374
|
-
stepTo({
|
|
1526
|
+
stepTo({
|
|
1527
|
+
kind: 'granted',
|
|
1528
|
+
at: awakeNow(),
|
|
1529
|
+
wallAt: Date.now(),
|
|
1530
|
+
testing: isTesting(),
|
|
1531
|
+
fixture: !fixtureDone,
|
|
1532
|
+
});
|
|
1375
1533
|
} else if (byte === 114) {
|
|
1376
|
-
stepTo({ kind: 'released', at:
|
|
1534
|
+
stepTo({ kind: 'released', at: awakeNow() });
|
|
1377
1535
|
} else {
|
|
1378
|
-
failGrant(`unexpected byte ${byte}`);
|
|
1536
|
+
failGrant(descriptorFailure(`unexpected byte ${byte}`));
|
|
1379
1537
|
return;
|
|
1380
1538
|
}
|
|
1381
1539
|
}
|
|
@@ -1415,16 +1573,22 @@ async function runCommand(
|
|
|
1415
1573
|
// The runner is gone, and with it every test that could report on the run's progress channel. Its sub-pool closes
|
|
1416
1574
|
// first, so no descendant still draws on tokens the grant returns; then the grant goes back to its parent or the host.
|
|
1417
1575
|
await closeProgress();
|
|
1576
|
+
await closeCaseGate();
|
|
1418
1577
|
await releaseCpu();
|
|
1419
1578
|
|
|
1420
1579
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
1421
1580
|
if (code !== 0 && state.expiry === null) {
|
|
1422
|
-
//
|
|
1423
|
-
// SIGKILL): the signal is named, never folded
|
|
1581
|
+
// The executor ended it for a cause of its own, which is said; otherwise no bound of this executor ended it, so a
|
|
1582
|
+
// signal it died of came from outside (the kernel's OOM killer sends SIGKILL): the signal is named, never folded
|
|
1583
|
+
// into a bare status.
|
|
1584
|
+
const signalled = exitCode === null && exitSignal !== null;
|
|
1585
|
+
const endedBy = endedFor();
|
|
1424
1586
|
const ended =
|
|
1425
|
-
|
|
1426
|
-
? `was
|
|
1427
|
-
:
|
|
1587
|
+
endedBy !== null
|
|
1588
|
+
? `was ended by bounded-exec because ${endedBy.message} (${signalled ? `${exitSignal}, ` : ''}status ${code})`
|
|
1589
|
+
: signalled
|
|
1590
|
+
? `was killed by ${exitSignal} (status ${code})`
|
|
1591
|
+
: `exited with status ${code}`;
|
|
1428
1592
|
appendStderr(
|
|
1429
1593
|
fixture !== null && !mainStarted
|
|
1430
1594
|
? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
|
|
@@ -1459,9 +1623,19 @@ async function runCommand(
|
|
|
1459
1623
|
}
|
|
1460
1624
|
const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
|
|
1461
1625
|
|
|
1626
|
+
// A Go run sums nothing itself, and a reader that keeps part of a long output cannot count it: the sum ends the output.
|
|
1627
|
+
const goRun = progress.goSummary();
|
|
1628
|
+
if (goRun !== null) {
|
|
1629
|
+
appendStdout(
|
|
1630
|
+
goRunSummaryText(
|
|
1631
|
+
goRun,
|
|
1632
|
+
declaredGoTests(goRun.failedTests, cwd, context.root, mergeEnv(lease?.directory ?? ordinaryTemp, env)),
|
|
1633
|
+
),
|
|
1634
|
+
);
|
|
1635
|
+
}
|
|
1462
1636
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1463
1637
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1464
|
-
const operationalFailure =
|
|
1638
|
+
const operationalFailure = operationalCause();
|
|
1465
1639
|
await recordRun(
|
|
1466
1640
|
code,
|
|
1467
1641
|
state.expiry,
|
|
@@ -1479,7 +1653,7 @@ async function runCommand(
|
|
|
1479
1653
|
code,
|
|
1480
1654
|
});
|
|
1481
1655
|
return {
|
|
1482
|
-
success:
|
|
1656
|
+
success: operationalFailure === null && resultFailure === null && state.expiry === null && code === 0,
|
|
1483
1657
|
terminalOutput: outputChunks.join(''),
|
|
1484
1658
|
...(killedForMemory ? { killedForMemory: true } : {}),
|
|
1485
1659
|
};
|