@smoothbricks/nx-plugin 0.7.18 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -99
- package/dist/.build-inputs.json +35 -29
- package/dist/built-output-packages.js +1 -1
- package/dist/cargo-closure-input.js +1 -1
- package/dist/cargo-source-hash.d.ts +3 -21
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +4 -91
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +2 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +7 -5
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +14 -11
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +207 -69
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +3 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +66 -48
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +28 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +30 -9
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +8 -5
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +187 -283
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress.d.ts +3 -0
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +2 -2
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +26 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +26 -5
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +105 -73
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/task-results.js +2 -1
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1179 -133
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +105 -11
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/package.json +67 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/cargo-closure-input.ts +1 -1
- package/src/cargo-source-hash.ts +16 -132
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +1 -2
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +28 -15
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +280 -106
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +70 -52
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +36 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +213 -182
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress.ts +37 -2
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +2 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +39 -6
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +112 -78
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/workspace-config-policy.ts +161 -43
- package/test-timings.json +157 -147
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -8,13 +8,16 @@ import { Duplex, Readable } from 'node:stream';
|
|
|
8
8
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
9
9
|
import { DEADLINE_PATIENCE, describeMembersLeft, killProcessGroup, processGroupHasMembers, signalProcessGroup, } from '../../process-group.js';
|
|
10
10
|
import { AdmissionObservation } from './admission-queue.js';
|
|
11
|
+
import { atClockDeadline, openAwakeClock } from './awake-clock.js';
|
|
11
12
|
import { earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
13
|
+
import { gateCases, runWallMs, serveCaseGate, TEST_GATE_ENV, TEST_GATE_SERVED_ENV, } from './case-gate.js';
|
|
12
14
|
import { DIAGNOSTIC_CONTEXT_ENV, DIAGNOSTIC_MODULE_ENV, grantMemoryEnvironment } from './child-exit-diagnostics.js';
|
|
13
|
-
import { createCpuPool } from './cpu-pool.js';
|
|
15
|
+
import { createCpuPool, lendParentToken } from './cpu-pool.js';
|
|
14
16
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
17
|
+
import { declaredGoTests, goRunSummaryText } from './go-run-summary.js';
|
|
15
18
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
16
19
|
import { PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
17
|
-
import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, } from './mbx-pool.js';
|
|
20
|
+
import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, withReservationAfterFixture, } from './mbx-pool.js';
|
|
18
21
|
import { describeUnserved, RESERVATIONS_ENV, reserveCores, runEndpoint } from './mbx-reservations.js';
|
|
19
22
|
import { killedFromOutside, runUntilNotKilledForMemory } from './memory-retry.js';
|
|
20
23
|
import { createNoSpaceWatch } from './no-space.js';
|
|
@@ -29,6 +32,7 @@ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from
|
|
|
29
32
|
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
|
|
30
33
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
31
34
|
import { boundedCommand } from './schema.js';
|
|
35
|
+
import { readLastWalls, suiteMinimum } from './suite-minimum.js';
|
|
32
36
|
import { createTestResultWriter } from './test-result-writer.js';
|
|
33
37
|
import { BUN_TEST, clearTaskDirectory, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
34
38
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
@@ -161,8 +165,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
161
165
|
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
162
166
|
// killed run replaced, not by the peak it reached before it was killed.
|
|
163
167
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
168
|
+
// The walls its last run recorded size the minimum its reservation names (suite-minimum.ts), read before the same
|
|
169
|
+
// clear removes them.
|
|
170
|
+
const lastWalls = directory !== null ? await readLastWalls(directory) : new Map();
|
|
164
171
|
return runUntilNotKilledForMemory((alone) => runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
165
172
|
lastPeak,
|
|
173
|
+
lastWalls,
|
|
166
174
|
alone,
|
|
167
175
|
}), inherited !== null, context.record?.task.id);
|
|
168
176
|
}
|
|
@@ -170,7 +178,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
170
178
|
async function runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, attempt) {
|
|
171
179
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
172
180
|
const directory = context.record?.directory ?? null;
|
|
173
|
-
const { lastPeak } = attempt;
|
|
181
|
+
const { lastPeak, lastWalls } = attempt;
|
|
174
182
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
175
183
|
if (directory !== null) {
|
|
176
184
|
await clearTaskDirectory(directory);
|
|
@@ -229,12 +237,21 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
229
237
|
await writeRecord(recordDirectory, {
|
|
230
238
|
task: task.id,
|
|
231
239
|
hash: task.hash,
|
|
232
|
-
verdict: judgeOperationalFailure(1, null),
|
|
240
|
+
verdict: judgeOperationalFailure(1, null, { kind: 'executor', message }),
|
|
233
241
|
peakBytes: null,
|
|
234
242
|
});
|
|
235
243
|
}
|
|
236
244
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
237
245
|
};
|
|
246
|
+
let awakeNow = () => performance.now();
|
|
247
|
+
if (idleTimeoutMs !== undefined) {
|
|
248
|
+
try {
|
|
249
|
+
awakeNow = await openAwakeClock();
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
return refuseUnstarted(`bounded-exec: cannot read its awake idle clock: ${String(error)}\n`);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
238
255
|
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
239
256
|
// target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
|
|
240
257
|
if ((options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
|
|
@@ -402,6 +419,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
402
419
|
parallelism: chargeParallelism,
|
|
403
420
|
alone: attempt.alone,
|
|
404
421
|
task: context.record?.task.id,
|
|
422
|
+
gated: demand.kind === 'nextest' && process.env[TEST_GATE_SERVED_ENV] === '1',
|
|
405
423
|
});
|
|
406
424
|
for (const said of parent.said)
|
|
407
425
|
appendStderr(said);
|
|
@@ -412,16 +430,26 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
412
430
|
// With mbx the leaf's admission is mbx's one pool, not a parent's tokens. A leaf mbx does not admit itself asks for
|
|
413
431
|
// its runner's count, a maximum, and its runner runs on the cores it is granted, which only the command learns
|
|
414
432
|
// (specs/cowshed/05_gateway.md, "Parallelism inside a job"). No grant and no sub-pool beside it.
|
|
415
|
-
const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested);
|
|
433
|
+
const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested, process.env[TEST_GATE_SERVED_ENV] === '1');
|
|
434
|
+
// A nextest leaf granted fewer of its parent pool's tokens than it asked takes the case gate where the run's mbx
|
|
435
|
+
// gates cases: its runner starts every thread of its ask, and a case that finds no token of the grant free asks the
|
|
436
|
+
// pool for one of its own (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold).
|
|
437
|
+
const lender = grant !== null &&
|
|
438
|
+
inherited !== null &&
|
|
439
|
+
demand.kind === 'nextest' &&
|
|
440
|
+
grant.tokens < demand.want &&
|
|
441
|
+
process.env[TEST_GATE_SERVED_ENV] === '1'
|
|
442
|
+
? { own: grant.tokens, pool: inherited }
|
|
443
|
+
: null;
|
|
416
444
|
if (grant !== null) {
|
|
417
|
-
sized = sizeRunner(demand, requested, grant.tokens);
|
|
445
|
+
sized = sizeRunner(demand, requested, grant.tokens, lender !== null);
|
|
418
446
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
419
447
|
for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
|
|
420
448
|
appendStderr(said);
|
|
421
449
|
}
|
|
422
450
|
}
|
|
423
451
|
else if (leaf?.kind === 'reserve') {
|
|
424
|
-
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
452
|
+
sized = { command: sizeRunnerToGrant(leaf.demand, requested, leaf.gated), env: {} };
|
|
425
453
|
}
|
|
426
454
|
else if (leaf !== null) {
|
|
427
455
|
sized = sizeRunner(demand, requested, demand.want);
|
|
@@ -432,6 +460,26 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
432
460
|
let pool = null;
|
|
433
461
|
let reservation = null;
|
|
434
462
|
const releaseReservation = () => reservation?.release();
|
|
463
|
+
// A gated nextest leaf's case gate (case-gate.ts; specs/cowshed/05_gateway.md, "A case starts on a core"): served
|
|
464
|
+
// before the command starts, decided from the leaf's grant on, and closed once the command is gone, when every core a
|
|
465
|
+
// case still holds goes back to the pool and the leaf says what its cases ran on.
|
|
466
|
+
let caseGate = null;
|
|
467
|
+
let gated = null;
|
|
468
|
+
let caseGateClosed = null;
|
|
469
|
+
const closeCaseGate = () => {
|
|
470
|
+
caseGateClosed ??= (async () => {
|
|
471
|
+
const left = (await caseGate?.close()) ?? null;
|
|
472
|
+
if (left !== null)
|
|
473
|
+
appendStderr(`bounded-exec: ${left}\n`);
|
|
474
|
+
const decided = gated;
|
|
475
|
+
if (decided === null)
|
|
476
|
+
return;
|
|
477
|
+
decided.releaseAll();
|
|
478
|
+
const { own, lent, lentCoreMs, waitedCaseMs } = decided.cases.summary();
|
|
479
|
+
tell(`cowshed: ${grant !== null ? 'cpu-pool' : 'mbx reserve'} cases own=${own} lent=${lent} lent_core_s=${Math.round(lentCoreMs / 1000)} waited_case_s=${Math.round(waitedCaseMs / 1000)}\n`);
|
|
480
|
+
})();
|
|
481
|
+
return caseGateClosed;
|
|
482
|
+
};
|
|
435
483
|
// The run's progress channel (progress-channel.ts), served when the run has an idle bound for its records to restart,
|
|
436
484
|
// and closed once its command is gone. A channel that leaves its socket behind says so; the run's verdict stands.
|
|
437
485
|
let progressChannel = null;
|
|
@@ -467,6 +515,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
467
515
|
tell,
|
|
468
516
|
step: (event) => stepTo(event),
|
|
469
517
|
testing: () => isTesting(),
|
|
518
|
+
awakeNow,
|
|
470
519
|
queueHost,
|
|
471
520
|
});
|
|
472
521
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
@@ -492,8 +541,9 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
492
541
|
}
|
|
493
542
|
});
|
|
494
543
|
const admissionQueue = queue.queueWait(reservation !== null);
|
|
495
|
-
const
|
|
496
|
-
|
|
544
|
+
const cause = operational ?? (resultFailure === null ? null : { kind: 'executor', message: resultFailure });
|
|
545
|
+
const verdict = cause !== null
|
|
546
|
+
? judgeOperationalFailure(exitCode, report, cause)
|
|
497
547
|
: judgeRun({
|
|
498
548
|
exitCode,
|
|
499
549
|
elapsedMs,
|
|
@@ -521,9 +571,10 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
521
571
|
appendStderr(message);
|
|
522
572
|
await releaseCpu();
|
|
523
573
|
await closeProgress();
|
|
574
|
+
await closeCaseGate();
|
|
524
575
|
// The command never started, so its lease holds nothing of its own to keep.
|
|
525
576
|
await releaseLease(false);
|
|
526
|
-
await recordRun(1, null, Date.now() - asked,
|
|
577
|
+
await recordRun(1, null, Date.now() - asked, { kind: 'executor', message: message.trim() }, null, [], [], null);
|
|
527
578
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
528
579
|
};
|
|
529
580
|
if (parentRefusal !== null) {
|
|
@@ -554,13 +605,13 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
554
605
|
const served = await serveProgressChannel({
|
|
555
606
|
moved: () => {
|
|
556
607
|
if (fixtureDone)
|
|
557
|
-
stepTo({ kind: 'reported', at:
|
|
608
|
+
stepTo({ kind: 'reported', at: awakeNow(), testing: isTesting() });
|
|
558
609
|
},
|
|
559
610
|
invalid: (detail) => {
|
|
560
611
|
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
561
612
|
return;
|
|
562
|
-
lifecycleFailure = `progress channel: ${detail}
|
|
563
|
-
appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
|
|
613
|
+
lifecycleFailure = { kind: 'executor', message: `progress channel: ${detail}` };
|
|
614
|
+
appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
|
|
564
615
|
termination = terminate();
|
|
565
616
|
},
|
|
566
617
|
});
|
|
@@ -569,6 +620,29 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
569
620
|
}
|
|
570
621
|
progressChannel = served.channel;
|
|
571
622
|
}
|
|
623
|
+
// A case that arrives before the leaf's grant has no core to take and no leaf to wait for: its connection is refused,
|
|
624
|
+
// and its shim fails it by name.
|
|
625
|
+
if (reserved?.leaf.gated === true || lender !== null) {
|
|
626
|
+
const served = await serveCaseGate({
|
|
627
|
+
arrived: (id) => gated?.cases.arrived(id) ?? false,
|
|
628
|
+
ended: (id) => gated?.cases.ended(id),
|
|
629
|
+
invalid: (detail) => {
|
|
630
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
631
|
+
return;
|
|
632
|
+
lifecycleFailure = { kind: 'executor', message: `case gate: ${detail}` };
|
|
633
|
+
appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
|
|
634
|
+
termination = terminate();
|
|
635
|
+
},
|
|
636
|
+
});
|
|
637
|
+
if (!served.ok) {
|
|
638
|
+
return refuse(`bounded-exec: the leaf's case gate could not be served (${served.error}); refusing to run the command\n`);
|
|
639
|
+
}
|
|
640
|
+
caseGate = served.gate;
|
|
641
|
+
if (lender !== null) {
|
|
642
|
+
const gate = served.gate;
|
|
643
|
+
gated = gateCases(lender.own, lastWalls, (id) => gate.start(id), (id, left) => lendParentToken(lender.pool, `nextest:${id}`, left));
|
|
644
|
+
}
|
|
645
|
+
}
|
|
572
646
|
const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
|
|
573
647
|
const command = directory === null
|
|
574
648
|
? withProgressReport(native.command)
|
|
@@ -586,24 +660,34 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
586
660
|
...(await grantMemoryEnvironment(grant?.charged ?? null, process.env)),
|
|
587
661
|
...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
|
|
588
662
|
...(progressChannel === null ? {} : { [PROGRESS_SOCKET_ENV]: progressChannel.socket }),
|
|
663
|
+
...(caseGate === null ? {} : { [TEST_GATE_ENV]: caseGate.socket }),
|
|
589
664
|
};
|
|
590
665
|
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
591
|
-
// of the command: the progress bound runs during it and its own ceiling does not. Under mbx a leaf
|
|
592
|
-
//
|
|
593
|
-
//
|
|
666
|
+
// of the command: the progress bound runs during it and its own ceiling does not. Under mbx a reserved leaf's
|
|
667
|
+
// fixture runs before its reservation, its ceiling from spawn (`withReservationAfterFixture`); a leaf that names its
|
|
668
|
+
// units counts them at the end of that segment, its fixture's own output on stderr (`withCountedReservation`).
|
|
594
669
|
const counted = process.platform !== 'win32' && leaf?.kind === 'reserve' && fixture !== null && options.unitsCommand !== undefined;
|
|
595
670
|
const segment = fixture === null ? null : counted ? `{ ${fixture.command}; } >&2 && ${options.unitsCommand}` : fixture.command;
|
|
596
671
|
const fixtureHooked = process.platform === 'win32' || segment === null ? null : withFixtureLifecycle(segment, command);
|
|
597
672
|
const hooked = process.platform === 'win32' ? null : (fixtureHooked ?? withRunnerLifecycle(command));
|
|
598
673
|
const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
|
|
599
674
|
let lifecycleFailure = null;
|
|
675
|
+
/** Why the executor ended or failed the run whatever its command did, if it did: the verdict's operational cause. */
|
|
676
|
+
/** The cause the executor ended the command for, if it ended it (closures set `lifecycleFailure`). */
|
|
677
|
+
const endedFor = () => lifecycleFailure;
|
|
678
|
+
const operationalCause = () => endedFor() ??
|
|
679
|
+
(state.poolCloseError !== null
|
|
680
|
+
? { kind: 'executor', message: `cpu pool did not close cleanly: ${state.poolCloseError}` }
|
|
681
|
+
: state.retireError !== null
|
|
682
|
+
? { kind: 'executor', message: `could not retire the command's processes: ${state.retireError}` }
|
|
683
|
+
: null);
|
|
600
684
|
const isTesting = () => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
|
|
601
685
|
const startedAt = Date.now();
|
|
602
686
|
// With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
|
|
603
687
|
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
604
688
|
const unadmitted = hooked?.command ?? (segment === null ? command : `${segment} && ${command}`);
|
|
605
|
-
const shellCommand = leaf?.kind === 'reserve' &&
|
|
606
|
-
? withCountedReservation(fixtureHooked.fixture, fixtureHooked.main, leaf)
|
|
689
|
+
const shellCommand = leaf?.kind === 'reserve' && fixtureHooked !== null
|
|
690
|
+
? (counted ? withCountedReservation : withReservationAfterFixture)(fixtureHooked.fixture, fixtureHooked.main, leaf)
|
|
607
691
|
: withMbxLeaf(unadmitted, leaf);
|
|
608
692
|
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
609
693
|
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
@@ -689,11 +773,16 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
689
773
|
// keep printing. A running test reports its own progress on the progress channel instead.
|
|
690
774
|
const progress = createRunProgress(command, resultWriter === null
|
|
691
775
|
? undefined
|
|
692
|
-
: (
|
|
776
|
+
: (closed) => {
|
|
777
|
+
// nextest times a case from its shim's start: the wait at the gate is taken off its wall and named beside it.
|
|
778
|
+
const waitMs = gated?.cases.waitedMs(closed.id) ?? 0;
|
|
779
|
+
const result = closed.durationMs === null || waitMs === 0
|
|
780
|
+
? closed
|
|
781
|
+
: { ...closed, durationMs: runWallMs(closed.durationMs, waitMs), waitMs };
|
|
693
782
|
const error = resultWriter?.append(result) ?? null;
|
|
694
783
|
if (error !== null && resultFailure === null) {
|
|
695
784
|
resultFailure = error;
|
|
696
|
-
lifecycleFailure ??= error;
|
|
785
|
+
lifecycleFailure ??= { kind: 'executor', message: error };
|
|
697
786
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
698
787
|
termination = terminate();
|
|
699
788
|
}
|
|
@@ -712,20 +801,20 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
712
801
|
let fixtureDone = fixture === null || lifecycle === null;
|
|
713
802
|
// Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
|
|
714
803
|
let mainStarted = fixture === null || lifecycle === null;
|
|
715
|
-
let
|
|
804
|
+
let cancelCaptureTimer;
|
|
716
805
|
let idleCapture;
|
|
717
806
|
const cancelCapture = () => {
|
|
718
|
-
|
|
807
|
+
cancelCaptureTimer?.();
|
|
719
808
|
idleCapture?.controller.abort();
|
|
720
809
|
};
|
|
721
|
-
const scheduleCapture = (
|
|
810
|
+
const scheduleCapture = (awakeExpiresAtMs) => {
|
|
722
811
|
const capture = {
|
|
723
|
-
|
|
812
|
+
scheduledExpiryAtMs: Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow())),
|
|
724
813
|
controller: new AbortController(),
|
|
725
814
|
result: { kind: 'pending', startedAtMs: null },
|
|
726
815
|
};
|
|
727
816
|
idleCapture = capture;
|
|
728
|
-
|
|
817
|
+
cancelCaptureTimer = atClockDeadline(awakeExpiresAtMs - HANG_CAPTURE_MS, awakeNow, () => {
|
|
729
818
|
if (state.settled || state.expiry !== null || idleCapture !== capture)
|
|
730
819
|
return;
|
|
731
820
|
const group = child.pid;
|
|
@@ -734,20 +823,23 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
734
823
|
return;
|
|
735
824
|
}
|
|
736
825
|
idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
|
|
826
|
+
capture.scheduledExpiryAtMs = Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow()));
|
|
737
827
|
void runnerPid
|
|
738
828
|
.then((pid) => {
|
|
829
|
+
if (capture.controller.signal.aborted || awakeNow() >= awakeExpiresAtMs)
|
|
830
|
+
return;
|
|
739
831
|
if (pid === null)
|
|
740
832
|
throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
|
|
741
833
|
return captureDiagnostics({
|
|
742
834
|
pid,
|
|
743
835
|
pgid: process.platform === 'win32' ? null : group,
|
|
744
|
-
|
|
745
|
-
|
|
836
|
+
awakeExpiresAtMs,
|
|
837
|
+
awakeNow,
|
|
746
838
|
signal: capture.controller.signal,
|
|
747
839
|
});
|
|
748
840
|
})
|
|
749
841
|
.then((snapshot) => {
|
|
750
|
-
if (idleCapture === capture)
|
|
842
|
+
if (snapshot !== undefined && idleCapture === capture)
|
|
751
843
|
idleCapture.result = { kind: 'captured', snapshot };
|
|
752
844
|
}, (error) => {
|
|
753
845
|
if (idleCapture === capture) {
|
|
@@ -757,7 +849,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
757
849
|
};
|
|
758
850
|
}
|
|
759
851
|
});
|
|
760
|
-
}
|
|
852
|
+
});
|
|
761
853
|
};
|
|
762
854
|
const expiredAs = (kind) => kind === 'idle'
|
|
763
855
|
? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
|
|
@@ -765,26 +857,30 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
765
857
|
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
766
858
|
: { kind, limitMs: timeoutMs ?? 0 };
|
|
767
859
|
let checkingIdle = false;
|
|
860
|
+
const nextBound = () => earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds, awakeNow(), Date.now());
|
|
768
861
|
const armBoundTimer = () => {
|
|
769
862
|
clearTimeout(boundTimer);
|
|
770
863
|
if (state.settled || state.expiry !== null)
|
|
771
864
|
return;
|
|
772
|
-
const next =
|
|
865
|
+
const next = nextBound();
|
|
773
866
|
if (next !== null) {
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
}
|
|
867
|
+
if (next.kind === 'idle' && idleCapture !== undefined && next.remainingMs > 0) {
|
|
868
|
+
idleCapture.scheduledExpiryAtMs = Date.now() + Math.ceil(next.remainingMs);
|
|
869
|
+
}
|
|
870
|
+
boundTimer = setTimeout(() => void confirmExpiry(), Math.ceil(Math.max(0, next.remainingMs)));
|
|
777
871
|
}
|
|
778
872
|
};
|
|
779
|
-
const confirmExpiry = async (
|
|
873
|
+
const confirmExpiry = async () => {
|
|
780
874
|
if (state.settled || state.expiry !== null)
|
|
781
875
|
return;
|
|
782
|
-
const scheduled =
|
|
783
|
-
if (scheduled
|
|
876
|
+
const scheduled = nextBound();
|
|
877
|
+
if (scheduled === null)
|
|
784
878
|
return;
|
|
785
|
-
if (
|
|
786
|
-
|
|
787
|
-
|
|
879
|
+
if (scheduled.remainingMs > 0) {
|
|
880
|
+
armBoundTimer();
|
|
881
|
+
return;
|
|
882
|
+
}
|
|
883
|
+
if (scheduled.kind === 'idle' && leaf?.kind === 'cargo') {
|
|
788
884
|
checkingIdle = true;
|
|
789
885
|
armBoundTimer();
|
|
790
886
|
try {
|
|
@@ -798,8 +894,8 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
798
894
|
}
|
|
799
895
|
if (state.settled || state.expiry !== null)
|
|
800
896
|
return;
|
|
801
|
-
const next =
|
|
802
|
-
if (next !== null && next.
|
|
897
|
+
const next = nextBound();
|
|
898
|
+
if (next !== null && next.remainingMs <= 0)
|
|
803
899
|
expire(expiredAs(next.kind));
|
|
804
900
|
else
|
|
805
901
|
armBoundTimer();
|
|
@@ -808,11 +904,14 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
808
904
|
if (state.settled || state.expiry !== null)
|
|
809
905
|
return;
|
|
810
906
|
const idleBefore = bounds.idle;
|
|
811
|
-
|
|
907
|
+
const next = stepBounds(bounds, {
|
|
812
908
|
...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
|
|
813
909
|
...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
|
|
814
910
|
...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
|
|
815
911
|
}, event);
|
|
912
|
+
if (next === bounds)
|
|
913
|
+
return;
|
|
914
|
+
bounds = next;
|
|
816
915
|
if (bounds.idle !== idleBefore) {
|
|
817
916
|
cancelCapture();
|
|
818
917
|
idleCapture = undefined;
|
|
@@ -825,7 +924,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
825
924
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
826
925
|
// message extend the very bound that just fired.
|
|
827
926
|
const updateIdle = (finished, wasTesting) => {
|
|
828
|
-
stepTo({ kind: 'progress', at:
|
|
927
|
+
stepTo({ kind: 'progress', at: awakeNow(), testing: isTesting(), wasTesting, finished });
|
|
829
928
|
};
|
|
830
929
|
const scanProgress = (stream, text) => {
|
|
831
930
|
if (!fixtureDone)
|
|
@@ -833,7 +932,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
833
932
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
834
933
|
const error = progress.error();
|
|
835
934
|
if (error !== null && lifecycleFailure === null) {
|
|
836
|
-
lifecycleFailure = error;
|
|
935
|
+
lifecycleFailure = { kind: 'executor', message: error };
|
|
837
936
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
838
937
|
termination = terminate();
|
|
839
938
|
}
|
|
@@ -845,7 +944,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
845
944
|
const failLifecycle = (detail, wasTesting = isTesting()) => {
|
|
846
945
|
if (lifecycleFailure !== null || state.expiry !== null)
|
|
847
946
|
return;
|
|
848
|
-
lifecycleFailure = detail;
|
|
947
|
+
lifecycleFailure = { kind: 'executor', message: `runner lifecycle failed: ${detail}` };
|
|
849
948
|
appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
|
|
850
949
|
updateIdle(false, wasTesting);
|
|
851
950
|
};
|
|
@@ -924,10 +1023,11 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
924
1023
|
return killChildTree();
|
|
925
1024
|
})
|
|
926
1025
|
.finally(closeProgress)
|
|
1026
|
+
.finally(closeCaseGate)
|
|
927
1027
|
.finally(releaseCpu)
|
|
928
1028
|
.finally(() => releaseLease(true))
|
|
929
1029
|
.finally(releaseReservation)
|
|
930
|
-
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt,
|
|
1030
|
+
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, operationalCause(), state.noSpace === null ? null : await state.noSpace, state.lockWaits, state.cpuRising, null))
|
|
931
1031
|
.finally(() => process.kill(process.pid, signal));
|
|
932
1032
|
};
|
|
933
1033
|
const onSigint = () => onTerminationSignal('SIGINT');
|
|
@@ -985,7 +1085,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
985
1085
|
const observed = idleCapture?.result;
|
|
986
1086
|
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
987
1087
|
state.cpuRising = observed?.kind === 'captured' ? observed.snapshot.cpuRising : [];
|
|
988
|
-
appendStderr(`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.
|
|
1088
|
+
appendStderr(`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.scheduledExpiryAtMs ?? 'unavailable'}\n`);
|
|
989
1089
|
if (observed?.kind === 'captured') {
|
|
990
1090
|
appendStderr(`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${Date.now() - observed.snapshot.capturedAtMs}\n${observed.snapshot.text}`);
|
|
991
1091
|
}
|
|
@@ -1061,24 +1161,29 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1061
1161
|
// fire on a command the fixture already handed over to.
|
|
1062
1162
|
stepTo({
|
|
1063
1163
|
kind: 'spawn',
|
|
1064
|
-
at:
|
|
1164
|
+
at: awakeNow(),
|
|
1165
|
+
wallAt: Date.now(),
|
|
1065
1166
|
testing: isTesting(),
|
|
1066
1167
|
fixture: fixture !== null && lifecycle !== null,
|
|
1067
|
-
fixtureFirst:
|
|
1168
|
+
fixtureFirst: leaf?.kind === 'reserve' && fixtureHooked !== null,
|
|
1068
1169
|
queued,
|
|
1069
1170
|
});
|
|
1070
1171
|
if (leaf !== null) {
|
|
1071
1172
|
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1072
1173
|
if (!(channel instanceof Duplex))
|
|
1073
1174
|
throw new Error('an mbx leaf requires its mbx descriptor');
|
|
1074
|
-
const
|
|
1175
|
+
const descriptorFailure = (detail) => ({
|
|
1176
|
+
kind: 'executor',
|
|
1177
|
+
message: `mbx grant descriptor: ${detail}`,
|
|
1178
|
+
});
|
|
1179
|
+
const failGrant = (cause) => {
|
|
1075
1180
|
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
1076
1181
|
return;
|
|
1077
|
-
lifecycleFailure
|
|
1078
|
-
appendStderr(`bounded-exec: ${
|
|
1182
|
+
lifecycleFailure = cause;
|
|
1183
|
+
appendStderr(`bounded-exec: ${cause.message}\n`);
|
|
1079
1184
|
termination = terminate();
|
|
1080
1185
|
};
|
|
1081
|
-
channel.on('error', (error) => failGrant(error.message));
|
|
1186
|
+
channel.on('error', (error) => failGrant(descriptorFailure(error.message)));
|
|
1082
1187
|
if (reserved !== null) {
|
|
1083
1188
|
// The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
|
|
1084
1189
|
// ends the descriptor unanswered: the shell's read fails, and nothing runs.
|
|
@@ -1087,7 +1192,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1087
1192
|
channel.setEncoding('utf8');
|
|
1088
1193
|
channel.on('data', (text) => {
|
|
1089
1194
|
if (reservation !== null) {
|
|
1090
|
-
failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
|
|
1195
|
+
failGrant(descriptorFailure(`the command asked for cores a second time: ${JSON.stringify(text)}`));
|
|
1091
1196
|
return;
|
|
1092
1197
|
}
|
|
1093
1198
|
asked += text;
|
|
@@ -1097,17 +1202,26 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1097
1202
|
const line = asked.slice(0, end);
|
|
1098
1203
|
const cpus = askedCores(line, reservedLeaf.demand.want);
|
|
1099
1204
|
if (cpus === null) {
|
|
1100
|
-
failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
|
|
1205
|
+
failGrant(descriptorFailure(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`));
|
|
1101
1206
|
channel.end();
|
|
1102
1207
|
return;
|
|
1103
1208
|
}
|
|
1104
1209
|
queue.waiting();
|
|
1105
|
-
|
|
1106
|
-
|
|
1210
|
+
// A whole-suite test leaf names the fewest threads at which its recorded walls keep a test finishing within
|
|
1211
|
+
// its own no-progress bound (specs/cowshed/05_gateway.md, "A minimum for a whole suite").
|
|
1212
|
+
const min = idleTimeoutMs === undefined ? 1 : suiteMinimum([...lastWalls.values()], cpus, idleTimeoutMs);
|
|
1213
|
+
const named = min > 1 ? ` min=${min}` : '';
|
|
1214
|
+
tell(`cowshed: mbx reserve wait start cpus=${cpus}${named} priority=${reservedLeaf.priority}\n`);
|
|
1215
|
+
const held = reserveCores(endpoint, cpus, reservedLeaf.priority, min);
|
|
1107
1216
|
reservation = held;
|
|
1108
1217
|
void held.granted.then((granted) => {
|
|
1109
1218
|
if (!granted.ok) {
|
|
1110
|
-
failGrant(
|
|
1219
|
+
failGrant({
|
|
1220
|
+
kind: 'reservation-refused',
|
|
1221
|
+
endpoint: granted.refusal.endpoint,
|
|
1222
|
+
errno: granted.refusal.errno,
|
|
1223
|
+
message: `mbx did not grant the reservation: ${granted.refusal.message}`,
|
|
1224
|
+
});
|
|
1111
1225
|
channel.end();
|
|
1112
1226
|
return;
|
|
1113
1227
|
}
|
|
@@ -1116,8 +1230,15 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1116
1230
|
return;
|
|
1117
1231
|
queue.admitted = true;
|
|
1118
1232
|
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1233
|
+
if (caseGate !== null) {
|
|
1234
|
+
const gate = caseGate;
|
|
1235
|
+
gated = gateCases(granted.cpus, lastWalls, (id) => gate.start(id), (id) => {
|
|
1236
|
+
const held = reserveCores(endpoint, 1, 'low', 1, `nextest:${id}`);
|
|
1237
|
+
return { granted: held.granted.then((answer) => answer.ok), release: () => held.release() };
|
|
1238
|
+
});
|
|
1239
|
+
}
|
|
1119
1240
|
channel.end(`${granted.cpus}\n`);
|
|
1120
|
-
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1241
|
+
stepTo({ kind: 'granted', at: awakeNow(), wallAt: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1121
1242
|
void granted.lost.then((why) => {
|
|
1122
1243
|
if (state.settled || cpuReleased !== null)
|
|
1123
1244
|
return;
|
|
@@ -1137,13 +1258,19 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1137
1258
|
said = true;
|
|
1138
1259
|
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1139
1260
|
}
|
|
1140
|
-
stepTo({
|
|
1261
|
+
stepTo({
|
|
1262
|
+
kind: 'granted',
|
|
1263
|
+
at: awakeNow(),
|
|
1264
|
+
wallAt: Date.now(),
|
|
1265
|
+
testing: isTesting(),
|
|
1266
|
+
fixture: !fixtureDone,
|
|
1267
|
+
});
|
|
1141
1268
|
}
|
|
1142
1269
|
else if (byte === 114) {
|
|
1143
|
-
stepTo({ kind: 'released', at:
|
|
1270
|
+
stepTo({ kind: 'released', at: awakeNow() });
|
|
1144
1271
|
}
|
|
1145
1272
|
else {
|
|
1146
|
-
failGrant(`unexpected byte ${byte}`);
|
|
1273
|
+
failGrant(descriptorFailure(`unexpected byte ${byte}`));
|
|
1147
1274
|
return;
|
|
1148
1275
|
}
|
|
1149
1276
|
}
|
|
@@ -1179,14 +1306,20 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1179
1306
|
// The runner is gone, and with it every test that could report on the run's progress channel. Its sub-pool closes
|
|
1180
1307
|
// first, so no descendant still draws on tokens the grant returns; then the grant goes back to its parent or the host.
|
|
1181
1308
|
await closeProgress();
|
|
1309
|
+
await closeCaseGate();
|
|
1182
1310
|
await releaseCpu();
|
|
1183
1311
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
1184
1312
|
if (code !== 0 && state.expiry === null) {
|
|
1185
|
-
//
|
|
1186
|
-
// SIGKILL): the signal is named, never folded
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1313
|
+
// The executor ended it for a cause of its own, which is said; otherwise no bound of this executor ended it, so a
|
|
1314
|
+
// signal it died of came from outside (the kernel's OOM killer sends SIGKILL): the signal is named, never folded
|
|
1315
|
+
// into a bare status.
|
|
1316
|
+
const signalled = exitCode === null && exitSignal !== null;
|
|
1317
|
+
const endedBy = endedFor();
|
|
1318
|
+
const ended = endedBy !== null
|
|
1319
|
+
? `was ended by bounded-exec because ${endedBy.message} (${signalled ? `${exitSignal}, ` : ''}status ${code})`
|
|
1320
|
+
: signalled
|
|
1321
|
+
? `was killed by ${exitSignal} (status ${code})`
|
|
1322
|
+
: `exited with status ${code}`;
|
|
1190
1323
|
appendStderr(fixture !== null && !mainStarted
|
|
1191
1324
|
? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
|
|
1192
1325
|
: `Command ${ended}: ${command}\n`);
|
|
@@ -1215,9 +1348,14 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1215
1348
|
appendStderr(`bounded-exec: this run's memory peak was not measured (${reported.reason}); its record names none\n`);
|
|
1216
1349
|
}
|
|
1217
1350
|
const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
|
|
1351
|
+
// A Go run sums nothing itself, and a reader that keeps part of a long output cannot count it: the sum ends the output.
|
|
1352
|
+
const goRun = progress.goSummary();
|
|
1353
|
+
if (goRun !== null) {
|
|
1354
|
+
appendStdout(goRunSummaryText(goRun, declaredGoTests(goRun.failedTests, cwd, context.root, mergeEnv(lease?.directory ?? ordinaryTemp, env))));
|
|
1355
|
+
}
|
|
1218
1356
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1219
1357
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1220
|
-
const operationalFailure =
|
|
1358
|
+
const operationalFailure = operationalCause();
|
|
1221
1359
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, state.cpuRising, peakBytes);
|
|
1222
1360
|
const killedForMemory = killedFromOutside({
|
|
1223
1361
|
sentSignal: state.sentSignal,
|
|
@@ -1226,7 +1364,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1226
1364
|
code,
|
|
1227
1365
|
});
|
|
1228
1366
|
return {
|
|
1229
|
-
success:
|
|
1367
|
+
success: operationalFailure === null && resultFailure === null && state.expiry === null && code === 0,
|
|
1230
1368
|
terminalOutput: outputChunks.join(''),
|
|
1231
1369
|
...(killedForMemory ? { killedForMemory: true } : {}),
|
|
1232
1370
|
};
|
|
@@ -52,5 +52,5 @@ export type HeldLease = {
|
|
|
52
52
|
* refusal. CPU admission instead uses its own control/silence policy: no compatibility half-close,
|
|
53
53
|
* and no total queue bound (specs/cowshed/05_gateway.md, "Deadlines").
|
|
54
54
|
*/
|
|
55
|
-
export declare function holdLease(socket: string, request: Readonly<Record<string, unknown>>, what: string, bounds: LeaseBounds | CpuLeaseWait, onQueued?: () => void): Promise<HeldLease>;
|
|
55
|
+
export declare function holdLease(socket: string, request: Readonly<Record<string, unknown>>, what: string, bounds: LeaseBounds | CpuLeaseWait, onQueued?: () => void, withdrawn?: AbortSignal): Promise<HeldLease>;
|
|
56
56
|
//# sourceMappingURL=gateway-lease.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gateway-lease.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/gateway-lease.ts"],"names":[],"mappings":"AAEA,OAAO,EAA8C,YAAY,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAK5G;;;;;;;;;;GAUG;AAEH,oFAAoF;AACpF,eAAO,MAAM,cAAc,wCAAwC,CAAC;AAEpE,MAAM,WAAW,WAAW;IAC1B,wEAAwE;IACxE,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,YAAY,EAAE,MAAM,CAAC;IACrB,2FAA2F;IAC3F,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,UAAU,GAAG,gBAAgB,GAAG,OAAO,CAAC;AAE/E,gGAAgG;AAChG,MAAM,MAAM,QAAQ,GAAG;IAAE,OAAO,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,UAAU,CAAA;CAAE,CAAC;AAElH;;;;GAIG;AACH,MAAM,MAAM,SAAS,GACjB;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAAC,OAAO,EAAE,MAAM,IAAI,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,GACxG,QAAQ,CAAC;AAEb;;;;;;GAMG;AACH,wBAAgB,SAAS,CACvB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAC1C,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,WAAW,GAAG,YAAY,EAClC,QAAQ,CAAC,EAAE,MAAM,IAAI,
|
|
1
|
+
{"version":3,"file":"gateway-lease.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/gateway-lease.ts"],"names":[],"mappings":"AAEA,OAAO,EAA8C,YAAY,EAAE,KAAK,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAK5G;;;;;;;;;;GAUG;AAEH,oFAAoF;AACpF,eAAO,MAAM,cAAc,wCAAwC,CAAC;AAEpE,MAAM,WAAW,WAAW;IAC1B,wEAAwE;IACxE,KAAK,EAAE,MAAM,CAAC;IACd,sDAAsD;IACtD,YAAY,EAAE,MAAM,CAAC;IACrB,2FAA2F;IAC3F,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;CACxB;AAED;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG,QAAQ,GAAG,UAAU,GAAG,gBAAgB,GAAG,OAAO,CAAC;AAE/E,gGAAgG;AAChG,MAAM,MAAM,QAAQ,GAAG;IAAE,OAAO,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,aAAa,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,UAAU,CAAA;CAAE,CAAC;AAElH;;;;GAIG;AACH,MAAM,MAAM,SAAS,GACjB;IAAE,OAAO,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAAC,OAAO,EAAE,MAAM,IAAI,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,GACxG,QAAQ,CAAC;AAEb;;;;;;GAMG;AACH,wBAAgB,SAAS,CACvB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EAC1C,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,WAAW,GAAG,YAAY,EAClC,QAAQ,CAAC,EAAE,MAAM,IAAI,EACrB,SAAS,CAAC,EAAE,WAAW,GACtB,OAAO,CAAC,SAAS,CAAC,CAgLpB"}
|