@smoothbricks/nx-plugin 0.7.17 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +315 -114
- package/dist/.build-inputs.json +47 -38
- package/dist/built-output-packages.js +1 -1
- package/dist/bun-lock-scope.d.ts +128 -20
- package/dist/bun-lock-scope.d.ts.map +1 -1
- package/dist/bun-lock-scope.js +267 -44
- package/dist/cargo-closure-input.d.ts +8 -6
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +15 -12
- package/dist/cargo-source-hash.d.ts +26 -25
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +84 -105
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts +20 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +43 -22
- package/dist/ensure-built-receipt.d.ts +3 -2
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +3 -2
- package/dist/eslint-config-closure.d.ts +8 -3
- package/dist/eslint-config-closure.d.ts.map +1 -1
- package/dist/eslint-config-closure.js +24 -5
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +19 -6
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +22 -15
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/cpu-rise.d.ts +73 -0
- package/dist/executors/bounded-exec/cpu-rise.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-rise.js +111 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts +1 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +256 -86
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +11 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +125 -47
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +42 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +45 -8
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +12 -6
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +190 -284
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress-channel.d.ts +38 -0
- package/dist/executors/bounded-exec/progress-channel.d.ts.map +1 -0
- package/dist/executors/bounded-exec/progress-channel.js +113 -0
- package/dist/executors/bounded-exec/progress.d.ts +5 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts +24 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +123 -16
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +7 -2
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/schema.js +17 -1
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +39 -10
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +60 -14
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +294 -152
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/managed-files/generator.d.ts +5 -1
- package/dist/managed-files/generator.d.ts.map +1 -1
- package/dist/managed-files/generator.js +22 -7
- package/dist/managed-files/render-memo.d.ts +13 -0
- package/dist/managed-files/render-memo.d.ts.map +1 -0
- package/dist/managed-files/render-memo.js +94 -0
- package/dist/task-results.js +2 -1
- package/dist/test-program.d.ts +11 -6
- package/dist/test-program.d.ts.map +1 -1
- package/dist/test-program.js +39 -35
- package/dist/tool-bins.d.ts +12 -0
- package/dist/tool-bins.d.ts.map +1 -1
- package/dist/tool-bins.js +31 -6
- package/dist/typecheck-test-policy.d.ts +10 -1
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +40 -24
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/dist/workspace-index.d.ts +14 -1
- package/dist/workspace-index.d.ts.map +1 -1
- package/dist/workspace-index.js +36 -4
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1851 -221
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +122 -14
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/managed/templates/github/actions/save-nix-devenv/action.yml +19 -1
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +68 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/bun-lock-scope.ts +293 -40
- package/src/cargo-closure-input.ts +20 -12
- package/src/cargo-source-hash.ts +119 -145
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +59 -21
- package/src/ensure-built-receipt.ts +3 -2
- package/src/eslint-config-closure.ts +27 -6
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +41 -19
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/cpu-rise.ts +156 -0
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +341 -123
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +150 -50
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +57 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +216 -183
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress-channel.ts +129 -0
- package/src/executors/bounded-exec/progress.ts +39 -3
- package/src/executors/bounded-exec/ram-temp.ts +137 -17
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +21 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +83 -22
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +357 -166
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/managed-files/generator.ts +21 -6
- package/src/managed-files/render-memo.ts +135 -0
- package/src/test-program.ts +36 -41
- package/src/tool-bins.ts +35 -6
- package/src/typecheck-test-policy.ts +49 -24
- package/src/workspace-config-policy.ts +161 -43
- package/src/workspace-index.ts +40 -5
- package/test-timings.json +157 -141
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -8,13 +8,16 @@ import { Duplex, Readable } from 'node:stream';
|
|
|
8
8
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
9
9
|
import { DEADLINE_PATIENCE, describeMembersLeft, killProcessGroup, processGroupHasMembers, signalProcessGroup, } from '../../process-group.js';
|
|
10
10
|
import { AdmissionObservation } from './admission-queue.js';
|
|
11
|
+
import { atClockDeadline, openAwakeClock } from './awake-clock.js';
|
|
11
12
|
import { earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
13
|
+
import { gateCases, runWallMs, serveCaseGate, TEST_GATE_ENV, TEST_GATE_SERVED_ENV, } from './case-gate.js';
|
|
12
14
|
import { DIAGNOSTIC_CONTEXT_ENV, DIAGNOSTIC_MODULE_ENV, grantMemoryEnvironment } from './child-exit-diagnostics.js';
|
|
13
|
-
import { createCpuPool } from './cpu-pool.js';
|
|
15
|
+
import { createCpuPool, lendParentToken } from './cpu-pool.js';
|
|
14
16
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
17
|
+
import { declaredGoTests, goRunSummaryText } from './go-run-summary.js';
|
|
15
18
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
16
19
|
import { PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
17
|
-
import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, } from './mbx-pool.js';
|
|
20
|
+
import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, withReservationAfterFixture, } from './mbx-pool.js';
|
|
18
21
|
import { describeUnserved, RESERVATIONS_ENV, reserveCores, runEndpoint } from './mbx-reservations.js';
|
|
19
22
|
import { killedFromOutside, runUntilNotKilledForMemory } from './memory-retry.js';
|
|
20
23
|
import { createNoSpaceWatch } from './no-space.js';
|
|
@@ -23,10 +26,13 @@ import { askParentPool, describeGrant } from './parent-grant.js';
|
|
|
23
26
|
import { readLastPeak } from './peak-charge.js';
|
|
24
27
|
import { CPU_POOL_ENV, readInheritedCpuPool } from './pool-environment.js';
|
|
25
28
|
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
29
|
+
import { PROGRESS_SOCKET_ENV, serveProgressChannel } from './progress-channel.js';
|
|
26
30
|
import { createHostCommands, describeCoverageGap, describeRamTempError, describeSignalled, LEFTOVER_GRACE_MS, PRODUCTION, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
|
|
27
31
|
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
28
32
|
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
|
|
29
33
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
34
|
+
import { boundedCommand } from './schema.js';
|
|
35
|
+
import { readLastWalls, suiteMinimum } from './suite-minimum.js';
|
|
30
36
|
import { createTestResultWriter } from './test-result-writer.js';
|
|
31
37
|
import { BUN_TEST, clearTaskDirectory, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
32
38
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
@@ -159,8 +165,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
159
165
|
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
160
166
|
// killed run replaced, not by the peak it reached before it was killed.
|
|
161
167
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
168
|
+
// The walls its last run recorded size the minimum its reservation names (suite-minimum.ts), read before the same
|
|
169
|
+
// clear removes them.
|
|
170
|
+
const lastWalls = directory !== null ? await readLastWalls(directory) : new Map();
|
|
162
171
|
return runUntilNotKilledForMemory((alone) => runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
163
172
|
lastPeak,
|
|
173
|
+
lastWalls,
|
|
164
174
|
alone,
|
|
165
175
|
}), inherited !== null, context.record?.task.id);
|
|
166
176
|
}
|
|
@@ -168,12 +178,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
168
178
|
async function runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, attempt) {
|
|
169
179
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
170
180
|
const directory = context.record?.directory ?? null;
|
|
171
|
-
const { lastPeak } = attempt;
|
|
181
|
+
const { lastPeak, lastWalls } = attempt;
|
|
172
182
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
173
183
|
if (directory !== null) {
|
|
174
184
|
await clearTaskDirectory(directory);
|
|
175
185
|
}
|
|
176
|
-
const requested =
|
|
186
|
+
const requested = boundedCommand(options);
|
|
177
187
|
const timeoutMs = options.timeoutMs;
|
|
178
188
|
const idleTimeoutMs = options.idleTimeoutMs;
|
|
179
189
|
const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
|
|
@@ -192,6 +202,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
192
202
|
retireError: null,
|
|
193
203
|
noSpace: null,
|
|
194
204
|
lockWaits: [],
|
|
205
|
+
cpuRising: [],
|
|
195
206
|
};
|
|
196
207
|
const appendStdout = (chunk) => {
|
|
197
208
|
const text = chunk.toString();
|
|
@@ -226,12 +237,21 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
226
237
|
await writeRecord(recordDirectory, {
|
|
227
238
|
task: task.id,
|
|
228
239
|
hash: task.hash,
|
|
229
|
-
verdict: judgeOperationalFailure(1, null),
|
|
240
|
+
verdict: judgeOperationalFailure(1, null, { kind: 'executor', message }),
|
|
230
241
|
peakBytes: null,
|
|
231
242
|
});
|
|
232
243
|
}
|
|
233
244
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
234
245
|
};
|
|
246
|
+
let awakeNow = () => performance.now();
|
|
247
|
+
if (idleTimeoutMs !== undefined) {
|
|
248
|
+
try {
|
|
249
|
+
awakeNow = await openAwakeClock();
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
return refuseUnstarted(`bounded-exec: cannot read its awake idle clock: ${String(error)}\n`);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
235
255
|
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
236
256
|
// target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
|
|
237
257
|
if ((options.fixtureCommand === undefined) !== (options.fixtureTimeoutMs === undefined) ||
|
|
@@ -399,6 +419,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
399
419
|
parallelism: chargeParallelism,
|
|
400
420
|
alone: attempt.alone,
|
|
401
421
|
task: context.record?.task.id,
|
|
422
|
+
gated: demand.kind === 'nextest' && process.env[TEST_GATE_SERVED_ENV] === '1',
|
|
402
423
|
});
|
|
403
424
|
for (const said of parent.said)
|
|
404
425
|
appendStderr(said);
|
|
@@ -409,16 +430,26 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
409
430
|
// With mbx the leaf's admission is mbx's one pool, not a parent's tokens. A leaf mbx does not admit itself asks for
|
|
410
431
|
// its runner's count, a maximum, and its runner runs on the cores it is granted, which only the command learns
|
|
411
432
|
// (specs/cowshed/05_gateway.md, "Parallelism inside a job"). No grant and no sub-pool beside it.
|
|
412
|
-
const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested);
|
|
433
|
+
const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested, process.env[TEST_GATE_SERVED_ENV] === '1');
|
|
434
|
+
// A nextest leaf granted fewer of its parent pool's tokens than it asked takes the case gate where the run's mbx
|
|
435
|
+
// gates cases: its runner starts every thread of its ask, and a case that finds no token of the grant free asks the
|
|
436
|
+
// pool for one of its own (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold).
|
|
437
|
+
const lender = grant !== null &&
|
|
438
|
+
inherited !== null &&
|
|
439
|
+
demand.kind === 'nextest' &&
|
|
440
|
+
grant.tokens < demand.want &&
|
|
441
|
+
process.env[TEST_GATE_SERVED_ENV] === '1'
|
|
442
|
+
? { own: grant.tokens, pool: inherited }
|
|
443
|
+
: null;
|
|
413
444
|
if (grant !== null) {
|
|
414
|
-
sized = sizeRunner(demand, requested, grant.tokens);
|
|
445
|
+
sized = sizeRunner(demand, requested, grant.tokens, lender !== null);
|
|
415
446
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
416
447
|
for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
|
|
417
448
|
appendStderr(said);
|
|
418
449
|
}
|
|
419
450
|
}
|
|
420
451
|
else if (leaf?.kind === 'reserve') {
|
|
421
|
-
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
452
|
+
sized = { command: sizeRunnerToGrant(leaf.demand, requested, leaf.gated), env: {} };
|
|
422
453
|
}
|
|
423
454
|
else if (leaf !== null) {
|
|
424
455
|
sized = sizeRunner(demand, requested, demand.want);
|
|
@@ -429,6 +460,38 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
429
460
|
let pool = null;
|
|
430
461
|
let reservation = null;
|
|
431
462
|
const releaseReservation = () => reservation?.release();
|
|
463
|
+
// A gated nextest leaf's case gate (case-gate.ts; specs/cowshed/05_gateway.md, "A case starts on a core"): served
|
|
464
|
+
// before the command starts, decided from the leaf's grant on, and closed once the command is gone, when every core a
|
|
465
|
+
// case still holds goes back to the pool and the leaf says what its cases ran on.
|
|
466
|
+
let caseGate = null;
|
|
467
|
+
let gated = null;
|
|
468
|
+
let caseGateClosed = null;
|
|
469
|
+
const closeCaseGate = () => {
|
|
470
|
+
caseGateClosed ??= (async () => {
|
|
471
|
+
const left = (await caseGate?.close()) ?? null;
|
|
472
|
+
if (left !== null)
|
|
473
|
+
appendStderr(`bounded-exec: ${left}\n`);
|
|
474
|
+
const decided = gated;
|
|
475
|
+
if (decided === null)
|
|
476
|
+
return;
|
|
477
|
+
decided.releaseAll();
|
|
478
|
+
const { own, lent, lentCoreMs, waitedCaseMs } = decided.cases.summary();
|
|
479
|
+
tell(`cowshed: ${grant !== null ? 'cpu-pool' : 'mbx reserve'} cases own=${own} lent=${lent} lent_core_s=${Math.round(lentCoreMs / 1000)} waited_case_s=${Math.round(waitedCaseMs / 1000)}\n`);
|
|
480
|
+
})();
|
|
481
|
+
return caseGateClosed;
|
|
482
|
+
};
|
|
483
|
+
// The run's progress channel (progress-channel.ts), served when the run has an idle bound for its records to restart,
|
|
484
|
+
// and closed once its command is gone. A channel that leaves its socket behind says so; the run's verdict stands.
|
|
485
|
+
let progressChannel = null;
|
|
486
|
+
let progressClosed = null;
|
|
487
|
+
const closeProgress = () => {
|
|
488
|
+
progressClosed ??= (async () => {
|
|
489
|
+
const left = (await progressChannel?.close()) ?? null;
|
|
490
|
+
if (left !== null)
|
|
491
|
+
appendStderr(`bounded-exec: ${left}\n`);
|
|
492
|
+
})();
|
|
493
|
+
return progressClosed;
|
|
494
|
+
};
|
|
432
495
|
let cpuReleased = null;
|
|
433
496
|
const releaseCpu = () => {
|
|
434
497
|
cpuReleased ??= (async () => {
|
|
@@ -452,11 +515,12 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
452
515
|
tell,
|
|
453
516
|
step: (event) => stepTo(event),
|
|
454
517
|
testing: () => isTesting(),
|
|
518
|
+
awakeNow,
|
|
455
519
|
queueHost,
|
|
456
520
|
});
|
|
457
521
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
458
522
|
// Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
|
|
459
|
-
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits, peakBytes) => {
|
|
523
|
+
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits, cpuRising, peakBytes) => {
|
|
460
524
|
if (context.record === undefined) {
|
|
461
525
|
return;
|
|
462
526
|
}
|
|
@@ -477,8 +541,9 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
477
541
|
}
|
|
478
542
|
});
|
|
479
543
|
const admissionQueue = queue.queueWait(reservation !== null);
|
|
480
|
-
const
|
|
481
|
-
|
|
544
|
+
const cause = operational ?? (resultFailure === null ? null : { kind: 'executor', message: resultFailure });
|
|
545
|
+
const verdict = cause !== null
|
|
546
|
+
? judgeOperationalFailure(exitCode, report, cause)
|
|
482
547
|
: judgeRun({
|
|
483
548
|
exitCode,
|
|
484
549
|
elapsedMs,
|
|
@@ -486,6 +551,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
486
551
|
report,
|
|
487
552
|
noSpace,
|
|
488
553
|
lockWaits,
|
|
554
|
+
cpuRising,
|
|
489
555
|
admissionQueue,
|
|
490
556
|
});
|
|
491
557
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
@@ -504,9 +570,11 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
504
570
|
const refuse = async (message) => {
|
|
505
571
|
appendStderr(message);
|
|
506
572
|
await releaseCpu();
|
|
573
|
+
await closeProgress();
|
|
574
|
+
await closeCaseGate();
|
|
507
575
|
// The command never started, so its lease holds nothing of its own to keep.
|
|
508
576
|
await releaseLease(false);
|
|
509
|
-
await recordRun(1, null, Date.now() - asked,
|
|
577
|
+
await recordRun(1, null, Date.now() - asked, { kind: 'executor', message: message.trim() }, null, [], [], null);
|
|
510
578
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
511
579
|
};
|
|
512
580
|
if (parentRefusal !== null) {
|
|
@@ -529,6 +597,52 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
529
597
|
}
|
|
530
598
|
pool = created.pool;
|
|
531
599
|
}
|
|
600
|
+
// A running test reports its own progress on the run's progress channel, and each record restarts the idle bound as a
|
|
601
|
+
// finished test does; a record that is not one fails the run as a broken reporter. A run with no idle bound has
|
|
602
|
+
// nothing for a record to restart and is served none. Only the command learns the socket, from its environment, so
|
|
603
|
+
// these handlers run after it is spawned and its bounds are wired below.
|
|
604
|
+
if (idleTimeoutMs !== undefined) {
|
|
605
|
+
const served = await serveProgressChannel({
|
|
606
|
+
moved: () => {
|
|
607
|
+
if (fixtureDone)
|
|
608
|
+
stepTo({ kind: 'reported', at: awakeNow(), testing: isTesting() });
|
|
609
|
+
},
|
|
610
|
+
invalid: (detail) => {
|
|
611
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
612
|
+
return;
|
|
613
|
+
lifecycleFailure = { kind: 'executor', message: `progress channel: ${detail}` };
|
|
614
|
+
appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
|
|
615
|
+
termination = terminate();
|
|
616
|
+
},
|
|
617
|
+
});
|
|
618
|
+
if (!served.ok) {
|
|
619
|
+
return refuse(`bounded-exec: the run's progress channel could not be served (${served.error}); refusing to run the command\n`);
|
|
620
|
+
}
|
|
621
|
+
progressChannel = served.channel;
|
|
622
|
+
}
|
|
623
|
+
// A case that arrives before the leaf's grant has no core to take and no leaf to wait for: its connection is refused,
|
|
624
|
+
// and its shim fails it by name.
|
|
625
|
+
if (reserved?.leaf.gated === true || lender !== null) {
|
|
626
|
+
const served = await serveCaseGate({
|
|
627
|
+
arrived: (id) => gated?.cases.arrived(id) ?? false,
|
|
628
|
+
ended: (id) => gated?.cases.ended(id),
|
|
629
|
+
invalid: (detail) => {
|
|
630
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
631
|
+
return;
|
|
632
|
+
lifecycleFailure = { kind: 'executor', message: `case gate: ${detail}` };
|
|
633
|
+
appendStderr(`bounded-exec: ${lifecycleFailure.message}\n`);
|
|
634
|
+
termination = terminate();
|
|
635
|
+
},
|
|
636
|
+
});
|
|
637
|
+
if (!served.ok) {
|
|
638
|
+
return refuse(`bounded-exec: the leaf's case gate could not be served (${served.error}); refusing to run the command\n`);
|
|
639
|
+
}
|
|
640
|
+
caseGate = served.gate;
|
|
641
|
+
if (lender !== null) {
|
|
642
|
+
const gate = served.gate;
|
|
643
|
+
gated = gateCases(lender.own, lastWalls, (id) => gate.start(id), (id, left) => lendParentToken(lender.pool, `nextest:${id}`, left));
|
|
644
|
+
}
|
|
645
|
+
}
|
|
532
646
|
const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
|
|
533
647
|
const command = directory === null
|
|
534
648
|
? withProgressReport(native.command)
|
|
@@ -545,24 +659,35 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
545
659
|
...(pool?.environment ?? {}),
|
|
546
660
|
...(await grantMemoryEnvironment(grant?.charged ?? null, process.env)),
|
|
547
661
|
...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
|
|
662
|
+
...(progressChannel === null ? {} : { [PROGRESS_SOCKET_ENV]: progressChannel.socket }),
|
|
663
|
+
...(caseGate === null ? {} : { [TEST_GATE_ENV]: caseGate.socket }),
|
|
548
664
|
};
|
|
549
665
|
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
550
|
-
// of the command: the progress bound runs during it and its own ceiling does not. Under mbx a leaf
|
|
551
|
-
//
|
|
552
|
-
//
|
|
666
|
+
// of the command: the progress bound runs during it and its own ceiling does not. Under mbx a reserved leaf's
|
|
667
|
+
// fixture runs before its reservation, its ceiling from spawn (`withReservationAfterFixture`); a leaf that names its
|
|
668
|
+
// units counts them at the end of that segment, its fixture's own output on stderr (`withCountedReservation`).
|
|
553
669
|
const counted = process.platform !== 'win32' && leaf?.kind === 'reserve' && fixture !== null && options.unitsCommand !== undefined;
|
|
554
670
|
const segment = fixture === null ? null : counted ? `{ ${fixture.command}; } >&2 && ${options.unitsCommand}` : fixture.command;
|
|
555
671
|
const fixtureHooked = process.platform === 'win32' || segment === null ? null : withFixtureLifecycle(segment, command);
|
|
556
672
|
const hooked = process.platform === 'win32' ? null : (fixtureHooked ?? withRunnerLifecycle(command));
|
|
557
673
|
const lifecycle = hooked === null ? null : createRunnerLifecycle(hooked.runners);
|
|
558
674
|
let lifecycleFailure = null;
|
|
675
|
+
/** Why the executor ended or failed the run whatever its command did, if it did: the verdict's operational cause. */
|
|
676
|
+
/** The cause the executor ended the command for, if it ended it (closures set `lifecycleFailure`). */
|
|
677
|
+
const endedFor = () => lifecycleFailure;
|
|
678
|
+
const operationalCause = () => endedFor() ??
|
|
679
|
+
(state.poolCloseError !== null
|
|
680
|
+
? { kind: 'executor', message: `cpu pool did not close cleanly: ${state.poolCloseError}` }
|
|
681
|
+
: state.retireError !== null
|
|
682
|
+
? { kind: 'executor', message: `could not retire the command's processes: ${state.retireError}` }
|
|
683
|
+
: null);
|
|
559
684
|
const isTesting = () => lifecycleFailure !== null || (lifecycle?.isTesting() ?? true);
|
|
560
685
|
const startedAt = Date.now();
|
|
561
686
|
// With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
|
|
562
687
|
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
563
688
|
const unadmitted = hooked?.command ?? (segment === null ? command : `${segment} && ${command}`);
|
|
564
|
-
const shellCommand = leaf?.kind === 'reserve' &&
|
|
565
|
-
? withCountedReservation(fixtureHooked.fixture, fixtureHooked.main, leaf)
|
|
689
|
+
const shellCommand = leaf?.kind === 'reserve' && fixtureHooked !== null
|
|
690
|
+
? (counted ? withCountedReservation : withReservationAfterFixture)(fixtureHooked.fixture, fixtureHooked.main, leaf)
|
|
566
691
|
: withMbxLeaf(unadmitted, leaf);
|
|
567
692
|
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
568
693
|
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
@@ -644,14 +769,20 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
644
769
|
return answer;
|
|
645
770
|
});
|
|
646
771
|
}
|
|
647
|
-
// Progress is a finished test,
|
|
772
|
+
// Progress in the output is a finished test, never output itself: a runner's SLOW warnings and a hung test's heartbeat
|
|
773
|
+
// keep printing. A running test reports its own progress on the progress channel instead.
|
|
648
774
|
const progress = createRunProgress(command, resultWriter === null
|
|
649
775
|
? undefined
|
|
650
|
-
: (
|
|
776
|
+
: (closed) => {
|
|
777
|
+
// nextest times a case from its shim's start: the wait at the gate is taken off its wall and named beside it.
|
|
778
|
+
const waitMs = gated?.cases.waitedMs(closed.id) ?? 0;
|
|
779
|
+
const result = closed.durationMs === null || waitMs === 0
|
|
780
|
+
? closed
|
|
781
|
+
: { ...closed, durationMs: runWallMs(closed.durationMs, waitMs), waitMs };
|
|
651
782
|
const error = resultWriter?.append(result) ?? null;
|
|
652
783
|
if (error !== null && resultFailure === null) {
|
|
653
784
|
resultFailure = error;
|
|
654
|
-
lifecycleFailure ??= error;
|
|
785
|
+
lifecycleFailure ??= { kind: 'executor', message: error };
|
|
655
786
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
656
787
|
termination = terminate();
|
|
657
788
|
}
|
|
@@ -670,20 +801,20 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
670
801
|
let fixtureDone = fixture === null || lifecycle === null;
|
|
671
802
|
// Whether a segment past the fixture ever announced: without one, the shell's exit is the fixture's.
|
|
672
803
|
let mainStarted = fixture === null || lifecycle === null;
|
|
673
|
-
let
|
|
804
|
+
let cancelCaptureTimer;
|
|
674
805
|
let idleCapture;
|
|
675
806
|
const cancelCapture = () => {
|
|
676
|
-
|
|
807
|
+
cancelCaptureTimer?.();
|
|
677
808
|
idleCapture?.controller.abort();
|
|
678
809
|
};
|
|
679
|
-
const scheduleCapture = (
|
|
810
|
+
const scheduleCapture = (awakeExpiresAtMs) => {
|
|
680
811
|
const capture = {
|
|
681
|
-
|
|
812
|
+
scheduledExpiryAtMs: Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow())),
|
|
682
813
|
controller: new AbortController(),
|
|
683
814
|
result: { kind: 'pending', startedAtMs: null },
|
|
684
815
|
};
|
|
685
816
|
idleCapture = capture;
|
|
686
|
-
|
|
817
|
+
cancelCaptureTimer = atClockDeadline(awakeExpiresAtMs - HANG_CAPTURE_MS, awakeNow, () => {
|
|
687
818
|
if (state.settled || state.expiry !== null || idleCapture !== capture)
|
|
688
819
|
return;
|
|
689
820
|
const group = child.pid;
|
|
@@ -692,20 +823,23 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
692
823
|
return;
|
|
693
824
|
}
|
|
694
825
|
idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
|
|
826
|
+
capture.scheduledExpiryAtMs = Date.now() + Math.ceil(Math.max(0, awakeExpiresAtMs - awakeNow()));
|
|
695
827
|
void runnerPid
|
|
696
828
|
.then((pid) => {
|
|
829
|
+
if (capture.controller.signal.aborted || awakeNow() >= awakeExpiresAtMs)
|
|
830
|
+
return;
|
|
697
831
|
if (pid === null)
|
|
698
832
|
throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
|
|
699
833
|
return captureDiagnostics({
|
|
700
834
|
pid,
|
|
701
835
|
pgid: process.platform === 'win32' ? null : group,
|
|
702
|
-
|
|
703
|
-
|
|
836
|
+
awakeExpiresAtMs,
|
|
837
|
+
awakeNow,
|
|
704
838
|
signal: capture.controller.signal,
|
|
705
839
|
});
|
|
706
840
|
})
|
|
707
841
|
.then((snapshot) => {
|
|
708
|
-
if (idleCapture === capture)
|
|
842
|
+
if (snapshot !== undefined && idleCapture === capture)
|
|
709
843
|
idleCapture.result = { kind: 'captured', snapshot };
|
|
710
844
|
}, (error) => {
|
|
711
845
|
if (idleCapture === capture) {
|
|
@@ -715,7 +849,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
715
849
|
};
|
|
716
850
|
}
|
|
717
851
|
});
|
|
718
|
-
}
|
|
852
|
+
});
|
|
719
853
|
};
|
|
720
854
|
const expiredAs = (kind) => kind === 'idle'
|
|
721
855
|
? { kind, limitMs: idleTimeoutMs ?? 0, tests: progress.inFlight() }
|
|
@@ -723,26 +857,30 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
723
857
|
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
724
858
|
: { kind, limitMs: timeoutMs ?? 0 };
|
|
725
859
|
let checkingIdle = false;
|
|
860
|
+
const nextBound = () => earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds, awakeNow(), Date.now());
|
|
726
861
|
const armBoundTimer = () => {
|
|
727
862
|
clearTimeout(boundTimer);
|
|
728
863
|
if (state.settled || state.expiry !== null)
|
|
729
864
|
return;
|
|
730
|
-
const next =
|
|
865
|
+
const next = nextBound();
|
|
731
866
|
if (next !== null) {
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
}
|
|
867
|
+
if (next.kind === 'idle' && idleCapture !== undefined && next.remainingMs > 0) {
|
|
868
|
+
idleCapture.scheduledExpiryAtMs = Date.now() + Math.ceil(next.remainingMs);
|
|
869
|
+
}
|
|
870
|
+
boundTimer = setTimeout(() => void confirmExpiry(), Math.ceil(Math.max(0, next.remainingMs)));
|
|
735
871
|
}
|
|
736
872
|
};
|
|
737
|
-
const confirmExpiry = async (
|
|
873
|
+
const confirmExpiry = async () => {
|
|
738
874
|
if (state.settled || state.expiry !== null)
|
|
739
875
|
return;
|
|
740
|
-
const scheduled =
|
|
741
|
-
if (scheduled
|
|
876
|
+
const scheduled = nextBound();
|
|
877
|
+
if (scheduled === null)
|
|
742
878
|
return;
|
|
743
|
-
if (
|
|
744
|
-
|
|
745
|
-
|
|
879
|
+
if (scheduled.remainingMs > 0) {
|
|
880
|
+
armBoundTimer();
|
|
881
|
+
return;
|
|
882
|
+
}
|
|
883
|
+
if (scheduled.kind === 'idle' && leaf?.kind === 'cargo') {
|
|
746
884
|
checkingIdle = true;
|
|
747
885
|
armBoundTimer();
|
|
748
886
|
try {
|
|
@@ -756,8 +894,8 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
756
894
|
}
|
|
757
895
|
if (state.settled || state.expiry !== null)
|
|
758
896
|
return;
|
|
759
|
-
const next =
|
|
760
|
-
if (next !== null && next.
|
|
897
|
+
const next = nextBound();
|
|
898
|
+
if (next !== null && next.remainingMs <= 0)
|
|
761
899
|
expire(expiredAs(next.kind));
|
|
762
900
|
else
|
|
763
901
|
armBoundTimer();
|
|
@@ -766,11 +904,14 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
766
904
|
if (state.settled || state.expiry !== null)
|
|
767
905
|
return;
|
|
768
906
|
const idleBefore = bounds.idle;
|
|
769
|
-
|
|
907
|
+
const next = stepBounds(bounds, {
|
|
770
908
|
...(idleTimeoutMs === undefined ? {} : { idleMs: idleTimeoutMs }),
|
|
771
909
|
...(fixture === null || lifecycle === null ? {} : { fixtureMs: fixture.timeoutMs }),
|
|
772
910
|
...(timeoutMs === undefined ? {} : { totalMs: timeoutMs }),
|
|
773
911
|
}, event);
|
|
912
|
+
if (next === bounds)
|
|
913
|
+
return;
|
|
914
|
+
bounds = next;
|
|
774
915
|
if (bounds.idle !== idleBefore) {
|
|
775
916
|
cancelCapture();
|
|
776
917
|
idleCapture = undefined;
|
|
@@ -783,7 +924,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
783
924
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
784
925
|
// message extend the very bound that just fired.
|
|
785
926
|
const updateIdle = (finished, wasTesting) => {
|
|
786
|
-
stepTo({ kind: 'progress', at:
|
|
927
|
+
stepTo({ kind: 'progress', at: awakeNow(), testing: isTesting(), wasTesting, finished });
|
|
787
928
|
};
|
|
788
929
|
const scanProgress = (stream, text) => {
|
|
789
930
|
if (!fixtureDone)
|
|
@@ -791,7 +932,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
791
932
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
792
933
|
const error = progress.error();
|
|
793
934
|
if (error !== null && lifecycleFailure === null) {
|
|
794
|
-
lifecycleFailure = error;
|
|
935
|
+
lifecycleFailure = { kind: 'executor', message: error };
|
|
795
936
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
796
937
|
termination = terminate();
|
|
797
938
|
}
|
|
@@ -803,7 +944,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
803
944
|
const failLifecycle = (detail, wasTesting = isTesting()) => {
|
|
804
945
|
if (lifecycleFailure !== null || state.expiry !== null)
|
|
805
946
|
return;
|
|
806
|
-
lifecycleFailure = detail;
|
|
947
|
+
lifecycleFailure = { kind: 'executor', message: `runner lifecycle failed: ${detail}` };
|
|
807
948
|
appendStderr(`bounded-exec: runner lifecycle failed: ${detail}\n`);
|
|
808
949
|
updateIdle(false, wasTesting);
|
|
809
950
|
};
|
|
@@ -881,10 +1022,12 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
881
1022
|
queue.stop();
|
|
882
1023
|
return killChildTree();
|
|
883
1024
|
})
|
|
1025
|
+
.finally(closeProgress)
|
|
1026
|
+
.finally(closeCaseGate)
|
|
884
1027
|
.finally(releaseCpu)
|
|
885
1028
|
.finally(() => releaseLease(true))
|
|
886
1029
|
.finally(releaseReservation)
|
|
887
|
-
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt,
|
|
1030
|
+
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, operationalCause(), state.noSpace === null ? null : await state.noSpace, state.lockWaits, state.cpuRising, null))
|
|
888
1031
|
.finally(() => process.kill(process.pid, signal));
|
|
889
1032
|
};
|
|
890
1033
|
const onSigint = () => onTerminationSignal('SIGINT');
|
|
@@ -930,7 +1073,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
930
1073
|
clearTimeout(boundTimer);
|
|
931
1074
|
const elapsedMs = Date.now() - startedAt;
|
|
932
1075
|
appendStderr(bound.kind === 'idle'
|
|
933
|
-
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
1076
|
+
? `\nCommand made no progress: no test finished or reported progress for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
934
1077
|
? `still running: ${bound.tests.join(', ')}`
|
|
935
1078
|
: 'the runner named no test still running'} (cwd=${cwd}): ${command}\n${queue.unconfirmed === null
|
|
936
1079
|
? ''
|
|
@@ -941,7 +1084,8 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
941
1084
|
if (bound.kind === 'idle') {
|
|
942
1085
|
const observed = idleCapture?.result;
|
|
943
1086
|
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
944
|
-
|
|
1087
|
+
state.cpuRising = observed?.kind === 'captured' ? observed.snapshot.cpuRising : [];
|
|
1088
|
+
appendStderr(`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.scheduledExpiryAtMs ?? 'unavailable'}\n`);
|
|
945
1089
|
if (observed?.kind === 'captured') {
|
|
946
1090
|
appendStderr(`Hang diagnostics: outcome=${observed.snapshot.outcome} capturedAtMs=${observed.snapshot.capturedAtMs} actualExpiryOffsetMs=${Date.now() - observed.snapshot.capturedAtMs}\n${observed.snapshot.text}`);
|
|
947
1091
|
}
|
|
@@ -1017,24 +1161,29 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1017
1161
|
// fire on a command the fixture already handed over to.
|
|
1018
1162
|
stepTo({
|
|
1019
1163
|
kind: 'spawn',
|
|
1020
|
-
at:
|
|
1164
|
+
at: awakeNow(),
|
|
1165
|
+
wallAt: Date.now(),
|
|
1021
1166
|
testing: isTesting(),
|
|
1022
1167
|
fixture: fixture !== null && lifecycle !== null,
|
|
1023
|
-
fixtureFirst:
|
|
1168
|
+
fixtureFirst: leaf?.kind === 'reserve' && fixtureHooked !== null,
|
|
1024
1169
|
queued,
|
|
1025
1170
|
});
|
|
1026
1171
|
if (leaf !== null) {
|
|
1027
1172
|
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1028
1173
|
if (!(channel instanceof Duplex))
|
|
1029
1174
|
throw new Error('an mbx leaf requires its mbx descriptor');
|
|
1030
|
-
const
|
|
1175
|
+
const descriptorFailure = (detail) => ({
|
|
1176
|
+
kind: 'executor',
|
|
1177
|
+
message: `mbx grant descriptor: ${detail}`,
|
|
1178
|
+
});
|
|
1179
|
+
const failGrant = (cause) => {
|
|
1031
1180
|
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
1032
1181
|
return;
|
|
1033
|
-
lifecycleFailure
|
|
1034
|
-
appendStderr(`bounded-exec: ${
|
|
1182
|
+
lifecycleFailure = cause;
|
|
1183
|
+
appendStderr(`bounded-exec: ${cause.message}\n`);
|
|
1035
1184
|
termination = terminate();
|
|
1036
1185
|
};
|
|
1037
|
-
channel.on('error', (error) => failGrant(error.message));
|
|
1186
|
+
channel.on('error', (error) => failGrant(descriptorFailure(error.message)));
|
|
1038
1187
|
if (reserved !== null) {
|
|
1039
1188
|
// The shell asks for its cores in one line and reads the count granted (mbx-pool.ts, `reservation`). A refusal
|
|
1040
1189
|
// ends the descriptor unanswered: the shell's read fails, and nothing runs.
|
|
@@ -1043,7 +1192,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1043
1192
|
channel.setEncoding('utf8');
|
|
1044
1193
|
channel.on('data', (text) => {
|
|
1045
1194
|
if (reservation !== null) {
|
|
1046
|
-
failGrant(`the command asked for cores a second time: ${JSON.stringify(text)}`);
|
|
1195
|
+
failGrant(descriptorFailure(`the command asked for cores a second time: ${JSON.stringify(text)}`));
|
|
1047
1196
|
return;
|
|
1048
1197
|
}
|
|
1049
1198
|
asked += text;
|
|
@@ -1053,17 +1202,26 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1053
1202
|
const line = asked.slice(0, end);
|
|
1054
1203
|
const cpus = askedCores(line, reservedLeaf.demand.want);
|
|
1055
1204
|
if (cpus === null) {
|
|
1056
|
-
failGrant(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`);
|
|
1205
|
+
failGrant(descriptorFailure(`the command asked for ${JSON.stringify(line)} cores, not 1 to ${reservedLeaf.demand.want}`));
|
|
1057
1206
|
channel.end();
|
|
1058
1207
|
return;
|
|
1059
1208
|
}
|
|
1060
1209
|
queue.waiting();
|
|
1061
|
-
|
|
1062
|
-
|
|
1210
|
+
// A whole-suite test leaf names the fewest threads at which its recorded walls keep a test finishing within
|
|
1211
|
+
// its own no-progress bound (specs/cowshed/05_gateway.md, "A minimum for a whole suite").
|
|
1212
|
+
const min = idleTimeoutMs === undefined ? 1 : suiteMinimum([...lastWalls.values()], cpus, idleTimeoutMs);
|
|
1213
|
+
const named = min > 1 ? ` min=${min}` : '';
|
|
1214
|
+
tell(`cowshed: mbx reserve wait start cpus=${cpus}${named} priority=${reservedLeaf.priority}\n`);
|
|
1215
|
+
const held = reserveCores(endpoint, cpus, reservedLeaf.priority, min);
|
|
1063
1216
|
reservation = held;
|
|
1064
1217
|
void held.granted.then((granted) => {
|
|
1065
1218
|
if (!granted.ok) {
|
|
1066
|
-
failGrant(
|
|
1219
|
+
failGrant({
|
|
1220
|
+
kind: 'reservation-refused',
|
|
1221
|
+
endpoint: granted.refusal.endpoint,
|
|
1222
|
+
errno: granted.refusal.errno,
|
|
1223
|
+
message: `mbx did not grant the reservation: ${granted.refusal.message}`,
|
|
1224
|
+
});
|
|
1067
1225
|
channel.end();
|
|
1068
1226
|
return;
|
|
1069
1227
|
}
|
|
@@ -1072,8 +1230,15 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1072
1230
|
return;
|
|
1073
1231
|
queue.admitted = true;
|
|
1074
1232
|
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1233
|
+
if (caseGate !== null) {
|
|
1234
|
+
const gate = caseGate;
|
|
1235
|
+
gated = gateCases(granted.cpus, lastWalls, (id) => gate.start(id), (id) => {
|
|
1236
|
+
const held = reserveCores(endpoint, 1, 'low', 1, `nextest:${id}`);
|
|
1237
|
+
return { granted: held.granted.then((answer) => answer.ok), release: () => held.release() };
|
|
1238
|
+
});
|
|
1239
|
+
}
|
|
1075
1240
|
channel.end(`${granted.cpus}\n`);
|
|
1076
|
-
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1241
|
+
stepTo({ kind: 'granted', at: awakeNow(), wallAt: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1077
1242
|
void granted.lost.then((why) => {
|
|
1078
1243
|
if (state.settled || cpuReleased !== null)
|
|
1079
1244
|
return;
|
|
@@ -1093,13 +1258,19 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1093
1258
|
said = true;
|
|
1094
1259
|
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1095
1260
|
}
|
|
1096
|
-
stepTo({
|
|
1261
|
+
stepTo({
|
|
1262
|
+
kind: 'granted',
|
|
1263
|
+
at: awakeNow(),
|
|
1264
|
+
wallAt: Date.now(),
|
|
1265
|
+
testing: isTesting(),
|
|
1266
|
+
fixture: !fixtureDone,
|
|
1267
|
+
});
|
|
1097
1268
|
}
|
|
1098
1269
|
else if (byte === 114) {
|
|
1099
|
-
stepTo({ kind: 'released', at:
|
|
1270
|
+
stepTo({ kind: 'released', at: awakeNow() });
|
|
1100
1271
|
}
|
|
1101
1272
|
else {
|
|
1102
|
-
failGrant(`unexpected byte ${byte}`);
|
|
1273
|
+
failGrant(descriptorFailure(`unexpected byte ${byte}`));
|
|
1103
1274
|
return;
|
|
1104
1275
|
}
|
|
1105
1276
|
}
|
|
@@ -1132,16 +1303,23 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1132
1303
|
clearTimeout(boundTimer);
|
|
1133
1304
|
cancelCapture();
|
|
1134
1305
|
removeSignalHandlers();
|
|
1135
|
-
// The runner is gone
|
|
1136
|
-
// then the grant goes back to its parent or the host.
|
|
1306
|
+
// The runner is gone, and with it every test that could report on the run's progress channel. Its sub-pool closes
|
|
1307
|
+
// first, so no descendant still draws on tokens the grant returns; then the grant goes back to its parent or the host.
|
|
1308
|
+
await closeProgress();
|
|
1309
|
+
await closeCaseGate();
|
|
1137
1310
|
await releaseCpu();
|
|
1138
1311
|
const code = exitCode ?? signalToExitCode(exitSignal);
|
|
1139
1312
|
if (code !== 0 && state.expiry === null) {
|
|
1140
|
-
//
|
|
1141
|
-
// SIGKILL): the signal is named, never folded
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1313
|
+
// The executor ended it for a cause of its own, which is said; otherwise no bound of this executor ended it, so a
|
|
1314
|
+
// signal it died of came from outside (the kernel's OOM killer sends SIGKILL): the signal is named, never folded
|
|
1315
|
+
// into a bare status.
|
|
1316
|
+
const signalled = exitCode === null && exitSignal !== null;
|
|
1317
|
+
const endedBy = endedFor();
|
|
1318
|
+
const ended = endedBy !== null
|
|
1319
|
+
? `was ended by bounded-exec because ${endedBy.message} (${signalled ? `${exitSignal}, ` : ''}status ${code})`
|
|
1320
|
+
: signalled
|
|
1321
|
+
? `was killed by ${exitSignal} (status ${code})`
|
|
1322
|
+
: `exited with status ${code}`;
|
|
1145
1323
|
appendStderr(fixture !== null && !mainStarted
|
|
1146
1324
|
? `Fixture command ${ended}: ${segment ?? fixture.command}\n`
|
|
1147
1325
|
: `Command ${ended}: ${command}\n`);
|
|
@@ -1170,10 +1348,15 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1170
1348
|
appendStderr(`bounded-exec: this run's memory peak was not measured (${reported.reason}); its record names none\n`);
|
|
1171
1349
|
}
|
|
1172
1350
|
const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
|
|
1351
|
+
// A Go run sums nothing itself, and a reader that keeps part of a long output cannot count it: the sum ends the output.
|
|
1352
|
+
const goRun = progress.goSummary();
|
|
1353
|
+
if (goRun !== null) {
|
|
1354
|
+
appendStdout(goRunSummaryText(goRun, declaredGoTests(goRun.failedTests, cwd, context.root, mergeEnv(lease?.directory ?? ordinaryTemp, env))));
|
|
1355
|
+
}
|
|
1173
1356
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1174
1357
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1175
|
-
const operationalFailure =
|
|
1176
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1358
|
+
const operationalFailure = operationalCause();
|
|
1359
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, state.cpuRising, peakBytes);
|
|
1177
1360
|
const killedForMemory = killedFromOutside({
|
|
1178
1361
|
sentSignal: state.sentSignal,
|
|
1179
1362
|
bounded: state.expiry !== null,
|
|
@@ -1181,7 +1364,7 @@ async function runCommand(options, context, killer, tempVolume, inherited, captu
|
|
|
1181
1364
|
code,
|
|
1182
1365
|
});
|
|
1183
1366
|
return {
|
|
1184
|
-
success:
|
|
1367
|
+
success: operationalFailure === null && resultFailure === null && state.expiry === null && code === 0,
|
|
1185
1368
|
terminalOutput: outputChunks.join(''),
|
|
1186
1369
|
...(killedForMemory ? { killedForMemory: true } : {}),
|
|
1187
1370
|
};
|
|
@@ -1199,19 +1382,6 @@ function resolveCwd(cwd, root) {
|
|
|
1199
1382
|
}
|
|
1200
1383
|
return isAbsolute(cwd) ? cwd : join(root, cwd);
|
|
1201
1384
|
}
|
|
1202
|
-
function buildCommand(options) {
|
|
1203
|
-
const parts = [options.command];
|
|
1204
|
-
if (Array.isArray(options.args)) {
|
|
1205
|
-
parts.push(...options.args);
|
|
1206
|
-
}
|
|
1207
|
-
else if (options.args) {
|
|
1208
|
-
parts.push(options.args);
|
|
1209
|
-
}
|
|
1210
|
-
if (options.forwardAllArgs !== false && options.__unparsed__?.length) {
|
|
1211
|
-
parts.push(...options.__unparsed__);
|
|
1212
|
-
}
|
|
1213
|
-
return parts.join(' ');
|
|
1214
|
-
}
|
|
1215
1385
|
/** Owned scratch is the command's TMPDIR unless the target names one itself. */
|
|
1216
1386
|
function mergeEnv(directory, env) {
|
|
1217
1387
|
const inherited = withoutPoolEnv(process.env);
|