@smoothbricks/nx-plugin 0.6.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +417 -171
- package/dist/.build-inputs.json +39 -28
- package/dist/bin/smoo-nx-go-test-build.d.ts +3 -0
- package/dist/bin/smoo-nx-go-test-build.d.ts.map +1 -0
- package/dist/bin/smoo-nx-go-test-build.js +79 -0
- package/dist/bin/smoo-nx-go-test-run.d.ts +3 -0
- package/dist/bin/smoo-nx-go-test-run.d.ts.map +1 -0
- package/dist/bin/smoo-nx-go-test-run.js +129 -0
- package/dist/bin/smoo-nx-publish.d.ts +3 -0
- package/dist/bin/smoo-nx-publish.d.ts.map +1 -0
- package/dist/bin/smoo-nx-publish.js +3 -0
- package/dist/bounded-test-policy.d.ts +1 -1
- package/dist/bounded-test-policy.d.ts.map +1 -1
- package/dist/bounded-test-policy.js +9 -8
- package/dist/cargo-retention-hook.d.ts +6 -2
- package/dist/cargo-retention-hook.d.ts.map +1 -1
- package/dist/cargo-retention-hook.js +13 -7
- package/dist/cargo-workspace.d.ts +10 -20
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +12 -22
- package/dist/cross-check-policy.d.ts +2 -2
- package/dist/cross-check-policy.js +2 -2
- package/dist/deploy-policy.d.ts +6 -0
- package/dist/deploy-policy.d.ts.map +1 -1
- package/dist/deploy-policy.js +11 -0
- package/dist/eslint-file-set.d.ts +4 -1
- package/dist/eslint-file-set.d.ts.map +1 -1
- package/dist/eslint-sources.d.ts +25 -0
- package/dist/eslint-sources.d.ts.map +1 -0
- package/dist/eslint-sources.js +99 -0
- package/dist/executors/all-verdicts/executor.d.ts +2 -1
- package/dist/executors/all-verdicts/executor.d.ts.map +1 -1
- package/dist/executors/all-verdicts/executor.js +48 -2
- package/dist/executors/all-verdicts/schema.d.ts +13 -0
- package/dist/executors/all-verdicts/schema.d.ts.map +1 -1
- package/dist/executors/all-verdicts/schema.js +7 -0
- package/dist/executors/bounded-exec/bounds.d.ts +57 -8
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +100 -11
- package/dist/executors/bounded-exec/executor.d.ts +2 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +289 -34
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +2 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +2 -2
- package/dist/executors/bounded-exec/mbx-pool.d.ts +56 -10
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +145 -11
- package/dist/executors/bounded-exec/mbx-queue.d.ts +30 -0
- package/dist/executors/bounded-exec/mbx-queue.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-queue.js +229 -0
- package/dist/executors/bounded-exec/progress.d.ts +9 -2
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +151 -25
- package/dist/executors/bounded-exec/runner-sizing.d.ts +40 -2
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +55 -4
- package/dist/executors/bounded-exec/runtime-pin.js +1 -1
- package/dist/executors/bounded-exec/test-result-writer.d.ts +30 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -0
- package/dist/executors/bounded-exec/test-result-writer.js +126 -0
- package/dist/executors/bounded-exec/test-results.d.ts +33 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -0
- package/dist/executors/bounded-exec/test-results.js +33 -0
- package/dist/executors/bounded-exec/verdict.d.ts +26 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +108 -4
- package/dist/executors/wrangler-deploy/deploy-hash.d.ts +10 -0
- package/dist/executors/wrangler-deploy/deploy-hash.d.ts.map +1 -0
- package/dist/executors/wrangler-deploy/deploy-hash.js +14 -0
- package/dist/executors/wrangler-deploy/executor.d.ts +24 -0
- package/dist/executors/wrangler-deploy/executor.d.ts.map +1 -0
- package/dist/executors/wrangler-deploy/executor.js +43 -0
- package/dist/executors/wrangler-deploy/hasher.d.ts +27 -0
- package/dist/executors/wrangler-deploy/hasher.d.ts.map +1 -0
- package/dist/executors/wrangler-deploy/hasher.js +90 -0
- package/dist/executors/wrangler-deploy/schema.d.ts +5 -0
- package/dist/executors/wrangler-deploy/schema.d.ts.map +1 -0
- package/dist/go-tests.d.ts +30 -0
- package/dist/go-tests.d.ts.map +1 -0
- package/dist/go-tests.js +43 -0
- package/dist/index.d.ts +12 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +338 -295
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +5 -0
- package/dist/nx-publish.d.ts +24 -0
- package/dist/nx-publish.d.ts.map +1 -0
- package/dist/nx-publish.js +150 -0
- package/dist/process-group.js +2 -2
- package/dist/runtime-input-policy.d.ts +17 -3
- package/dist/runtime-input-policy.d.ts.map +1 -1
- package/dist/runtime-input-policy.js +4 -4
- package/dist/test-timings.d.ts +23 -4
- package/dist/test-timings.d.ts.map +1 -1
- package/dist/test-timings.js +72 -12
- package/dist/tool-bins.d.ts +6 -8
- package/dist/tool-bins.d.ts.map +1 -1
- package/dist/tool-bins.js +7 -5
- package/dist/version-actions.cjs +10 -11
- package/dist/version-actions.d.cts.map +1 -1
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +17 -22
- package/dist/workspace-index.d.ts +7 -0
- package/dist/workspace-index.d.ts.map +1 -1
- package/dist/workspace-index.js +11 -0
- package/dist/workspace-manifest.js +3 -3
- package/executors.json +7 -6
- package/managed/raw/patches/nx@23.2.1.patch +118 -24
- package/managed/raw/patches/ttsc@0.34.1.patch +139 -0
- package/managed/raw/tooling/devenv +1 -2
- package/managed/raw/tooling/direnv/developer-links.ts +124 -51
- package/managed/raw/tooling/direnv/devenv.smoo.nix +34 -15
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +20 -10
- package/managed/raw/tooling/direnv/installed-bun-links.ts +7 -6
- package/managed/raw/tooling/direnv/setup-environment.ts +49 -24
- package/managed/raw/tooling/direnv/ttsc-env.sh +8 -6
- package/managed/raw/tooling/git-hooks/pinned-revisions-served.ts +252 -0
- package/managed/raw/tooling/git-hooks/pre-push.sh +34 -32
- package/managed/templates/github/actions/cache-node-modules/action.yml +1 -1
- package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +12 -11
- package/package.json +277 -11
- package/runtime/nx-publish/.build-inputs.json +11 -0
- package/runtime/nx-publish/cache_adapter.js +45 -0
- package/runtime/nx-publish/nx_publish.js +60 -0
- package/runtime-input.sh +62 -26
- package/src/bin/smoo-nx-go-test-build.ts +83 -0
- package/src/bin/smoo-nx-go-test-run.ts +140 -0
- package/src/bin/smoo-nx-publish.ts +4 -0
- package/src/bounded-test-policy.ts +9 -8
- package/src/cargo-retention-hook.ts +16 -10
- package/src/cargo-workspace.ts +12 -24
- package/src/cross-check-policy.ts +2 -2
- package/src/deploy-policy.ts +11 -0
- package/src/eslint-file-set.ts +4 -1
- package/src/eslint-sources.ts +134 -0
- package/src/executors/all-verdicts/executor.ts +58 -3
- package/src/executors/all-verdicts/schema.json +4 -0
- package/src/executors/all-verdicts/schema.ts +14 -0
- package/src/executors/bounded-exec/bounds.ts +132 -15
- package/src/executors/bounded-exec/executor.ts +329 -34
- package/src/executors/bounded-exec/hang-diagnostics.ts +3 -2
- package/src/executors/bounded-exec/mbx-pool.ts +158 -17
- package/src/executors/bounded-exec/mbx-queue.ts +170 -0
- package/src/executors/bounded-exec/progress.ts +171 -18
- package/src/executors/bounded-exec/runner-sizing.ts +73 -7
- package/src/executors/bounded-exec/runtime-pin.ts +1 -1
- package/src/executors/bounded-exec/test-result-writer.ts +152 -0
- package/src/executors/bounded-exec/test-results.ts +39 -0
- package/src/executors/bounded-exec/verdict.ts +115 -5
- package/src/executors/wrangler-deploy/deploy-hash.ts +15 -0
- package/src/executors/wrangler-deploy/executor.ts +63 -0
- package/src/executors/wrangler-deploy/hasher.ts +111 -0
- package/src/executors/wrangler-deploy/schema.json +16 -0
- package/src/executors/wrangler-deploy/schema.ts +4 -0
- package/src/go-tests.ts +67 -0
- package/src/index.ts +368 -323
- package/src/managed-files/files.ts +5 -0
- package/src/nx-publish.ts +157 -0
- package/src/process-group.ts +2 -2
- package/src/runtime-input-policy.ts +4 -4
- package/src/test-timings.ts +72 -14
- package/src/tool-bins.ts +13 -9
- package/src/version-actions.cts +10 -11
- package/src/workspace-config-policy.ts +20 -29
- package/src/workspace-index.ts +12 -0
- package/src/workspace-manifest.ts +3 -3
- package/test-timings.json +131 -125
- package/dist/executors/clean-outputs/executor.d.ts +0 -26
- package/dist/executors/clean-outputs/executor.d.ts.map +0 -1
- package/dist/executors/clean-outputs/executor.js +0 -83
- package/dist/executors/clean-outputs/schema.d.ts +0 -4
- package/dist/executors/clean-outputs/schema.d.ts.map +0 -1
- package/src/executors/clean-outputs/executor.ts +0 -131
- package/src/executors/clean-outputs/schema.json +0 -15
- package/src/executors/clean-outputs/schema.ts +0 -3
- /package/dist/executors/{clean-outputs → wrangler-deploy}/schema.js +0 -0
|
@@ -7,19 +7,21 @@ import { createInterface } from 'node:readline';
|
|
|
7
7
|
import { Duplex, Readable } from 'node:stream';
|
|
8
8
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
9
9
|
import { DEADLINE_PATIENCE, describeMembersLeft, killProcessGroup, processGroupHasMembers, signalProcessGroup, } from '../../process-group.js';
|
|
10
|
-
import { earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
10
|
+
import { admissionWaitMs, earliestBound, NO_ADMISSION_WAIT, stepAdmissionWait, stepBounds, UNARMED, } from './bounds.js';
|
|
11
11
|
import { CPU_POOL_ENV, createCpuPool, readInheritedCpuPool, takeParentCpuTokens, } from './cpu-pool.js';
|
|
12
12
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
13
13
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
14
|
-
import { mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
|
|
14
|
+
import { MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, nxTaskCores, runsPrebuiltNextest, withMbxLeaf, } from './mbx-pool.js';
|
|
15
|
+
import { MBX_QUEUE_POLL_MS, snapshotMbxQueue } from './mbx-queue.js';
|
|
15
16
|
import { createNoSpaceWatch } from './no-space.js';
|
|
16
17
|
import { COWSHED_EXECUTABLE_ENV, cowshedSubtreeOwner, HANDLE_FD, readAdmission, SUPERVISOR_SOCKET_ENV, } from './owned-subtree.js';
|
|
17
|
-
import { createRunProgress, withProgressReport } from './progress.js';
|
|
18
|
+
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
18
19
|
import { createHostCommands, describeCoverageGap, describeRamTempError, describeSignalled, LEFTOVER_GRACE_MS, PRODUCTION, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
|
|
19
20
|
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
20
|
-
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
|
|
21
|
+
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
|
|
21
22
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
22
|
-
import {
|
|
23
|
+
import { createTestResultWriter } from './test-result-writer.js';
|
|
24
|
+
import { BUN_TEST, clearTaskDirectory, describeAdmissionQueue, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
23
25
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
24
26
|
const REAP_AFTER_FORCE_KILL_MS = 2_000;
|
|
25
27
|
const EXIT_CODE_BY_SIGNAL = {
|
|
@@ -142,7 +144,7 @@ export function recordContext(context) {
|
|
|
142
144
|
* connection closes while the command runs (the lending parent's exit) ends its accounting, never the command: the
|
|
143
145
|
* command runs on over budget to its own exit, and the run says so.
|
|
144
146
|
*/
|
|
145
|
-
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null) {
|
|
147
|
+
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null, queueHost) {
|
|
146
148
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
147
149
|
const directory = context.record?.directory ?? null;
|
|
148
150
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
@@ -157,6 +159,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
157
159
|
? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
|
|
158
160
|
: null;
|
|
159
161
|
const outputChunks = [];
|
|
162
|
+
let resultWriter = null;
|
|
163
|
+
let resultFailure = null;
|
|
160
164
|
const state = {
|
|
161
165
|
settled: false,
|
|
162
166
|
expiry: null,
|
|
@@ -349,7 +353,11 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
349
353
|
// (`run-cpu-pool.ts`), so a task takes its tokens from a parent's pool; a command that inherits none runs exactly as
|
|
350
354
|
// asked and nothing is said.
|
|
351
355
|
const cores = availableParallelism();
|
|
352
|
-
|
|
356
|
+
// A prebuilt nextest leaf under mbx asks for its task's share of the host (`nxTaskCores`), not every core: it knows
|
|
357
|
+
// no count of its tests. A declared or named count still wins.
|
|
358
|
+
const mbx = mbxPool(process.env);
|
|
359
|
+
const leafCores = mbx !== null && runsPrebuiltNextest(requested) ? nxTaskCores(cores, { ...process.env, ...options.env }) : cores;
|
|
360
|
+
const demand = runnerDemand(requested, options.parallelism, leafCores, process.env, options.env);
|
|
353
361
|
const asked = Date.now();
|
|
354
362
|
// A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
|
|
355
363
|
// instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
@@ -372,16 +380,18 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
372
380
|
const waitedMs = Date.now() - asked;
|
|
373
381
|
let cpu;
|
|
374
382
|
let sized = { command: requested, env: {} };
|
|
375
|
-
// With mbx the leaf's admission is mbx's one pool, not a parent's tokens
|
|
376
|
-
//
|
|
377
|
-
// job"). No grant and no sub-pool beside it.
|
|
378
|
-
const
|
|
379
|
-
const leaf = mbx === null ? null : mbxLeaf(mbx, demand.kind, demand.want);
|
|
383
|
+
// With mbx the leaf's admission is mbx's one pool, not a parent's tokens. A leaf mbx does not admit itself asks for
|
|
384
|
+
// its runner's count, a maximum, and its runner runs on the cores it is granted, which only the command learns
|
|
385
|
+
// (specs/cowshed/05_gateway.md, "Parallelism inside a job"). No grant and no sub-pool beside it.
|
|
386
|
+
const leaf = mbx === null ? null : mbxLeaf(mbx, demand, requested);
|
|
380
387
|
if (grant !== null) {
|
|
381
388
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
382
389
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
383
390
|
appendStderr(`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`);
|
|
384
391
|
}
|
|
392
|
+
else if (leaf?.kind === 'reserve') {
|
|
393
|
+
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
394
|
+
}
|
|
385
395
|
else if (leaf !== null) {
|
|
386
396
|
sized = sizeRunner(demand, requested, demand.want);
|
|
387
397
|
}
|
|
@@ -404,18 +414,65 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
404
414
|
})();
|
|
405
415
|
return cpuReleased;
|
|
406
416
|
};
|
|
417
|
+
let mbxAdmitted = false;
|
|
418
|
+
let admissionWait = NO_ADMISSION_WAIT;
|
|
419
|
+
let lastQueueRequests = [];
|
|
420
|
+
/**
|
|
421
|
+
* Why the last admission snapshot failed, while the one after it has not succeeded; null otherwise. A snapshot only
|
|
422
|
+
* decides whether waiting on mbx admission is credited against the idle bound, so a failed one withholds that credit
|
|
423
|
+
* and ends nothing: the bounds decide, and an idle expiry names it.
|
|
424
|
+
*/
|
|
425
|
+
let unconfirmedAdmission = null;
|
|
426
|
+
/** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
|
|
427
|
+
let confirmedAdmissionAt = null;
|
|
428
|
+
const queueWait = () => mbxAdmitted || lastQueueRequests.length === 0
|
|
429
|
+
? null
|
|
430
|
+
: {
|
|
431
|
+
waitedMs: admissionWaitMs(admissionWait, performance.now()),
|
|
432
|
+
requests: lastQueueRequests,
|
|
433
|
+
};
|
|
434
|
+
const queueAbort = new AbortController();
|
|
435
|
+
let queueTimer;
|
|
436
|
+
let queueSnapshot = null;
|
|
437
|
+
const stopQueueObservation = () => {
|
|
438
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
|
|
439
|
+
clearInterval(queueTimer);
|
|
440
|
+
queueAbort.abort();
|
|
441
|
+
};
|
|
407
442
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
408
|
-
//
|
|
409
|
-
// fired, so it is never judged as a bound.
|
|
443
|
+
// Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
|
|
410
444
|
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits) => {
|
|
411
445
|
if (context.record === undefined) {
|
|
412
446
|
return;
|
|
413
447
|
}
|
|
414
448
|
const { task, directory: recordDirectory } = context.record;
|
|
415
|
-
const report = await readJunitReport(recordDirectory)
|
|
416
|
-
|
|
449
|
+
const report = await readJunitReport(recordDirectory, requested.search(BUN_TEST) !== -1 && resultWriter !== null
|
|
450
|
+
? (results) => {
|
|
451
|
+
const error = resultWriter?.reconcile(results) ?? null;
|
|
452
|
+
if (error !== null) {
|
|
453
|
+
resultFailure ??= error;
|
|
454
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
: undefined).finally(async () => {
|
|
458
|
+
const error = (await resultWriter?.close()) ?? null;
|
|
459
|
+
if (error !== null) {
|
|
460
|
+
resultFailure ??= error;
|
|
461
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
462
|
+
}
|
|
463
|
+
});
|
|
464
|
+
const admissionQueue = queueWait();
|
|
465
|
+
const verdict = operational || resultFailure !== null
|
|
417
466
|
? judgeOperationalFailure(exitCode, report)
|
|
418
|
-
: judgeRun({
|
|
467
|
+
: judgeRun({
|
|
468
|
+
exitCode,
|
|
469
|
+
elapsedMs,
|
|
470
|
+
expiry,
|
|
471
|
+
report,
|
|
472
|
+
noSpace,
|
|
473
|
+
lockWaits,
|
|
474
|
+
admissionQueue,
|
|
475
|
+
});
|
|
419
476
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
420
477
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
421
478
|
}
|
|
@@ -448,9 +505,10 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
448
505
|
}
|
|
449
506
|
pool = created.pool;
|
|
450
507
|
}
|
|
508
|
+
const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
|
|
451
509
|
const command = directory === null
|
|
452
|
-
? withProgressReport(
|
|
453
|
-
: await withJunitReport(withProgressReport(
|
|
510
|
+
? withProgressReport(native.command)
|
|
511
|
+
: await withJunitReport(withProgressReport(native.command), directory);
|
|
454
512
|
// A runner bounded-exec cannot see in the command (a script that spawns `bun test`) writes its JUnit report to this
|
|
455
513
|
// path when it is set. The grant's sizing outranks the target's own environment: the runner may use what it was
|
|
456
514
|
// granted, not what it was configured with. The sub-pool's environment comes last: a descendant reads only the pool
|
|
@@ -459,7 +517,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
459
517
|
...(directory === null ? {} : { [JUNIT_REPORT_ENV]: join(directory, JUNIT_FILE) }),
|
|
460
518
|
...targetEnv(options.env, appendStderr),
|
|
461
519
|
...sized.env,
|
|
520
|
+
...native.env,
|
|
462
521
|
...(pool?.environment ?? {}),
|
|
522
|
+
...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
|
|
463
523
|
};
|
|
464
524
|
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
465
525
|
// of the command: the progress bound runs during it and its own ceiling does not.
|
|
@@ -475,26 +535,37 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
475
535
|
// With a supervisor, `cowshed subtree-root` asks it for a handle on itself, writes the answer to the handle
|
|
476
536
|
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
477
537
|
const unadmitted = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
|
|
478
|
-
const shellCommand =
|
|
538
|
+
const shellCommand = withMbxLeaf(unadmitted, leaf);
|
|
539
|
+
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
540
|
+
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
541
|
+
// fd 4 is unused without a subtree admission, so the grant takes fd 5 either way.
|
|
542
|
+
const grantSlots = leaf !== null ? ['ignore', 'pipe'] : [];
|
|
543
|
+
const rootGrantSlot = leaf !== null ? ['pipe'] : [];
|
|
479
544
|
const root = owner?.rootArgv(shellCommand);
|
|
480
545
|
let child;
|
|
546
|
+
const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
|
|
547
|
+
if (results !== null && !results.ok)
|
|
548
|
+
return refuse(`bounded-exec: ${results.error}\n`);
|
|
549
|
+
resultWriter = results?.ok ? results.writer : null;
|
|
481
550
|
try {
|
|
482
551
|
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
483
|
-
//
|
|
552
|
+
// cowshed closes fd 4 before exec; fd 5 carries mbx's admission/release stream, including Cargo descendants.
|
|
484
553
|
child =
|
|
485
554
|
root === undefined
|
|
486
555
|
? spawn(shellCommand, [], {
|
|
487
556
|
cwd,
|
|
488
557
|
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
489
558
|
shell: true,
|
|
490
|
-
stdio: hooked === null
|
|
559
|
+
stdio: hooked === null && leaf === null
|
|
560
|
+
? 'pipe'
|
|
561
|
+
: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
|
|
491
562
|
detached: process.platform !== 'win32',
|
|
492
563
|
windowsHide: true,
|
|
493
564
|
})
|
|
494
565
|
: spawn(root.file, root.args, {
|
|
495
566
|
cwd,
|
|
496
567
|
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
497
|
-
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
|
|
568
|
+
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot],
|
|
498
569
|
detached: process.platform !== 'win32',
|
|
499
570
|
windowsHide: true,
|
|
500
571
|
});
|
|
@@ -518,7 +589,17 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
518
589
|
});
|
|
519
590
|
}
|
|
520
591
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
521
|
-
const progress = createRunProgress(command
|
|
592
|
+
const progress = createRunProgress(command, resultWriter === null
|
|
593
|
+
? undefined
|
|
594
|
+
: (result) => {
|
|
595
|
+
const error = resultWriter?.append(result) ?? null;
|
|
596
|
+
if (error !== null && resultFailure === null) {
|
|
597
|
+
resultFailure = error;
|
|
598
|
+
lifecycleFailure ??= error;
|
|
599
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
600
|
+
termination = terminate();
|
|
601
|
+
}
|
|
602
|
+
});
|
|
522
603
|
// A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
|
|
523
604
|
// since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
|
|
524
605
|
const noSpace = createNoSpaceWatch();
|
|
@@ -581,6 +662,46 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
581
662
|
: kind === 'fixture'
|
|
582
663
|
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
583
664
|
: { kind, limitMs: timeoutMs ?? 0 };
|
|
665
|
+
let checkingIdle = false;
|
|
666
|
+
const armBoundTimer = () => {
|
|
667
|
+
clearTimeout(boundTimer);
|
|
668
|
+
if (state.settled || state.expiry !== null)
|
|
669
|
+
return;
|
|
670
|
+
const next = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
|
|
671
|
+
if (next !== null) {
|
|
672
|
+
boundTimer = setTimeout(() => {
|
|
673
|
+
void confirmExpiry(next.kind, next.at);
|
|
674
|
+
}, Math.max(0, next.at - Date.now()));
|
|
675
|
+
}
|
|
676
|
+
};
|
|
677
|
+
const confirmExpiry = async (kind, at) => {
|
|
678
|
+
if (state.settled || state.expiry !== null)
|
|
679
|
+
return;
|
|
680
|
+
const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
|
|
681
|
+
if (scheduled?.kind !== kind || scheduled.at !== at)
|
|
682
|
+
return;
|
|
683
|
+
if (kind === 'idle' && leaf !== null) {
|
|
684
|
+
if (checkingIdle)
|
|
685
|
+
return;
|
|
686
|
+
checkingIdle = true;
|
|
687
|
+
armBoundTimer();
|
|
688
|
+
try {
|
|
689
|
+
// Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
|
|
690
|
+
// ceilings remain armed while this observation runs.
|
|
691
|
+
await refreshQueue(true);
|
|
692
|
+
}
|
|
693
|
+
finally {
|
|
694
|
+
checkingIdle = false;
|
|
695
|
+
}
|
|
696
|
+
}
|
|
697
|
+
if (state.settled || state.expiry !== null)
|
|
698
|
+
return;
|
|
699
|
+
const next = earliestBound(bounds);
|
|
700
|
+
if (next !== null && next.at <= Date.now())
|
|
701
|
+
expire(expiredAs(next.kind));
|
|
702
|
+
else
|
|
703
|
+
armBoundTimer();
|
|
704
|
+
};
|
|
584
705
|
const stepTo = (event) => {
|
|
585
706
|
if (state.settled || state.expiry !== null)
|
|
586
707
|
return;
|
|
@@ -596,10 +717,71 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
596
717
|
if (bounds.idle !== null)
|
|
597
718
|
scheduleCapture(bounds.idle);
|
|
598
719
|
}
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
720
|
+
armBoundTimer();
|
|
721
|
+
};
|
|
722
|
+
let lastQueueDiagnostic = '';
|
|
723
|
+
let lastQueueDiagnosticAt = null;
|
|
724
|
+
let lastSnapshotDiagnostics = '';
|
|
725
|
+
const unconfirmAdmission = (detail) => {
|
|
726
|
+
if (state.settled || state.expiry !== null || queueAbort.signal.aborted)
|
|
727
|
+
return;
|
|
728
|
+
if (unconfirmedAdmission !== detail) {
|
|
729
|
+
tell(`bounded-exec: mbx admission snapshot failed: ${detail}; admission is unconfirmed, so this wait earns no idle credit until a snapshot succeeds\n`);
|
|
730
|
+
}
|
|
731
|
+
unconfirmedAdmission = detail;
|
|
732
|
+
lastQueueRequests = [];
|
|
733
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: false });
|
|
734
|
+
// The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
|
|
735
|
+
stepTo({ kind: 'admission-queue', at: confirmedAdmissionAt ?? Date.now(), waiting: false, testing: isTesting() });
|
|
736
|
+
};
|
|
737
|
+
const refreshQueue = async (fresh) => {
|
|
738
|
+
if (fresh && queueSnapshot !== null)
|
|
739
|
+
await queueSnapshot;
|
|
740
|
+
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined)
|
|
741
|
+
return;
|
|
742
|
+
const pid = child.pid;
|
|
743
|
+
queueSnapshot ??= (async () => {
|
|
744
|
+
const snapshot = await snapshotMbxQueue(pid, mergeEnv(lease?.directory ?? ordinaryTemp, env), queueAbort.signal, queueHost);
|
|
745
|
+
if (state.settled || state.expiry !== null || queueAbort.signal.aborted)
|
|
746
|
+
return;
|
|
747
|
+
if (!snapshot.ok) {
|
|
748
|
+
unconfirmAdmission(snapshot.detail);
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
|
|
752
|
+
if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics)
|
|
753
|
+
tell(`bounded-exec: ${diagnostics}\n`);
|
|
754
|
+
lastSnapshotDiagnostics = diagnostics;
|
|
755
|
+
const waiting = snapshot.requests.length > 0;
|
|
756
|
+
const observedAt = Date.now();
|
|
757
|
+
const observedWaitAt = performance.now();
|
|
758
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
|
|
759
|
+
unconfirmedAdmission = null;
|
|
760
|
+
confirmedAdmissionAt = observedAt;
|
|
761
|
+
if (waiting) {
|
|
762
|
+
lastQueueRequests = snapshot.requests;
|
|
763
|
+
const positions = JSON.stringify(snapshot.requests);
|
|
764
|
+
if (positions !== lastQueueDiagnostic ||
|
|
765
|
+
lastQueueDiagnosticAt === null ||
|
|
766
|
+
Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS) {
|
|
767
|
+
const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
|
|
768
|
+
tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
|
|
769
|
+
lastQueueDiagnostic = positions;
|
|
770
|
+
lastQueueDiagnosticAt = Date.now();
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
else {
|
|
774
|
+
lastQueueDiagnostic = '';
|
|
775
|
+
}
|
|
776
|
+
stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
|
|
777
|
+
})()
|
|
778
|
+
.catch((error) => {
|
|
779
|
+
unconfirmAdmission(error instanceof Error ? error.message : String(error));
|
|
780
|
+
})
|
|
781
|
+
.finally(() => {
|
|
782
|
+
queueSnapshot = null;
|
|
783
|
+
});
|
|
784
|
+
await queueSnapshot;
|
|
603
785
|
};
|
|
604
786
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
605
787
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
@@ -611,6 +793,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
611
793
|
if (!fixtureDone)
|
|
612
794
|
return;
|
|
613
795
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
796
|
+
const error = progress.error();
|
|
797
|
+
if (error !== null && lifecycleFailure === null) {
|
|
798
|
+
lifecycleFailure = error;
|
|
799
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
800
|
+
termination = terminate();
|
|
801
|
+
}
|
|
614
802
|
};
|
|
615
803
|
if (lifecycle !== null) {
|
|
616
804
|
const channel = child.stdio[3];
|
|
@@ -652,7 +840,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
652
840
|
}
|
|
653
841
|
child.stdout?.on('data', (chunk) => {
|
|
654
842
|
const text = chunk.toString();
|
|
655
|
-
scanProgress('stdout',
|
|
843
|
+
scanProgress('stdout', chunk);
|
|
656
844
|
const said = noSpace.scan('stdout', text);
|
|
657
845
|
if (said !== null)
|
|
658
846
|
state.noSpace = sight(said);
|
|
@@ -660,7 +848,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
660
848
|
});
|
|
661
849
|
child.stderr?.on('data', (chunk) => {
|
|
662
850
|
const text = chunk.toString();
|
|
663
|
-
scanProgress('stderr',
|
|
851
|
+
scanProgress('stderr', chunk);
|
|
664
852
|
const said = noSpace.scan('stderr', text);
|
|
665
853
|
if (said !== null)
|
|
666
854
|
state.noSpace = sight(said);
|
|
@@ -684,12 +872,21 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
684
872
|
const onProcessExit = () => {
|
|
685
873
|
void killChildTree();
|
|
686
874
|
};
|
|
875
|
+
let signalCleanup = null;
|
|
687
876
|
const onTerminationSignal = (signal) => {
|
|
688
877
|
removeSignalHandlers();
|
|
689
878
|
cancelCapture();
|
|
690
|
-
|
|
879
|
+
clearTimeout(boundTimer);
|
|
880
|
+
// Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
|
|
881
|
+
// leaves the admission unconfirmed, never a manufactured queue or admission.
|
|
882
|
+
signalCleanup = (leaf === null ? Promise.resolve() : refreshQueue(false))
|
|
883
|
+
.finally(() => {
|
|
884
|
+
stopQueueObservation();
|
|
885
|
+
return killChildTree();
|
|
886
|
+
})
|
|
691
887
|
.finally(releaseCpu)
|
|
692
888
|
.finally(() => releaseLease(true))
|
|
889
|
+
.finally(async () => recordRun(signalToExitCode(signal), state.expiry, Date.now() - startedAt, state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null, state.noSpace === null ? null : await state.noSpace, state.lockWaits))
|
|
693
890
|
.finally(() => process.kill(process.pid, signal));
|
|
694
891
|
};
|
|
695
892
|
const onSigint = () => onTerminationSignal('SIGINT');
|
|
@@ -713,10 +910,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
713
910
|
child.once('error', (error) => {
|
|
714
911
|
appendStderr(`${error.message}\n`);
|
|
715
912
|
state.settled = true;
|
|
913
|
+
stopQueueObservation();
|
|
716
914
|
resolve({ code: 1, signal: null });
|
|
717
915
|
});
|
|
718
916
|
child.once(hooked === null ? 'exit' : 'close', (code, signal) => {
|
|
719
917
|
state.settled = true;
|
|
918
|
+
stopQueueObservation();
|
|
720
919
|
resolve({ code, signal });
|
|
721
920
|
});
|
|
722
921
|
});
|
|
@@ -735,7 +934,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
735
934
|
appendStderr(bound.kind === 'idle'
|
|
736
935
|
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
737
936
|
? `still running: ${bound.tests.join(', ')}`
|
|
738
|
-
: 'the runner named no test still running'} (cwd=${cwd}): ${command}\n
|
|
937
|
+
: 'the runner named no test still running'} (cwd=${cwd}): ${command}\n${unconfirmedAdmission === null
|
|
938
|
+
? ''
|
|
939
|
+
: `The last mbx admission snapshot failed (${unconfirmedAdmission}), so the wait it could not confirm earned no idle credit\n`}`
|
|
739
940
|
: bound.kind === 'fixture'
|
|
740
941
|
? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${fixture?.command ?? command}\n`
|
|
741
942
|
: `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`);
|
|
@@ -815,7 +1016,55 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
815
1016
|
};
|
|
816
1017
|
// The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
|
|
817
1018
|
// fire on a command the fixture already handed over to.
|
|
818
|
-
stepTo({
|
|
1019
|
+
stepTo({
|
|
1020
|
+
kind: 'spawn',
|
|
1021
|
+
at: Date.now(),
|
|
1022
|
+
testing: isTesting(),
|
|
1023
|
+
fixture: fixture !== null && lifecycle !== null,
|
|
1024
|
+
queued,
|
|
1025
|
+
});
|
|
1026
|
+
// Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
|
|
1027
|
+
if (leaf !== null) {
|
|
1028
|
+
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1029
|
+
if (!(channel instanceof Readable))
|
|
1030
|
+
throw new Error('an mbx leaf requires its grant descriptor');
|
|
1031
|
+
let said = false;
|
|
1032
|
+
const failGrant = (detail) => {
|
|
1033
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled)
|
|
1034
|
+
return;
|
|
1035
|
+
lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
|
|
1036
|
+
appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
|
|
1037
|
+
termination = terminate();
|
|
1038
|
+
};
|
|
1039
|
+
channel.on('data', (chunk) => {
|
|
1040
|
+
for (const byte of chunk) {
|
|
1041
|
+
if (byte === 103) {
|
|
1042
|
+
if (!said) {
|
|
1043
|
+
mbxAdmitted = true;
|
|
1044
|
+
said = true;
|
|
1045
|
+
tell(leaf.kind === 'reserve'
|
|
1046
|
+
? `cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms asked=${leaf.demand.want} priority=${leaf.priority}\n`
|
|
1047
|
+
: `cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1048
|
+
}
|
|
1049
|
+
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1050
|
+
}
|
|
1051
|
+
else if (byte === 114) {
|
|
1052
|
+
stepTo({ kind: 'released', at: Date.now() });
|
|
1053
|
+
}
|
|
1054
|
+
else {
|
|
1055
|
+
failGrant(`unexpected byte ${byte}`);
|
|
1056
|
+
return;
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
});
|
|
1060
|
+
channel.on('error', (error) => failGrant(error.message));
|
|
1061
|
+
}
|
|
1062
|
+
if (leaf !== null) {
|
|
1063
|
+
queueTimer = setInterval(() => {
|
|
1064
|
+
void refreshQueue(false);
|
|
1065
|
+
}, MBX_QUEUE_POLL_MS);
|
|
1066
|
+
void refreshQueue(false);
|
|
1067
|
+
}
|
|
819
1068
|
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
820
1069
|
// under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
|
|
821
1070
|
// else; only a bound or a signal stops the command. Our own release after the command settled is not a loss;
|
|
@@ -830,6 +1079,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
830
1079
|
});
|
|
831
1080
|
}
|
|
832
1081
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
1082
|
+
if (signalCleanup !== null) {
|
|
1083
|
+
await signalCleanup;
|
|
1084
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
1085
|
+
}
|
|
1086
|
+
stopQueueObservation();
|
|
1087
|
+
await queueSnapshot;
|
|
833
1088
|
await termination;
|
|
834
1089
|
clearTimeout(boundTimer);
|
|
835
1090
|
cancelCapture();
|
|
@@ -865,7 +1120,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
865
1120
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
866
1121
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
867
1122
|
return {
|
|
868
|
-
success: !operationalFailure && state.expiry === null && code === 0,
|
|
1123
|
+
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
869
1124
|
terminalOutput: outputChunks.join(''),
|
|
870
1125
|
};
|
|
871
1126
|
}
|
|
@@ -49,4 +49,6 @@ export declare function captureHangDiagnostics(request: HangCaptureRequest, sour
|
|
|
49
49
|
* waits here, and the kernel lists it as a member of its own group until then, so a group left listed is expected.
|
|
50
50
|
*/
|
|
51
51
|
export declare function stopProbe(file: string, pid: number): boolean;
|
|
52
|
+
/** A probe owns its group and output pipes; its deadline never waits for an unresponsive child to reap. */
|
|
53
|
+
export declare function runProbe(file: string, args: readonly string[], timeoutMs: number, signal: AbortSignal, environment?: NodeJS.ProcessEnv): Promise<HangProbeResult>;
|
|
52
54
|
//# sourceMappingURL=hang-diagnostics.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hang-diagnostics.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/hang-diagnostics.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,QAAQ,EAKd,MAAM,gBAAgB,CAAC;AAExB;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAoCtC,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE,SAAS,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3C,0HAA0H;IAC1H,SAAS,EAAE,SAAS,QAAQ,EAAE,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,eAAe,GACvB;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAClE;IAAE,MAAM,EAAE,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvG,qGAAqG;AACrG,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC1B,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CAChH;AAUD,0FAA0F;AAC1F,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,kBAAkB,EAC3B,MAAM,GAAE,kBAAyB,GAChC,OAAO,CAAC,eAAe,CAAC,CAgR1B;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAM5D"}
|
|
1
|
+
{"version":3,"file":"hang-diagnostics.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/hang-diagnostics.ts"],"names":[],"mappings":"AAGA,OAAO,EAEL,KAAK,QAAQ,EAKd,MAAM,gBAAgB,CAAC;AAExB;;;GAGG;AACH,eAAO,MAAM,eAAe,QAAS,CAAC;AAoCtC,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,sEAAsE;IACtE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,OAAO,EAAE,SAAS,GAAG,MAAM,GAAG,YAAY,CAAC;IAC3C,0HAA0H;IAC1H,SAAS,EAAE,SAAS,QAAQ,EAAE,CAAC;IAC/B,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,MAAM,eAAe,GACvB;IAAE,MAAM,EAAE,QAAQ,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GAClE;IAAE,MAAM,EAAE,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvG,qGAAqG;AACrG,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;IAC1B,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,MAAM,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;CAChH;AAUD,0FAA0F;AAC1F,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,kBAAkB,EAC3B,MAAM,GAAE,kBAAyB,GAChC,OAAO,CAAC,eAAe,CAAC,CAgR1B;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAM5D;AAED,2GAA2G;AAC3G,wBAAgB,QAAQ,CACtB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,WAAW,EACnB,WAAW,CAAC,EAAE,MAAM,CAAC,UAAU,GAC9B,OAAO,CAAC,eAAe,CAAC,CA6D1B"}
|
|
@@ -306,14 +306,14 @@ export function stopProbe(file, pid) {
|
|
|
306
306
|
return killProcessGroupNow(pid, TEARDOWN_PATIENCE);
|
|
307
307
|
}
|
|
308
308
|
/** A probe owns its group and output pipes; its deadline never waits for an unresponsive child to reap. */
|
|
309
|
-
function runProbe(file, args, timeoutMs, signal) {
|
|
309
|
+
export function runProbe(file, args, timeoutMs, signal, environment) {
|
|
310
310
|
// Keep the package's es2022-compatible event-completion convention.
|
|
311
311
|
return new Promise((resolve) => {
|
|
312
312
|
if (signal.aborted) {
|
|
313
313
|
resolve({ status: 'timed-out', detail: 'capture deadline already reached', stdout: '', stderr: '' });
|
|
314
314
|
return;
|
|
315
315
|
}
|
|
316
|
-
const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
316
|
+
const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: environment });
|
|
317
317
|
let stdout = '';
|
|
318
318
|
let stderr = '';
|
|
319
319
|
let bytes = 0;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type Demand, type ReservedDemand } from './runner-sizing.js';
|
|
2
2
|
/**
|
|
3
3
|
* mbx's machine-wide pool as a leaf runner meets it (specs/cowshed/05_gateway.md, "Host scheduling", "Parallelism inside
|
|
4
4
|
* a job"). With the mbx capability the canonical environment names the one pool (`MBX_SCHEDULER=1` beside the shared
|
|
@@ -8,24 +8,70 @@ import type { RunnerKind } from './runner-sizing.js';
|
|
|
8
8
|
export interface MbxPool {
|
|
9
9
|
readonly priority: 'normal' | 'low';
|
|
10
10
|
}
|
|
11
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged, or
|
|
13
|
+
* where this run is already inside an mbx reservation: `mbx reserve` hands its command `MBX_SCHED_DISABLE=1`, and nested
|
|
14
|
+
* non-Cargo work shares the enclosing grant instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism
|
|
15
|
+
* inside a job"). A second reservation would wait in the pool while its parent holds a grant, so under a full pool the
|
|
16
|
+
* nested leaf queued behind the very work it belongs to.
|
|
17
|
+
*/
|
|
12
18
|
export declare function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | null;
|
|
13
19
|
/**
|
|
14
|
-
* How a leaf enters the pool once
|
|
15
|
-
* itself, so wrapping it would charge it twice
|
|
16
|
-
* `mbx reserve
|
|
20
|
+
* How a leaf enters the pool once. A Cargo leaf that builds runs through mbx, which admits its compilers and supported
|
|
21
|
+
* test work itself, so wrapping it would charge it twice. Any other leaf reserves for its whole run through
|
|
22
|
+
* `mbx reserve --cpus <its runner's count>`, a maximum: mbx grants as many of those cores as are free once one is, and
|
|
23
|
+
* the runner runs on the count it is granted (`sizeRunnerToGrant`). That includes a nextest run of test binaries
|
|
24
|
+
* already built (the inferred extracted-archive run, or one from `--archive-file`): it compiles nothing, and beneath a
|
|
25
|
+
* reservation mbx schedules none of its test processes, so they run inside the one grant. Admitted one by one under
|
|
26
|
+
* `MBX_SCHEDULER_TESTS=1`, each queued for a ticket of its own: in a downstream gate on 2026-10-30, codec tests of at
|
|
27
|
+
* most 0.25 s took 73.5 s apiece, and their 121 tests run in 1.18 s unqueued (specs/cowshed/05_gateway.md,
|
|
28
|
+
* "Parallelism inside a job").
|
|
17
29
|
*/
|
|
18
30
|
export type MbxLeaf = {
|
|
19
31
|
readonly kind: 'cargo';
|
|
20
32
|
} | {
|
|
21
33
|
readonly kind: 'reserve';
|
|
22
|
-
readonly cpus: number;
|
|
23
34
|
readonly priority: 'normal' | 'low';
|
|
35
|
+
readonly demand: ReservedDemand;
|
|
24
36
|
};
|
|
25
|
-
|
|
37
|
+
/** Whether `command` runs nextest over test binaries already built, which compiles nothing. */
|
|
38
|
+
export declare function runsPrebuiltNextest(command: string): boolean;
|
|
39
|
+
/**
|
|
40
|
+
* A Vitest or Go test leaf enters no pool, `null` (owner's ruling, 2026-10-31): it holds nothing and runs its own
|
|
41
|
+
* default workers, uncapped. Reserved like any other leaf, it asked for that default, which sizes to the host: Vitest one
|
|
42
|
+
* worker per core but one, Go one package per core. So each held 17 or 18 of an 18-core pool, and mbx's ordered admission
|
|
43
|
+
* kept every reservation queued behind it waiting while the pool drained. On 2026-10-31 a downstream gate ran 4.9 of 18
|
|
44
|
+
* cores busy while 25 of its tasks queued a minute or more, and a gate's Go test leaf waited 401 s for all 18 cores.
|
|
45
|
+
*/
|
|
46
|
+
export declare function mbxLeaf(pool: MbxPool, demand: Demand, command: string): MbxLeaf | null;
|
|
47
|
+
/**
|
|
48
|
+
* The cores one task of an Nx run counts as its own: the host's cores divided among the tasks Nx runs at once, at
|
|
49
|
+
* least one. Nx reads that count from `NX_PARALLEL`, a number or a share of the cores (`100%`), else runs three.
|
|
50
|
+
*
|
|
51
|
+
* A prebuilt nextest leaf asks for this share rather than its runner's own count, every core by default, because it
|
|
52
|
+
* does not know how many tests it runs: mbx grants a reservation as many of its cores as are free, so a leaf of a few
|
|
53
|
+
* long tests asking for every core would hold the idle rest for its whole run (specs/cowshed/05_gateway.md, "Open
|
|
54
|
+
* questions"). Before mbx granted reservations their free cores, every cargo-test leaf of a gate reserved the whole
|
|
55
|
+
* pool and they ran one at a time: on 2026-10-30 a gate's 18-core host ran 4.2 cores busy, its leaves each holding
|
|
56
|
+
* `mbx reserve --cpus 18`.
|
|
57
|
+
*/
|
|
58
|
+
export declare function nxTaskCores(cores: number, environment: Readonly<NodeJS.ProcessEnv>): number;
|
|
59
|
+
/**
|
|
60
|
+
* Only a known scheduled execution may stop its idle clock at zero holds. Plain Cargo tests can run doctests;
|
|
61
|
+
* archive-file targets, direct binaries and other runners are not proven scheduled at spawn (11_shell.md,
|
|
62
|
+
* "Cargo test-runner boundaries"). The inferred in-place nextest command is a supported Cargo invocation.
|
|
63
|
+
*/
|
|
64
|
+
export declare function mbxIdleGated(leaf: MbxLeaf | null, command: string, environment: Readonly<NodeJS.ProcessEnv>): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* mbx writes one byte per admission (g) and permit drop (r). Cargo descendants inherit this descriptor through
|
|
67
|
+
* MBX_GRANT_FD; reserve names it with --grant-fd and closes it on exec of the external command.
|
|
68
|
+
*/
|
|
69
|
+
export declare const MBX_GRANT_FD = 5;
|
|
26
70
|
/**
|
|
27
|
-
* `command` as the shell runs it under its leaf's admission
|
|
28
|
-
*
|
|
71
|
+
* `command` as the shell runs it under its leaf's admission, or as it is for a leaf in no pool (`null`). A reserved leaf
|
|
72
|
+
* asks for its runner's count, and its reservation first sizes the runner to what mbx granted (`grantedSizing`) and says
|
|
73
|
+
* so on stderr. `mbx` is the canonical environment's, whose realpath cowshed's admission proved to be the host's pinned
|
|
74
|
+
* mbx; it writes the grant on this descriptor itself.
|
|
29
75
|
*/
|
|
30
|
-
export declare function withMbxLeaf(command: string, leaf: MbxLeaf): string;
|
|
76
|
+
export declare function withMbxLeaf(command: string, leaf: MbxLeaf | null): string;
|
|
31
77
|
//# sourceMappingURL=mbx-pool.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mbx-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/mbx-pool.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"mbx-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/mbx-pool.ts"],"names":[],"mappings":"AACA,OAAO,EAAkB,KAAK,MAAM,EAAiC,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGrH;;;;;GAKG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,CAAC;CACrC;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,GAAG,IAAI,CAKhF;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,OAAO,GACf;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAC1B;IACE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,QAAQ,GAAG,KAAK,CAAC;IACpC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;CACjC,CAAC;AAKN,+FAA+F;AAC/F,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAE5D;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAoBtF;AAKD;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,MAAM,CAM3F;AAQD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,OAAO,CAkDrH;AAED;;;GAGG;AACH,eAAO,MAAM,YAAY,IAAI,CAAC;AAE9B;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,IAAI,GAAG,MAAM,CAOzE"}
|