@smoothbricks/nx-plugin 0.6.2 → 0.6.3-0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +281 -73
- package/dist/.build-inputs.json +20 -14
- package/dist/bin/smoo-nx-publish.d.ts +3 -0
- package/dist/bin/smoo-nx-publish.d.ts.map +1 -0
- package/dist/bin/smoo-nx-publish.js +3 -0
- package/dist/cargo-retention-hook.d.ts +6 -2
- package/dist/cargo-retention-hook.d.ts.map +1 -1
- package/dist/cargo-retention-hook.js +13 -7
- package/dist/eslint-file-set.d.ts +4 -1
- package/dist/eslint-file-set.d.ts.map +1 -1
- package/dist/eslint-sources.d.ts +25 -0
- package/dist/eslint-sources.d.ts.map +1 -0
- package/dist/eslint-sources.js +99 -0
- package/dist/executors/all-verdicts/executor.d.ts +2 -1
- package/dist/executors/all-verdicts/executor.d.ts.map +1 -1
- package/dist/executors/all-verdicts/executor.js +48 -2
- package/dist/executors/all-verdicts/schema.d.ts +13 -0
- package/dist/executors/all-verdicts/schema.d.ts.map +1 -1
- package/dist/executors/all-verdicts/schema.js +7 -0
- package/dist/executors/bounded-exec/bounds.d.ts +57 -8
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +100 -11
- package/dist/executors/bounded-exec/executor.d.ts +2 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +270 -27
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +2 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +2 -2
- package/dist/executors/bounded-exec/mbx-pool.d.ts +20 -3
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +84 -4
- package/dist/executors/bounded-exec/mbx-queue.d.ts +30 -0
- package/dist/executors/bounded-exec/mbx-queue.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-queue.js +229 -0
- package/dist/executors/bounded-exec/progress.d.ts +9 -2
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +148 -22
- package/dist/executors/bounded-exec/test-result-writer.d.ts +30 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -0
- package/dist/executors/bounded-exec/test-result-writer.js +126 -0
- package/dist/executors/bounded-exec/test-results.d.ts +33 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -0
- package/dist/executors/bounded-exec/test-results.js +33 -0
- package/dist/executors/bounded-exec/verdict.d.ts +20 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +102 -4
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +67 -65
- package/dist/nx-publish.d.ts +24 -0
- package/dist/nx-publish.d.ts.map +1 -0
- package/dist/nx-publish.js +150 -0
- package/dist/runtime-input-policy.d.ts +14 -0
- package/dist/runtime-input-policy.d.ts.map +1 -1
- package/dist/runtime-input-policy.js +1 -1
- package/dist/tool-bins.d.ts +6 -8
- package/dist/tool-bins.d.ts.map +1 -1
- package/dist/tool-bins.js +7 -5
- package/dist/workspace-index.d.ts +7 -0
- package/dist/workspace-index.d.ts.map +1 -1
- package/dist/workspace-index.js +11 -0
- package/executors.json +1 -1
- package/managed/raw/patches/ttsc@0.34.1.patch +75 -0
- package/managed/raw/tooling/devenv +1 -2
- package/managed/raw/tooling/direnv/developer-links.ts +3 -0
- package/managed/raw/tooling/direnv/devenv.smoo.nix +34 -15
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +20 -10
- package/managed/raw/tooling/direnv/setup-environment.ts +35 -20
- package/managed/raw/tooling/direnv/ttsc-env.sh +8 -6
- package/managed/templates/github/actions/cache-node-modules/action.yml +1 -1
- package/managed/templates/github/actions/cache-ttsc-plugins/action.yml +12 -11
- package/package.json +223 -7
- package/runtime/nx-publish/cache_adapter.js +45 -0
- package/runtime/nx-publish/nx_publish.js +60 -0
- package/runtime-input.sh +12 -3
- package/src/bin/smoo-nx-publish.ts +4 -0
- package/src/cargo-retention-hook.ts +16 -10
- package/src/eslint-file-set.ts +4 -1
- package/src/eslint-sources.ts +134 -0
- package/src/executors/all-verdicts/executor.ts +58 -3
- package/src/executors/all-verdicts/schema.json +4 -0
- package/src/executors/all-verdicts/schema.ts +14 -0
- package/src/executors/bounded-exec/bounds.ts +132 -15
- package/src/executors/bounded-exec/executor.ts +303 -27
- package/src/executors/bounded-exec/hang-diagnostics.ts +3 -2
- package/src/executors/bounded-exec/mbx-pool.ts +82 -5
- package/src/executors/bounded-exec/mbx-queue.ts +170 -0
- package/src/executors/bounded-exec/progress.ts +168 -15
- package/src/executors/bounded-exec/test-result-writer.ts +152 -0
- package/src/executors/bounded-exec/test-results.ts +39 -0
- package/src/executors/bounded-exec/verdict.ts +108 -5
- package/src/index.ts +74 -70
- package/src/nx-publish.ts +157 -0
- package/src/runtime-input-policy.ts +1 -1
- package/src/tool-bins.ts +13 -9
- package/src/workspace-index.ts +12 -0
- package/test-timings.json +8 -3
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type ChildProcess, spawn } from 'node:child_process';
|
|
1
|
+
import { type ChildProcess, type IOType, spawn } from 'node:child_process';
|
|
2
2
|
import { accessSync, constants } from 'node:fs';
|
|
3
3
|
import { mkdtemp, rm } from 'node:fs/promises';
|
|
4
4
|
import { availableParallelism, userInfo } from 'node:os';
|
|
@@ -13,7 +13,17 @@ import {
|
|
|
13
13
|
processGroupHasMembers,
|
|
14
14
|
signalProcessGroup,
|
|
15
15
|
} from '../../process-group.js';
|
|
16
|
-
import {
|
|
16
|
+
import {
|
|
17
|
+
admissionWaitMs,
|
|
18
|
+
type BoundEvent,
|
|
19
|
+
type BoundKind,
|
|
20
|
+
type Bounds,
|
|
21
|
+
earliestBound,
|
|
22
|
+
NO_ADMISSION_WAIT,
|
|
23
|
+
stepAdmissionWait,
|
|
24
|
+
stepBounds,
|
|
25
|
+
UNARMED,
|
|
26
|
+
} from './bounds.js';
|
|
17
27
|
import {
|
|
18
28
|
CPU_POOL_ENV,
|
|
19
29
|
type CpuPool,
|
|
@@ -28,7 +38,8 @@ import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
|
28
38
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
29
39
|
import type { CpuWaitRow } from './lease-wait.js';
|
|
30
40
|
import type { LockWait } from './lock-wait.js';
|
|
31
|
-
import { mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
|
|
41
|
+
import { MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withMbxLeaf } from './mbx-pool.js';
|
|
42
|
+
import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
|
|
32
43
|
import { createNoSpaceWatch } from './no-space.js';
|
|
33
44
|
import {
|
|
34
45
|
COWSHED_EXECUTABLE_ENV,
|
|
@@ -39,7 +50,7 @@ import {
|
|
|
39
50
|
type SubtreeAdmission,
|
|
40
51
|
type SubtreeOwner,
|
|
41
52
|
} from './owned-subtree.js';
|
|
42
|
-
import { createRunProgress, withProgressReport } from './progress.js';
|
|
53
|
+
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
43
54
|
import {
|
|
44
55
|
createHostCommands,
|
|
45
56
|
describeCoverageGap,
|
|
@@ -56,10 +67,14 @@ import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from
|
|
|
56
67
|
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner } from './runner-sizing.js';
|
|
57
68
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
58
69
|
import type { BoundedExecOptions } from './schema.js';
|
|
70
|
+
import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
|
|
59
71
|
import {
|
|
72
|
+
type AdmissionQueueWait,
|
|
60
73
|
type BoundedExecRecord,
|
|
61
74
|
type BoundedExecTask,
|
|
75
|
+
BUN_TEST,
|
|
62
76
|
clearTaskDirectory,
|
|
77
|
+
describeAdmissionQueue,
|
|
63
78
|
describeNoSpace,
|
|
64
79
|
type ExpiredBound,
|
|
65
80
|
JUNIT_FILE,
|
|
@@ -291,6 +306,7 @@ export async function runBoundedExec(
|
|
|
291
306
|
inherited: InheritedCpuPool | null = null,
|
|
292
307
|
captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
|
|
293
308
|
owner: SubtreeOwner | null = null,
|
|
309
|
+
queueHost?: Parameters<typeof snapshotMbxQueue>[3],
|
|
294
310
|
): Promise<BoundedExecResult> {
|
|
295
311
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
296
312
|
const directory = context.record?.directory ?? null;
|
|
@@ -307,6 +323,8 @@ export async function runBoundedExec(
|
|
|
307
323
|
? { command: options.fixtureCommand, timeoutMs: options.fixtureTimeoutMs }
|
|
308
324
|
: null;
|
|
309
325
|
const outputChunks: string[] = [];
|
|
326
|
+
let resultWriter: TestResultWriter | null = null;
|
|
327
|
+
let resultFailure: string | null = null;
|
|
310
328
|
const state: RunState = {
|
|
311
329
|
settled: false,
|
|
312
330
|
expiry: null,
|
|
@@ -564,9 +582,27 @@ export async function runBoundedExec(
|
|
|
564
582
|
})();
|
|
565
583
|
return cpuReleased;
|
|
566
584
|
};
|
|
585
|
+
let mbxAdmitted = false;
|
|
586
|
+
let admissionWait = NO_ADMISSION_WAIT;
|
|
587
|
+
let lastQueueRequests: readonly MbxQueuedRequest[] = [];
|
|
588
|
+
let queueFailure: string | null = null;
|
|
589
|
+
const queueWait = (): AdmissionQueueWait | null =>
|
|
590
|
+
mbxAdmitted || lastQueueRequests.length === 0
|
|
591
|
+
? null
|
|
592
|
+
: {
|
|
593
|
+
waitedMs: admissionWaitMs(admissionWait, performance.now()),
|
|
594
|
+
requests: lastQueueRequests,
|
|
595
|
+
};
|
|
596
|
+
const queueAbort = new AbortController();
|
|
597
|
+
let queueTimer: NodeJS.Timeout | undefined;
|
|
598
|
+
let queueSnapshot: Promise<void> | null = null;
|
|
599
|
+
const stopQueueObservation = (): void => {
|
|
600
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
|
|
601
|
+
clearInterval(queueTimer);
|
|
602
|
+
queueAbort.abort();
|
|
603
|
+
};
|
|
567
604
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
568
|
-
//
|
|
569
|
-
// fired, so it is never judged as a bound.
|
|
605
|
+
// Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
|
|
570
606
|
const recordRun = async (
|
|
571
607
|
exitCode: number,
|
|
572
608
|
expiry: ExpiredBound | null,
|
|
@@ -579,10 +615,37 @@ export async function runBoundedExec(
|
|
|
579
615
|
return;
|
|
580
616
|
}
|
|
581
617
|
const { task, directory: recordDirectory } = context.record;
|
|
582
|
-
const report = await readJunitReport(
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
618
|
+
const report = await readJunitReport(
|
|
619
|
+
recordDirectory,
|
|
620
|
+
requested.search(BUN_TEST) !== -1 && resultWriter !== null
|
|
621
|
+
? (results) => {
|
|
622
|
+
const error = resultWriter?.reconcile(results) ?? null;
|
|
623
|
+
if (error !== null) {
|
|
624
|
+
resultFailure ??= error;
|
|
625
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
626
|
+
}
|
|
627
|
+
}
|
|
628
|
+
: undefined,
|
|
629
|
+
).finally(async () => {
|
|
630
|
+
const error = (await resultWriter?.close()) ?? null;
|
|
631
|
+
if (error !== null) {
|
|
632
|
+
resultFailure ??= error;
|
|
633
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
634
|
+
}
|
|
635
|
+
});
|
|
636
|
+
const admissionQueue = queueWait();
|
|
637
|
+
const verdict =
|
|
638
|
+
operational || resultFailure !== null
|
|
639
|
+
? judgeOperationalFailure(exitCode, report)
|
|
640
|
+
: judgeRun({
|
|
641
|
+
exitCode,
|
|
642
|
+
elapsedMs,
|
|
643
|
+
expiry,
|
|
644
|
+
report,
|
|
645
|
+
noSpace,
|
|
646
|
+
lockWaits,
|
|
647
|
+
admissionQueue,
|
|
648
|
+
});
|
|
586
649
|
if (verdict.outcome === 'bound' && verdict.bound === 'test') {
|
|
587
650
|
appendStderr(`Every failing test failed only on its per-test timeout: ${verdict.tests.join(', ')}\n`);
|
|
588
651
|
}
|
|
@@ -616,10 +679,11 @@ export async function runBoundedExec(
|
|
|
616
679
|
}
|
|
617
680
|
pool = created.pool;
|
|
618
681
|
}
|
|
682
|
+
const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
|
|
619
683
|
const command =
|
|
620
684
|
directory === null
|
|
621
|
-
? withProgressReport(
|
|
622
|
-
: await withJunitReport(withProgressReport(
|
|
685
|
+
? withProgressReport(native.command)
|
|
686
|
+
: await withJunitReport(withProgressReport(native.command), directory);
|
|
623
687
|
// A runner bounded-exec cannot see in the command (a script that spawns `bun test`) writes its JUnit report to this
|
|
624
688
|
// path when it is set. The grant's sizing outranks the target's own environment: the runner may use what it was
|
|
625
689
|
// granted, not what it was configured with. The sub-pool's environment comes last: a descendant reads only the pool
|
|
@@ -628,7 +692,9 @@ export async function runBoundedExec(
|
|
|
628
692
|
...(directory === null ? {} : { [JUNIT_REPORT_ENV]: join(directory, JUNIT_FILE) }),
|
|
629
693
|
...targetEnv(options.env, appendStderr),
|
|
630
694
|
...sized.env,
|
|
695
|
+
...native.env,
|
|
631
696
|
...(pool?.environment ?? {}),
|
|
697
|
+
...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
|
|
632
698
|
};
|
|
633
699
|
// A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
|
|
634
700
|
// of the command: the progress bound runs during it and its own ceiling does not.
|
|
@@ -646,25 +712,36 @@ export async function runBoundedExec(
|
|
|
646
712
|
// descriptor, and execs `/bin/sh -c command` in the same process: the command's pid, group and life are unchanged.
|
|
647
713
|
const unadmitted = hooked?.command ?? (fixture === null ? command : `${fixture.command} && ${command}`);
|
|
648
714
|
const shellCommand = leaf === null ? unadmitted : withMbxLeaf(unadmitted, leaf);
|
|
715
|
+
// Only leaves known to be scheduled can exclude zero-hold time. Others keep lifecycle hang protection.
|
|
716
|
+
const queued = mbxIdleGated(leaf, sized.command, { ...process.env, ...env });
|
|
717
|
+
// fd 4 is unused without a subtree admission, so the grant takes fd 5 either way.
|
|
718
|
+
const grantSlots: IOType[] = leaf !== null ? ['ignore', 'pipe'] : [];
|
|
719
|
+
const rootGrantSlot: IOType[] = leaf !== null ? ['pipe'] : [];
|
|
649
720
|
const root = owner?.rootArgv(shellCommand);
|
|
650
721
|
let child: ChildProcess;
|
|
722
|
+
const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
|
|
723
|
+
if (results !== null && !results.ok) return refuse(`bounded-exec: ${results.error}\n`);
|
|
724
|
+
resultWriter = results?.ok ? results.writer : null;
|
|
651
725
|
try {
|
|
652
726
|
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
653
|
-
//
|
|
727
|
+
// cowshed closes fd 4 before exec; fd 5 carries mbx's admission/release stream, including Cargo descendants.
|
|
654
728
|
child =
|
|
655
729
|
root === undefined
|
|
656
730
|
? spawn(shellCommand, [], {
|
|
657
731
|
cwd,
|
|
658
732
|
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
659
733
|
shell: true,
|
|
660
|
-
stdio:
|
|
734
|
+
stdio:
|
|
735
|
+
hooked === null && leaf === null
|
|
736
|
+
? 'pipe'
|
|
737
|
+
: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
|
|
661
738
|
detached: process.platform !== 'win32',
|
|
662
739
|
windowsHide: true,
|
|
663
740
|
})
|
|
664
741
|
: spawn(root.file, root.args, {
|
|
665
742
|
cwd,
|
|
666
743
|
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
667
|
-
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe'],
|
|
744
|
+
stdio: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot],
|
|
668
745
|
detached: process.platform !== 'win32',
|
|
669
746
|
windowsHide: true,
|
|
670
747
|
});
|
|
@@ -688,7 +765,20 @@ export async function runBoundedExec(
|
|
|
688
765
|
}
|
|
689
766
|
|
|
690
767
|
// Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
|
|
691
|
-
const progress = createRunProgress(
|
|
768
|
+
const progress = createRunProgress(
|
|
769
|
+
command,
|
|
770
|
+
resultWriter === null
|
|
771
|
+
? undefined
|
|
772
|
+
: (result) => {
|
|
773
|
+
const error = resultWriter?.append(result) ?? null;
|
|
774
|
+
if (error !== null && resultFailure === null) {
|
|
775
|
+
resultFailure = error;
|
|
776
|
+
lifecycleFailure ??= error;
|
|
777
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
778
|
+
termination = terminate();
|
|
779
|
+
}
|
|
780
|
+
},
|
|
781
|
+
);
|
|
692
782
|
// A write that found no space is said once, in the output, by whatever made it; the volume is read when it is said,
|
|
693
783
|
// since the writer usually removes what it half-wrote before it exits and the volume no longer looks full by then.
|
|
694
784
|
const noSpace = createNoSpaceWatch();
|
|
@@ -758,8 +848,43 @@ export async function runBoundedExec(
|
|
|
758
848
|
: kind === 'fixture'
|
|
759
849
|
? { kind, limitMs: fixture?.timeoutMs ?? 0 }
|
|
760
850
|
: { kind, limitMs: timeoutMs ?? 0 };
|
|
851
|
+
let checkingIdle = false;
|
|
852
|
+
const armBoundTimer = (): void => {
|
|
853
|
+
clearTimeout(boundTimer);
|
|
854
|
+
if (state.settled || state.expiry !== null || queueFailure !== null) return;
|
|
855
|
+
const next = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
|
|
856
|
+
if (next !== null) {
|
|
857
|
+
boundTimer = setTimeout(
|
|
858
|
+
() => {
|
|
859
|
+
void confirmExpiry(next.kind, next.at);
|
|
860
|
+
},
|
|
861
|
+
Math.max(0, next.at - Date.now()),
|
|
862
|
+
);
|
|
863
|
+
}
|
|
864
|
+
};
|
|
865
|
+
const confirmExpiry = async (kind: BoundKind, at: number): Promise<void> => {
|
|
866
|
+
if (state.settled || state.expiry !== null || queueFailure !== null) return;
|
|
867
|
+
const scheduled = earliestBound(checkingIdle ? { ...bounds, idle: null } : bounds);
|
|
868
|
+
if (scheduled?.kind !== kind || scheduled.at !== at) return;
|
|
869
|
+
if (kind === 'idle' && leaf !== null) {
|
|
870
|
+
if (checkingIdle) return;
|
|
871
|
+
checkingIdle = true;
|
|
872
|
+
armBoundTimer();
|
|
873
|
+
try {
|
|
874
|
+
// Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
|
|
875
|
+
// ceilings remain armed while this observation runs.
|
|
876
|
+
await refreshQueue(true);
|
|
877
|
+
} finally {
|
|
878
|
+
checkingIdle = false;
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
if (state.settled || state.expiry !== null || queueFailure !== null) return;
|
|
882
|
+
const next = earliestBound(bounds);
|
|
883
|
+
if (next !== null && next.at <= Date.now()) expire(expiredAs(next.kind));
|
|
884
|
+
else armBoundTimer();
|
|
885
|
+
};
|
|
761
886
|
const stepTo = (event: BoundEvent): void => {
|
|
762
|
-
if (state.settled || state.expiry !== null) return;
|
|
887
|
+
if (state.settled || state.expiry !== null || queueFailure !== null) return;
|
|
763
888
|
const idleBefore = bounds.idle;
|
|
764
889
|
bounds = stepBounds(
|
|
765
890
|
bounds,
|
|
@@ -775,9 +900,71 @@ export async function runBoundedExec(
|
|
|
775
900
|
idleCapture = undefined;
|
|
776
901
|
if (bounds.idle !== null) scheduleCapture(bounds.idle);
|
|
777
902
|
}
|
|
903
|
+
armBoundTimer();
|
|
904
|
+
};
|
|
905
|
+
|
|
906
|
+
let lastQueueDiagnostic = '';
|
|
907
|
+
let lastQueueDiagnosticAt: number | null = null;
|
|
908
|
+
let lastSnapshotDiagnostics = '';
|
|
909
|
+
const failQueueObservation = (detail: string): void => {
|
|
910
|
+
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || queueFailure !== null) return;
|
|
911
|
+
queueFailure = detail;
|
|
912
|
+
tell(
|
|
913
|
+
`bounded-exec: mbx admission snapshot failed: ${detail}; queue/admission state unconfirmed, not an idle wedge\n`,
|
|
914
|
+
);
|
|
778
915
|
clearTimeout(boundTimer);
|
|
779
|
-
|
|
780
|
-
|
|
916
|
+
cancelCapture();
|
|
917
|
+
stopQueueObservation();
|
|
918
|
+
termination = terminate();
|
|
919
|
+
};
|
|
920
|
+
const refreshQueue = async (fresh: boolean): Promise<void> => {
|
|
921
|
+
if (fresh && queueSnapshot !== null) await queueSnapshot;
|
|
922
|
+
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
|
|
923
|
+
const pid = child.pid;
|
|
924
|
+
queueSnapshot ??= (async () => {
|
|
925
|
+
const snapshot = await snapshotMbxQueue(
|
|
926
|
+
pid,
|
|
927
|
+
mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
928
|
+
queueAbort.signal,
|
|
929
|
+
queueHost,
|
|
930
|
+
);
|
|
931
|
+
if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
|
|
932
|
+
if (!snapshot.ok) {
|
|
933
|
+
failQueueObservation(snapshot.detail);
|
|
934
|
+
return;
|
|
935
|
+
}
|
|
936
|
+
const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
|
|
937
|
+
if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics) tell(`bounded-exec: ${diagnostics}\n`);
|
|
938
|
+
lastSnapshotDiagnostics = diagnostics;
|
|
939
|
+
const waiting = snapshot.requests.length > 0;
|
|
940
|
+
const observedAt = Date.now();
|
|
941
|
+
const observedWaitAt = performance.now();
|
|
942
|
+
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
|
|
943
|
+
if (waiting) {
|
|
944
|
+
lastQueueRequests = snapshot.requests;
|
|
945
|
+
const positions = JSON.stringify(snapshot.requests);
|
|
946
|
+
if (
|
|
947
|
+
positions !== lastQueueDiagnostic ||
|
|
948
|
+
lastQueueDiagnosticAt === null ||
|
|
949
|
+
Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS
|
|
950
|
+
) {
|
|
951
|
+
const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
|
|
952
|
+
tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
|
|
953
|
+
lastQueueDiagnostic = positions;
|
|
954
|
+
lastQueueDiagnosticAt = Date.now();
|
|
955
|
+
}
|
|
956
|
+
} else {
|
|
957
|
+
lastQueueDiagnostic = '';
|
|
958
|
+
}
|
|
959
|
+
stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
|
|
960
|
+
})()
|
|
961
|
+
.catch((error: unknown) => {
|
|
962
|
+
failQueueObservation(error instanceof Error ? error.message : String(error));
|
|
963
|
+
})
|
|
964
|
+
.finally(() => {
|
|
965
|
+
queueSnapshot = null;
|
|
966
|
+
});
|
|
967
|
+
await queueSnapshot;
|
|
781
968
|
};
|
|
782
969
|
|
|
783
970
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
@@ -786,9 +973,15 @@ export async function runBoundedExec(
|
|
|
786
973
|
const updateIdle = (finished: boolean, wasTesting: boolean): void => {
|
|
787
974
|
stepTo({ kind: 'progress', at: Date.now(), testing: isTesting(), wasTesting, finished });
|
|
788
975
|
};
|
|
789
|
-
const scanProgress = (stream: 'stdout' | 'stderr', text: string): void => {
|
|
976
|
+
const scanProgress = (stream: 'stdout' | 'stderr', text: Buffer | string): void => {
|
|
790
977
|
if (!fixtureDone) return;
|
|
791
978
|
updateIdle(progress.scan(stream, text), isTesting());
|
|
979
|
+
const error = progress.error();
|
|
980
|
+
if (error !== null && lifecycleFailure === null) {
|
|
981
|
+
lifecycleFailure = error;
|
|
982
|
+
appendStderr(`bounded-exec: ${error}\n`);
|
|
983
|
+
termination = terminate();
|
|
984
|
+
}
|
|
792
985
|
};
|
|
793
986
|
if (lifecycle !== null) {
|
|
794
987
|
const channel = child.stdio[3];
|
|
@@ -826,14 +1019,14 @@ export async function runBoundedExec(
|
|
|
826
1019
|
}
|
|
827
1020
|
child.stdout?.on('data', (chunk: Buffer | string) => {
|
|
828
1021
|
const text = chunk.toString();
|
|
829
|
-
scanProgress('stdout',
|
|
1022
|
+
scanProgress('stdout', chunk);
|
|
830
1023
|
const said = noSpace.scan('stdout', text);
|
|
831
1024
|
if (said !== null) state.noSpace = sight(said);
|
|
832
1025
|
appendStdout(text);
|
|
833
1026
|
});
|
|
834
1027
|
child.stderr?.on('data', (chunk: Buffer | string) => {
|
|
835
1028
|
const text = chunk.toString();
|
|
836
|
-
scanProgress('stderr',
|
|
1029
|
+
scanProgress('stderr', chunk);
|
|
837
1030
|
const said = noSpace.scan('stderr', text);
|
|
838
1031
|
if (said !== null) state.noSpace = sight(said);
|
|
839
1032
|
appendStderr(text);
|
|
@@ -857,12 +1050,33 @@ export async function runBoundedExec(
|
|
|
857
1050
|
const onProcessExit = (): void => {
|
|
858
1051
|
void killChildTree();
|
|
859
1052
|
};
|
|
1053
|
+
let signalCleanup: Promise<void> | null = null;
|
|
860
1054
|
const onTerminationSignal = (signal: NodeJS.Signals): void => {
|
|
861
1055
|
removeSignalHandlers();
|
|
862
1056
|
cancelCapture();
|
|
863
|
-
|
|
1057
|
+
clearTimeout(boundTimer);
|
|
1058
|
+
// Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
|
|
1059
|
+
// reports an operational failure, never a manufactured queue or admission.
|
|
1060
|
+
signalCleanup = (leaf === null ? Promise.resolve() : refreshQueue(false))
|
|
1061
|
+
.finally(() => {
|
|
1062
|
+
stopQueueObservation();
|
|
1063
|
+
return killChildTree();
|
|
1064
|
+
})
|
|
864
1065
|
.finally(releaseCpu)
|
|
865
1066
|
.finally(() => releaseLease(true))
|
|
1067
|
+
.finally(async () =>
|
|
1068
|
+
recordRun(
|
|
1069
|
+
signalToExitCode(signal),
|
|
1070
|
+
state.expiry,
|
|
1071
|
+
Date.now() - startedAt,
|
|
1072
|
+
state.poolCloseError !== null ||
|
|
1073
|
+
lifecycleFailure !== null ||
|
|
1074
|
+
state.retireError !== null ||
|
|
1075
|
+
queueFailure !== null,
|
|
1076
|
+
state.noSpace === null ? null : await state.noSpace,
|
|
1077
|
+
state.lockWaits,
|
|
1078
|
+
),
|
|
1079
|
+
)
|
|
866
1080
|
.finally(() => process.kill(process.pid, signal));
|
|
867
1081
|
};
|
|
868
1082
|
const onSigint = (): void => onTerminationSignal('SIGINT');
|
|
@@ -888,10 +1102,12 @@ export async function runBoundedExec(
|
|
|
888
1102
|
child.once('error', (error) => {
|
|
889
1103
|
appendStderr(`${error.message}\n`);
|
|
890
1104
|
state.settled = true;
|
|
1105
|
+
stopQueueObservation();
|
|
891
1106
|
resolve({ code: 1, signal: null });
|
|
892
1107
|
});
|
|
893
1108
|
child.once(hooked === null ? 'exit' : 'close', (code: number | null, signal: NodeJS.Signals | null) => {
|
|
894
1109
|
state.settled = true;
|
|
1110
|
+
stopQueueObservation();
|
|
895
1111
|
resolve({ code, signal });
|
|
896
1112
|
});
|
|
897
1113
|
});
|
|
@@ -1010,7 +1226,54 @@ export async function runBoundedExec(
|
|
|
1010
1226
|
|
|
1011
1227
|
// The fixture's ceiling lives and dies with its segment: the segment's terminal clears it, so it can never
|
|
1012
1228
|
// fire on a command the fixture already handed over to.
|
|
1013
|
-
stepTo({
|
|
1229
|
+
stepTo({
|
|
1230
|
+
kind: 'spawn',
|
|
1231
|
+
at: Date.now(),
|
|
1232
|
+
testing: isTesting(),
|
|
1233
|
+
fixture: fixture !== null && lifecycle !== null,
|
|
1234
|
+
queued,
|
|
1235
|
+
});
|
|
1236
|
+
// Read every byte after spawn accounting: a chunk can contain several grants and releases. EOF is neither.
|
|
1237
|
+
if (leaf !== null) {
|
|
1238
|
+
const channel = child.stdio.at(MBX_GRANT_FD);
|
|
1239
|
+
if (!(channel instanceof Readable)) throw new Error('an mbx leaf requires its grant descriptor');
|
|
1240
|
+
let said = false;
|
|
1241
|
+
const failGrant = (detail: string): void => {
|
|
1242
|
+
if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
|
|
1243
|
+
lifecycleFailure ??= `mbx grant descriptor: ${detail}`;
|
|
1244
|
+
appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
|
|
1245
|
+
termination = terminate();
|
|
1246
|
+
};
|
|
1247
|
+
channel.on('data', (chunk: Buffer) => {
|
|
1248
|
+
for (const byte of chunk) {
|
|
1249
|
+
if (byte === 103) {
|
|
1250
|
+
if (!said) {
|
|
1251
|
+
mbxAdmitted = true;
|
|
1252
|
+
said = true;
|
|
1253
|
+
tell(
|
|
1254
|
+
leaf.kind === 'reserve'
|
|
1255
|
+
? `cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms cpus=${leaf.cpus} priority=${leaf.priority}\n`
|
|
1256
|
+
: `cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`,
|
|
1257
|
+
);
|
|
1258
|
+
}
|
|
1259
|
+
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
1260
|
+
} else if (byte === 114) {
|
|
1261
|
+
stepTo({ kind: 'released', at: Date.now() });
|
|
1262
|
+
} else {
|
|
1263
|
+
failGrant(`unexpected byte ${byte}`);
|
|
1264
|
+
return;
|
|
1265
|
+
}
|
|
1266
|
+
}
|
|
1267
|
+
});
|
|
1268
|
+
channel.on('error', (error) => failGrant(error.message));
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
if (leaf !== null) {
|
|
1272
|
+
queueTimer = setInterval(() => {
|
|
1273
|
+
void refreshQueue(false);
|
|
1274
|
+
}, MBX_QUEUE_POLL_MS);
|
|
1275
|
+
void refreshQueue(false);
|
|
1276
|
+
}
|
|
1014
1277
|
|
|
1015
1278
|
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
1016
1279
|
// under it runs on over budget until it finishes. Its close, while the command runs, is said and changes nothing
|
|
@@ -1029,6 +1292,12 @@ export async function runBoundedExec(
|
|
|
1029
1292
|
}
|
|
1030
1293
|
|
|
1031
1294
|
const { code: exitCode, signal: exitSignal } = await exitPromise;
|
|
1295
|
+
if (signalCleanup !== null) {
|
|
1296
|
+
await signalCleanup;
|
|
1297
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
1298
|
+
}
|
|
1299
|
+
stopQueueObservation();
|
|
1300
|
+
await queueSnapshot;
|
|
1032
1301
|
await termination;
|
|
1033
1302
|
clearTimeout(boundTimer);
|
|
1034
1303
|
cancelCapture();
|
|
@@ -1062,14 +1331,21 @@ export async function runBoundedExec(
|
|
|
1062
1331
|
appendStderr(`${describeRamTempError(full)}\n`);
|
|
1063
1332
|
}
|
|
1064
1333
|
}
|
|
1065
|
-
await releaseLease(
|
|
1334
|
+
await releaseLease(
|
|
1335
|
+
code !== 0 ||
|
|
1336
|
+
state.expiry !== null ||
|
|
1337
|
+
state.poolCloseError !== null ||
|
|
1338
|
+
lifecycleFailure !== null ||
|
|
1339
|
+
queueFailure !== null,
|
|
1340
|
+
);
|
|
1066
1341
|
|
|
1067
1342
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1068
1343
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1069
|
-
const operationalFailure =
|
|
1344
|
+
const operationalFailure =
|
|
1345
|
+
state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null || queueFailure !== null;
|
|
1070
1346
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
1071
1347
|
return {
|
|
1072
|
-
success: !operationalFailure && state.expiry === null && code === 0,
|
|
1348
|
+
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1073
1349
|
terminalOutput: outputChunks.join(''),
|
|
1074
1350
|
};
|
|
1075
1351
|
}
|
|
@@ -383,11 +383,12 @@ export function stopProbe(file: string, pid: number): boolean {
|
|
|
383
383
|
}
|
|
384
384
|
|
|
385
385
|
/** A probe owns its group and output pipes; its deadline never waits for an unresponsive child to reap. */
|
|
386
|
-
function runProbe(
|
|
386
|
+
export function runProbe(
|
|
387
387
|
file: string,
|
|
388
388
|
args: readonly string[],
|
|
389
389
|
timeoutMs: number,
|
|
390
390
|
signal: AbortSignal,
|
|
391
|
+
environment?: NodeJS.ProcessEnv,
|
|
391
392
|
): Promise<HangProbeResult> {
|
|
392
393
|
// Keep the package's es2022-compatible event-completion convention.
|
|
393
394
|
return new Promise((resolve) => {
|
|
@@ -395,7 +396,7 @@ function runProbe(
|
|
|
395
396
|
resolve({ status: 'timed-out', detail: 'capture deadline already reached', stdout: '', stderr: '' });
|
|
396
397
|
return;
|
|
397
398
|
}
|
|
398
|
-
const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
|
399
|
+
const child = spawn(file, [...args], { detached: true, stdio: ['ignore', 'pipe', 'pipe'], env: environment });
|
|
399
400
|
let stdout = '';
|
|
400
401
|
let stderr = '';
|
|
401
402
|
let bytes = 0;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { shellTokens } from '../../runtime-input-policy.js';
|
|
2
|
+
import { invocationSpan, type RunnerKind } from './runner-sizing.js';
|
|
2
3
|
import { shellQuote } from './verdict.js';
|
|
3
4
|
|
|
4
5
|
/**
|
|
@@ -11,9 +12,15 @@ export interface MbxPool {
|
|
|
11
12
|
readonly priority: 'normal' | 'low';
|
|
12
13
|
}
|
|
13
14
|
|
|
14
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged, or
|
|
17
|
+
* where this run is already inside an mbx reservation: `mbx reserve` hands its command `MBX_SCHED_DISABLE=1`, and nested
|
|
18
|
+
* non-Cargo work shares the enclosing grant instead of reserving twice (specs/cowshed/05_gateway.md, "Parallelism
|
|
19
|
+
* inside a job"). A second reservation would wait in the pool while its parent holds a grant, so under a full pool the
|
|
20
|
+
* nested leaf queued behind the very work it belongs to.
|
|
21
|
+
*/
|
|
15
22
|
export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | null {
|
|
16
|
-
if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR) {
|
|
23
|
+
if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR || environment.MBX_SCHED_DISABLE === '1') {
|
|
17
24
|
return null;
|
|
18
25
|
}
|
|
19
26
|
return { priority: environment.MBX_SCHEDULER_PRIORITY === 'normal' ? 'normal' : 'low' };
|
|
@@ -36,13 +43,83 @@ export function mbxLeaf(pool: MbxPool, kind: RunnerKind, want: number): MbxLeaf
|
|
|
36
43
|
: { kind: 'reserve', cpus: Math.max(1, want), priority: pool.priority };
|
|
37
44
|
}
|
|
38
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Only a known scheduled execution may stop its idle clock at zero holds. Plain Cargo tests can run doctests;
|
|
48
|
+
* archive-file targets, direct binaries and other runners are not proven scheduled at spawn (11_shell.md,
|
|
49
|
+
* "Cargo test-runner boundaries"). The inferred extracted native nextest command is a supported Cargo invocation.
|
|
50
|
+
*/
|
|
51
|
+
export function mbxIdleGated(leaf: MbxLeaf | null, command: string, environment: Readonly<NodeJS.ProcessEnv>): boolean {
|
|
52
|
+
if (leaf === null) return false;
|
|
53
|
+
if (leaf.kind === 'reserve') return true;
|
|
54
|
+
if (environment.MBX_SCHEDULER_TESTS !== '1') return false;
|
|
55
|
+
if (environment.MBX_SCHEDULER === '0' || environment.MBX_SCHED_DISABLE === '1' || environment.MBX_DISABLE === '1')
|
|
56
|
+
return false;
|
|
57
|
+
// invocationSpan shares the lifecycle mapper's conservative shell-syntax boundary.
|
|
58
|
+
if (/[<>\n\r`\\]/.test(command)) return false;
|
|
59
|
+
if (/[^\S \t]/.test(command)) return false;
|
|
60
|
+
const invocation = /cargo(?:\s+(?:--frozen|--locked|--offline))*\s+(nextest\s+run|test)(?=\s|$)/;
|
|
61
|
+
const found = invocation.exec(command);
|
|
62
|
+
if (found === null) return false;
|
|
63
|
+
const prefix = command.slice(0, found.index).trim();
|
|
64
|
+
if (prefix !== '' && !/^extracted="\$\([^"\n]+\)"\s*&&$/.test(prefix)) return false;
|
|
65
|
+
let remaining = command.slice(found.index);
|
|
66
|
+
const unquoted = remaining.replace(/"[^"]*"|'[^']*'/g, '');
|
|
67
|
+
if (/[$*?{}~[\]]/.test(unquoted)) return false;
|
|
68
|
+
// Only the inferred command's path templates are known expansions; arbitrary words could supply mbx overrides.
|
|
69
|
+
const paths = remaining.replace(/\$PWD(?=\/)/g, '');
|
|
70
|
+
if ((prefix === '' ? paths : paths.replace(/\$extracted(?=\/)/g, '')).includes('$')) return false;
|
|
71
|
+
for (;;) {
|
|
72
|
+
const leg = invocation.exec(remaining);
|
|
73
|
+
if (leg === null || leg.index !== 0) return false;
|
|
74
|
+
const { argsEnd } = invocationSpan(remaining, invocation);
|
|
75
|
+
const args: string[] = [];
|
|
76
|
+
for (const token of shellTokens(remaining.slice(0, argsEnd), [])) {
|
|
77
|
+
if (token.kind !== 'word') return false;
|
|
78
|
+
if (token.text === '--') break;
|
|
79
|
+
if (
|
|
80
|
+
token.text.startsWith('$') &&
|
|
81
|
+
!(
|
|
82
|
+
token.text.startsWith('$extracted/') &&
|
|
83
|
+
['--binaries-metadata', '--cargo-metadata', '--target-dir-remap'].includes(args.at(-1) ?? '')
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
return false;
|
|
87
|
+
args.push(token.text);
|
|
88
|
+
}
|
|
89
|
+
if (
|
|
90
|
+
args.some((arg) =>
|
|
91
|
+
/^(?:--(?:config|directory|archive-file)(?:=|$)|--(?:doc|no-run|list|help)(?:=|$)|-C|-h$|\+)/.test(arg),
|
|
92
|
+
)
|
|
93
|
+
) {
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
if (
|
|
97
|
+
leg[1] === 'test' &&
|
|
98
|
+
!args.some((arg) => /^--(?:lib|tests|bins|examples|benches|all-targets|test|bin|example|bench)(?:=|$)/.test(arg))
|
|
99
|
+
) {
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
const suffix = remaining.slice(argsEnd).trim();
|
|
103
|
+
if (suffix === '') return true;
|
|
104
|
+
const separator = /^(?:&&|\|\||;)\s*/.exec(suffix);
|
|
105
|
+
if (separator === null) return false;
|
|
106
|
+
remaining = suffix.slice(separator[0].length);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* mbx writes one byte per admission (g) and permit drop (r). Cargo descendants inherit this descriptor through
|
|
112
|
+
* MBX_GRANT_FD; reserve names it with --grant-fd and closes it on exec of the external command.
|
|
113
|
+
*/
|
|
114
|
+
export const MBX_GRANT_FD = 5;
|
|
115
|
+
|
|
39
116
|
/**
|
|
40
117
|
* `command` as the shell runs it under its leaf's admission. `mbx` is the canonical environment's, whose realpath
|
|
41
|
-
* cowshed's admission proved to be the host's pinned mbx.
|
|
118
|
+
* cowshed's admission proved to be the host's pinned mbx; it writes the grant on this descriptor itself.
|
|
42
119
|
*/
|
|
43
120
|
export function withMbxLeaf(command: string, leaf: MbxLeaf): string {
|
|
44
121
|
if (leaf.kind === 'cargo') {
|
|
45
122
|
return command;
|
|
46
123
|
}
|
|
47
|
-
return `mbx reserve --cpus ${leaf.cpus} --priority ${leaf.priority} -- /bin/sh -c ${shellQuote(command)}`;
|
|
124
|
+
return `mbx reserve --grant-fd ${MBX_GRANT_FD} --cpus ${leaf.cpus} --priority ${leaf.priority} -- /bin/sh -c ${shellQuote(command)}`;
|
|
48
125
|
}
|