@smoothbricks/nx-plugin 0.7.11 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -25
- package/dist/.build-inputs.json +25 -17
- package/dist/built-output-packages.d.ts +27 -0
- package/dist/built-output-packages.d.ts.map +1 -0
- package/dist/built-output-packages.js +97 -0
- package/dist/cross-check-policy.d.ts +13 -1
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +13 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +68 -0
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -0
- package/dist/executors/bounded-exec/admission-queue.js +131 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts +14 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-grant.js +18 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +26 -24
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +88 -72
- package/dist/executors/bounded-exec/executor.d.ts +5 -7
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +69 -133
- package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
- package/dist/executors/bounded-exec/host-memory.js +6 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
- package/dist/executors/bounded-exec/memory-retry.d.ts +39 -0
- package/dist/executors/bounded-exec/memory-retry.d.ts.map +1 -0
- package/dist/executors/bounded-exec/memory-retry.js +33 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts +43 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/parent-grant.js +37 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts +17 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-charge.js +20 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +44 -8
- package/dist/executors/bounded-exec/pool-environment.d.ts +25 -0
- package/dist/executors/bounded-exec/pool-environment.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-environment.js +34 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +5 -18
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +12 -16
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -21
- package/dist/managed-files/ci-workflow.d.ts +29 -5
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +111 -16
- package/dist/managed-files/context.d.ts.map +1 -1
- package/dist/managed-files/context.js +1 -0
- package/dist/managed-files/files.d.ts +3 -1
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +2 -0
- package/dist/managed-files/publish-workflow.d.ts +6 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/managed-files/publish-workflow.js +16 -11
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +48 -14
- package/dist/typecheck-route.d.ts +28 -0
- package/dist/typecheck-route.d.ts.map +1 -0
- package/dist/typecheck-route.js +109 -0
- package/dist/workspace-manifest.d.ts +50 -5
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/executors.json +10 -0
- package/managed/raw/patches/nx@24.0.3.patch +16 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +93 -8
- package/managed/raw/tooling/direnv/apple-developer.sh +8 -1
- package/managed/raw/tooling/direnv/compiler-variables.sh +1 -1
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +4 -2
- package/managed/raw/tooling/direnv/secret-references.ts +19 -5
- package/managed/raw/tooling/git-hooks/pre-push.sh +20 -10
- package/managed/templates/github/actions/forgejo-artifact.cjs +9 -9
- package/managed/templates/github/actions/setup-devenv/action.yml +53 -0
- package/package.json +2 -2
- package/src/built-output-packages.ts +125 -0
- package/src/cross-check-policy.ts +14 -1
- package/src/executors/bounded-exec/admission-queue.ts +163 -0
- package/src/executors/bounded-exec/cpu-grant.ts +32 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
- package/src/executors/bounded-exec/cpu-pool.ts +123 -58
- package/src/executors/bounded-exec/executor.ts +93 -168
- package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
- package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
- package/src/executors/bounded-exec/host-memory.ts +25 -0
- package/src/executors/bounded-exec/memory-retry.ts +63 -0
- package/src/executors/bounded-exec/parent-grant.ts +75 -0
- package/src/executors/bounded-exec/peak-charge.ts +25 -0
- package/src/executors/bounded-exec/pool-admission.ts +66 -11
- package/src/executors/bounded-exec/pool-environment.ts +45 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +5 -4
- package/src/executors/bounded-exec/verdict.ts +17 -25
- package/src/index.ts +70 -21
- package/src/managed-files/ci-workflow.ts +143 -16
- package/src/managed-files/context.ts +1 -0
- package/src/managed-files/files.ts +5 -0
- package/src/managed-files/publish-workflow.ts +26 -10
- package/src/owned-installation.ts +63 -14
- package/src/typecheck-route.ts +64 -0
- package/src/workspace-manifest.ts +56 -5
- package/test-timings.json +141 -137
|
@@ -13,31 +13,13 @@ import {
|
|
|
13
13
|
processGroupHasMembers,
|
|
14
14
|
signalProcessGroup,
|
|
15
15
|
} from '../../process-group.js';
|
|
16
|
-
import {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
type BoundKind,
|
|
20
|
-
type Bounds,
|
|
21
|
-
earliestBound,
|
|
22
|
-
NO_ADMISSION_WAIT,
|
|
23
|
-
stepAdmissionWait,
|
|
24
|
-
stepBounds,
|
|
25
|
-
UNARMED,
|
|
26
|
-
} from './bounds.js';
|
|
27
|
-
import {
|
|
28
|
-
CPU_POOL_ENV,
|
|
29
|
-
type CpuPool,
|
|
30
|
-
createCpuPool,
|
|
31
|
-
type InheritedCpuPool,
|
|
32
|
-
readInheritedCpuPool,
|
|
33
|
-
type TokenGrant,
|
|
34
|
-
takeParentCpuTokens,
|
|
35
|
-
} from './cpu-pool.js';
|
|
16
|
+
import { AdmissionObservation, type QueueHost } from './admission-queue.js';
|
|
17
|
+
import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
18
|
+
import { type CpuPool, createCpuPool } from './cpu-pool.js';
|
|
36
19
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
37
20
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
38
21
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
39
22
|
import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
40
|
-
import type { CpuWaitRow } from './lease-wait.js';
|
|
41
23
|
import type { LockWait } from './lock-wait.js';
|
|
42
24
|
import {
|
|
43
25
|
askedCores,
|
|
@@ -48,8 +30,8 @@ import {
|
|
|
48
30
|
withCountedReservation,
|
|
49
31
|
withMbxLeaf,
|
|
50
32
|
} from './mbx-pool.js';
|
|
51
|
-
import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
|
|
52
33
|
import { describeUnserved, RESERVATIONS_ENV, type Reservation, reserveCores, runEndpoint } from './mbx-reservations.js';
|
|
34
|
+
import { killedFromOutside, type RunOutcome, runUntilNotKilledForMemory } from './memory-retry.js';
|
|
53
35
|
import { createNoSpaceWatch } from './no-space.js';
|
|
54
36
|
import {
|
|
55
37
|
COWSHED_EXECUTABLE_ENV,
|
|
@@ -60,6 +42,9 @@ import {
|
|
|
60
42
|
type SubtreeAdmission,
|
|
61
43
|
type SubtreeOwner,
|
|
62
44
|
} from './owned-subtree.js';
|
|
45
|
+
import { askParentPool, describeGrant } from './parent-grant.js';
|
|
46
|
+
import { type LastPeak, readLastPeak } from './peak-charge.js';
|
|
47
|
+
import { CPU_POOL_ENV, type InheritedCpuPool, readInheritedCpuPool } from './pool-environment.js';
|
|
63
48
|
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
64
49
|
import {
|
|
65
50
|
createHostCommands,
|
|
@@ -79,12 +64,10 @@ import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
|
79
64
|
import type { BoundedExecOptions } from './schema.js';
|
|
80
65
|
import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
|
|
81
66
|
import {
|
|
82
|
-
type AdmissionQueueWait,
|
|
83
67
|
type BoundedExecRecord,
|
|
84
68
|
type BoundedExecTask,
|
|
85
69
|
BUN_TEST,
|
|
86
70
|
clearTaskDirectory,
|
|
87
|
-
describeAdmissionQueue,
|
|
88
71
|
describeNoSpace,
|
|
89
72
|
type ExpiredBound,
|
|
90
73
|
JUNIT_FILE,
|
|
@@ -93,7 +76,6 @@ import {
|
|
|
93
76
|
judgeRun,
|
|
94
77
|
type NoSpaceSighting,
|
|
95
78
|
readJunitReport,
|
|
96
|
-
readLastPeak,
|
|
97
79
|
taskDirectory,
|
|
98
80
|
withJunitReport,
|
|
99
81
|
writeRecord,
|
|
@@ -131,15 +113,14 @@ export interface NxTaskContext {
|
|
|
131
113
|
taskGraph?: { tasks: Record<string, { hash?: string }> };
|
|
132
114
|
}
|
|
133
115
|
|
|
134
|
-
export
|
|
135
|
-
success: boolean;
|
|
136
|
-
terminalOutput: string;
|
|
137
|
-
}
|
|
116
|
+
export type BoundedExecResult = RunOutcome;
|
|
138
117
|
|
|
139
118
|
interface RunState {
|
|
140
119
|
settled: boolean;
|
|
141
120
|
expiry: ExpiredBound | null;
|
|
142
121
|
forceKillNeeded: boolean;
|
|
122
|
+
/** This executor sent the command's process group a signal: a SIGKILL it died of is not from outside. */
|
|
123
|
+
sentSignal: boolean;
|
|
143
124
|
/** The sub-pool's close error when it did not close cleanly: the run fails. */
|
|
144
125
|
poolCloseError: string | null;
|
|
145
126
|
/** Why the command's subtree could not be retired, when the supervisor's client failed: the run fails. */
|
|
@@ -150,8 +131,6 @@ interface RunState {
|
|
|
150
131
|
lockWaits: readonly LockWait[];
|
|
151
132
|
}
|
|
152
133
|
|
|
153
|
-
type GrantedTokens = Extract<TokenGrant, { granted: true }>;
|
|
154
|
-
|
|
155
134
|
export interface ProcessTreeKiller {
|
|
156
135
|
kill(pid: number, signal: NodeJS.Signals): Promise<void>;
|
|
157
136
|
}
|
|
@@ -311,12 +290,45 @@ export async function runBoundedExec(
|
|
|
311
290
|
inherited: InheritedCpuPool | null = null,
|
|
312
291
|
captureDiagnostics: typeof captureHangDiagnostics = captureHangDiagnostics,
|
|
313
292
|
owner: SubtreeOwner | null = null,
|
|
314
|
-
queueHost?:
|
|
293
|
+
queueHost?: QueueHost,
|
|
315
294
|
): Promise<BoundedExecResult> {
|
|
316
|
-
const cwd = resolveCwd(options.cwd, context.root);
|
|
317
295
|
const directory = context.record?.directory ?? null;
|
|
318
|
-
// The task's last recorded peak
|
|
296
|
+
// The task's last recorded peak, and the tokens it was lent when the peak was measured, are its charge in its parent's
|
|
297
|
+
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
298
|
+
// killed run replaced, not by the peak it reached before it was killed.
|
|
319
299
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
300
|
+
return runUntilNotKilledForMemory(
|
|
301
|
+
(alone) =>
|
|
302
|
+
runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
303
|
+
lastPeak,
|
|
304
|
+
alone,
|
|
305
|
+
}),
|
|
306
|
+
inherited !== null,
|
|
307
|
+
context.record?.task.id,
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/** What a run of the command is told that the run before it was not: the task's charge, and whether it runs alone. */
|
|
312
|
+
interface Attempt {
|
|
313
|
+
readonly lastPeak: LastPeak | null;
|
|
314
|
+
readonly alone: boolean;
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/** One run of the command, from the pool's grant to its record. */
|
|
318
|
+
async function runCommand(
|
|
319
|
+
options: BoundedExecOptions,
|
|
320
|
+
context: BoundedExecContext,
|
|
321
|
+
killer: ProcessTreeKiller,
|
|
322
|
+
tempVolume: TempVolume | null,
|
|
323
|
+
inherited: InheritedCpuPool | null,
|
|
324
|
+
captureDiagnostics: typeof captureHangDiagnostics,
|
|
325
|
+
owner: SubtreeOwner | null,
|
|
326
|
+
queueHost: QueueHost | undefined,
|
|
327
|
+
attempt: Attempt,
|
|
328
|
+
): Promise<BoundedExecResult> {
|
|
329
|
+
const cwd = resolveCwd(options.cwd, context.root);
|
|
330
|
+
const directory = context.record?.directory ?? null;
|
|
331
|
+
const { lastPeak } = attempt;
|
|
320
332
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
321
333
|
if (directory !== null) {
|
|
322
334
|
await clearTaskDirectory(directory);
|
|
@@ -336,6 +348,7 @@ export async function runBoundedExec(
|
|
|
336
348
|
settled: false,
|
|
337
349
|
expiry: null,
|
|
338
350
|
forceKillNeeded: false,
|
|
351
|
+
sentSignal: false,
|
|
339
352
|
poolCloseError: null,
|
|
340
353
|
retireError: null,
|
|
341
354
|
noSpace: null,
|
|
@@ -408,8 +421,10 @@ export async function runBoundedExec(
|
|
|
408
421
|
`bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`,
|
|
409
422
|
);
|
|
410
423
|
}
|
|
411
|
-
// A task with no recorded peak is charged nothing (null) until it has one.
|
|
424
|
+
// A task with no recorded peak is charged nothing (null) until it has one. A peak is charged in proportion to the
|
|
425
|
+
// tokens the run is lent when that is more than the run that measured it was lent.
|
|
412
426
|
const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
|
|
427
|
+
const chargeParallelism = lastPeak?.ok === true ? lastPeak.parallelism : null;
|
|
413
428
|
|
|
414
429
|
let lease: RamTempLease | null = null;
|
|
415
430
|
let ordinaryTemp: string | null = null;
|
|
@@ -553,20 +568,15 @@ export async function runBoundedExec(
|
|
|
553
568
|
const asked = Date.now();
|
|
554
569
|
// A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
|
|
555
570
|
// instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
} else {
|
|
566
|
-
parentRefusal = fromParent.reason;
|
|
567
|
-
parentWait = fromParent.wait;
|
|
568
|
-
}
|
|
569
|
-
}
|
|
571
|
+
const parent = await askParentPool(inherited, {
|
|
572
|
+
want: demand.want,
|
|
573
|
+
charge,
|
|
574
|
+
parallelism: chargeParallelism,
|
|
575
|
+
alone: attempt.alone,
|
|
576
|
+
task: context.record?.task.id,
|
|
577
|
+
});
|
|
578
|
+
for (const said of parent.said) appendStderr(said);
|
|
579
|
+
const { grant, refusal: parentRefusal, wait: parentWait } = parent;
|
|
570
580
|
const waitedMs = Date.now() - asked;
|
|
571
581
|
let cpu: BoundedExecRecord['cpu'];
|
|
572
582
|
let sized: { command: string; env: Record<string, string> } = { command: requested, env: {} };
|
|
@@ -577,13 +587,8 @@ export async function runBoundedExec(
|
|
|
577
587
|
if (grant !== null) {
|
|
578
588
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
579
589
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
);
|
|
583
|
-
if (grant.oversized !== null) {
|
|
584
|
-
appendStderr(
|
|
585
|
-
`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
|
|
586
|
-
);
|
|
590
|
+
for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
|
|
591
|
+
appendStderr(said);
|
|
587
592
|
}
|
|
588
593
|
} else if (leaf?.kind === 'reserve') {
|
|
589
594
|
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
@@ -611,33 +616,15 @@ export async function runBoundedExec(
|
|
|
611
616
|
})();
|
|
612
617
|
return cpuReleased;
|
|
613
618
|
};
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
/** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
|
|
624
|
-
let confirmedAdmissionAt: number | null = null;
|
|
625
|
-
// A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
|
|
626
|
-
const queueWait = (): AdmissionQueueWait | null =>
|
|
627
|
-
mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
|
|
628
|
-
? null
|
|
629
|
-
: {
|
|
630
|
-
waitedMs: admissionWaitMs(admissionWait, performance.now()),
|
|
631
|
-
requests: lastQueueRequests,
|
|
632
|
-
};
|
|
633
|
-
const queueAbort = new AbortController();
|
|
634
|
-
let queueTimer: NodeJS.Timeout | undefined;
|
|
635
|
-
let queueSnapshot: Promise<void> | null = null;
|
|
636
|
-
const stopQueueObservation = (): void => {
|
|
637
|
-
admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
|
|
638
|
-
clearInterval(queueTimer);
|
|
639
|
-
queueAbort.abort();
|
|
640
|
-
};
|
|
619
|
+
const queue = new AdmissionObservation({
|
|
620
|
+
over: () => state.settled || state.expiry !== null,
|
|
621
|
+
pid: () => child.pid,
|
|
622
|
+
environment: () => mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
623
|
+
tell,
|
|
624
|
+
step: (event) => stepTo(event),
|
|
625
|
+
testing: () => isTesting(),
|
|
626
|
+
queueHost,
|
|
627
|
+
});
|
|
641
628
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
642
629
|
// Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
|
|
643
630
|
const recordRun = async (
|
|
@@ -671,7 +658,7 @@ export async function runBoundedExec(
|
|
|
671
658
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
672
659
|
}
|
|
673
660
|
});
|
|
674
|
-
const admissionQueue = queueWait();
|
|
661
|
+
const admissionQueue = queue.queueWait(reservation !== null);
|
|
675
662
|
const verdict =
|
|
676
663
|
operational || resultFailure !== null
|
|
677
664
|
? judgeOperationalFailure(exitCode, report)
|
|
@@ -960,7 +947,7 @@ export async function runBoundedExec(
|
|
|
960
947
|
try {
|
|
961
948
|
// Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
|
|
962
949
|
// ceilings remain armed while this observation runs.
|
|
963
|
-
await
|
|
950
|
+
await queue.refresh(true);
|
|
964
951
|
} finally {
|
|
965
952
|
checkingIdle = false;
|
|
966
953
|
}
|
|
@@ -990,74 +977,6 @@ export async function runBoundedExec(
|
|
|
990
977
|
armBoundTimer();
|
|
991
978
|
};
|
|
992
979
|
|
|
993
|
-
let lastQueueDiagnostic = '';
|
|
994
|
-
let lastQueueDiagnosticAt: number | null = null;
|
|
995
|
-
let lastSnapshotDiagnostics = '';
|
|
996
|
-
const unconfirmAdmission = (detail: string): void => {
|
|
997
|
-
if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
|
|
998
|
-
if (unconfirmedAdmission !== detail) {
|
|
999
|
-
tell(
|
|
1000
|
-
`bounded-exec: mbx admission snapshot failed: ${detail}; admission is unconfirmed, so this wait earns no idle credit until a snapshot succeeds\n`,
|
|
1001
|
-
);
|
|
1002
|
-
}
|
|
1003
|
-
unconfirmedAdmission = detail;
|
|
1004
|
-
lastQueueRequests = [];
|
|
1005
|
-
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: false });
|
|
1006
|
-
// The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
|
|
1007
|
-
stepTo({ kind: 'admission-queue', at: confirmedAdmissionAt ?? Date.now(), waiting: false, testing: isTesting() });
|
|
1008
|
-
};
|
|
1009
|
-
const refreshQueue = async (fresh: boolean): Promise<void> => {
|
|
1010
|
-
if (fresh && queueSnapshot !== null) await queueSnapshot;
|
|
1011
|
-
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined) return;
|
|
1012
|
-
const pid = child.pid;
|
|
1013
|
-
queueSnapshot ??= (async () => {
|
|
1014
|
-
const snapshot = await snapshotMbxQueue(
|
|
1015
|
-
pid,
|
|
1016
|
-
mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
1017
|
-
queueAbort.signal,
|
|
1018
|
-
queueHost,
|
|
1019
|
-
);
|
|
1020
|
-
if (state.settled || state.expiry !== null || queueAbort.signal.aborted) return;
|
|
1021
|
-
if (!snapshot.ok) {
|
|
1022
|
-
unconfirmAdmission(snapshot.detail);
|
|
1023
|
-
return;
|
|
1024
|
-
}
|
|
1025
|
-
const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
|
|
1026
|
-
if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics) tell(`bounded-exec: ${diagnostics}\n`);
|
|
1027
|
-
lastSnapshotDiagnostics = diagnostics;
|
|
1028
|
-
const waiting = snapshot.requests.length > 0;
|
|
1029
|
-
const observedAt = Date.now();
|
|
1030
|
-
const observedWaitAt = performance.now();
|
|
1031
|
-
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
|
|
1032
|
-
unconfirmedAdmission = null;
|
|
1033
|
-
confirmedAdmissionAt = observedAt;
|
|
1034
|
-
if (waiting) {
|
|
1035
|
-
lastQueueRequests = snapshot.requests;
|
|
1036
|
-
const positions = JSON.stringify(snapshot.requests);
|
|
1037
|
-
if (
|
|
1038
|
-
positions !== lastQueueDiagnostic ||
|
|
1039
|
-
lastQueueDiagnosticAt === null ||
|
|
1040
|
-
Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS
|
|
1041
|
-
) {
|
|
1042
|
-
const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
|
|
1043
|
-
tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
|
|
1044
|
-
lastQueueDiagnostic = positions;
|
|
1045
|
-
lastQueueDiagnosticAt = Date.now();
|
|
1046
|
-
}
|
|
1047
|
-
} else {
|
|
1048
|
-
lastQueueDiagnostic = '';
|
|
1049
|
-
}
|
|
1050
|
-
stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
|
|
1051
|
-
})()
|
|
1052
|
-
.catch((error: unknown) => {
|
|
1053
|
-
unconfirmAdmission(error instanceof Error ? error.message : String(error));
|
|
1054
|
-
})
|
|
1055
|
-
.finally(() => {
|
|
1056
|
-
queueSnapshot = null;
|
|
1057
|
-
});
|
|
1058
|
-
await queueSnapshot;
|
|
1059
|
-
};
|
|
1060
|
-
|
|
1061
980
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
1062
981
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
1063
982
|
// message extend the very bound that just fired.
|
|
@@ -1131,6 +1050,7 @@ export async function runBoundedExec(
|
|
|
1131
1050
|
return;
|
|
1132
1051
|
}
|
|
1133
1052
|
try {
|
|
1053
|
+
state.sentSignal = true;
|
|
1134
1054
|
await killer.kill(pid, 'SIGTERM');
|
|
1135
1055
|
} catch (error) {
|
|
1136
1056
|
const reason = error instanceof Error ? error.message : String(error);
|
|
@@ -1148,9 +1068,9 @@ export async function runBoundedExec(
|
|
|
1148
1068
|
clearTimeout(boundTimer);
|
|
1149
1069
|
// Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
|
|
1150
1070
|
// leaves the admission unconfirmed, never a manufactured queue or admission.
|
|
1151
|
-
signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() :
|
|
1071
|
+
signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : queue.refresh(false))
|
|
1152
1072
|
.finally(() => {
|
|
1153
|
-
|
|
1073
|
+
queue.stop();
|
|
1154
1074
|
return killChildTree();
|
|
1155
1075
|
})
|
|
1156
1076
|
.finally(releaseCpu)
|
|
@@ -1192,12 +1112,12 @@ export async function runBoundedExec(
|
|
|
1192
1112
|
child.once('error', (error) => {
|
|
1193
1113
|
appendStderr(`${error.message}\n`);
|
|
1194
1114
|
state.settled = true;
|
|
1195
|
-
|
|
1115
|
+
queue.stop();
|
|
1196
1116
|
resolve({ code: 1, signal: null });
|
|
1197
1117
|
});
|
|
1198
1118
|
child.once(hooked === null ? 'exit' : 'close', (code: number | null, signal: NodeJS.Signals | null) => {
|
|
1199
1119
|
state.settled = true;
|
|
1200
|
-
|
|
1120
|
+
queue.stop();
|
|
1201
1121
|
resolve({ code, signal });
|
|
1202
1122
|
});
|
|
1203
1123
|
});
|
|
@@ -1221,9 +1141,9 @@ export async function runBoundedExec(
|
|
|
1221
1141
|
? `still running: ${bound.tests.join(', ')}`
|
|
1222
1142
|
: 'the runner named no test still running'
|
|
1223
1143
|
} (cwd=${cwd}): ${command}\n${
|
|
1224
|
-
|
|
1144
|
+
queue.unconfirmed === null
|
|
1225
1145
|
? ''
|
|
1226
|
-
: `The last mbx admission snapshot failed (${
|
|
1146
|
+
: `The last mbx admission snapshot failed (${queue.unconfirmed}), so the wait it could not confirm earned no idle credit\n`
|
|
1227
1147
|
}`
|
|
1228
1148
|
: bound.kind === 'fixture'
|
|
1229
1149
|
? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${segment ?? command}\n`
|
|
@@ -1280,6 +1200,7 @@ export async function runBoundedExec(
|
|
|
1280
1200
|
// cannot end is said, and a failure of the probe or of a signal is shown.
|
|
1281
1201
|
const forceKill = async (pid: number): Promise<void> => {
|
|
1282
1202
|
try {
|
|
1203
|
+
state.sentSignal = true;
|
|
1283
1204
|
if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
|
|
1284
1205
|
tell(`${describeMembersLeft(pid)}\n`);
|
|
1285
1206
|
}
|
|
@@ -1359,7 +1280,7 @@ export async function runBoundedExec(
|
|
|
1359
1280
|
channel.end();
|
|
1360
1281
|
return;
|
|
1361
1282
|
}
|
|
1362
|
-
|
|
1283
|
+
queue.waiting();
|
|
1363
1284
|
tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
|
|
1364
1285
|
const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
|
|
1365
1286
|
reservation = held;
|
|
@@ -1371,7 +1292,7 @@ export async function runBoundedExec(
|
|
|
1371
1292
|
}
|
|
1372
1293
|
// A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
|
|
1373
1294
|
if (state.settled || state.expiry !== null) return;
|
|
1374
|
-
|
|
1295
|
+
queue.admitted = true;
|
|
1375
1296
|
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1376
1297
|
channel.end(`${granted.cpus}\n`);
|
|
1377
1298
|
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
@@ -1390,7 +1311,7 @@ export async function runBoundedExec(
|
|
|
1390
1311
|
for (const byte of chunk) {
|
|
1391
1312
|
if (byte === 103) {
|
|
1392
1313
|
if (!said) {
|
|
1393
|
-
|
|
1314
|
+
queue.admitted = true;
|
|
1394
1315
|
said = true;
|
|
1395
1316
|
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1396
1317
|
}
|
|
@@ -1404,10 +1325,7 @@ export async function runBoundedExec(
|
|
|
1404
1325
|
}
|
|
1405
1326
|
});
|
|
1406
1327
|
// Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
|
|
1407
|
-
|
|
1408
|
-
void refreshQueue(false);
|
|
1409
|
-
}, MBX_QUEUE_POLL_MS);
|
|
1410
|
-
void refreshQueue(false);
|
|
1328
|
+
queue.poll();
|
|
1411
1329
|
}
|
|
1412
1330
|
}
|
|
1413
1331
|
|
|
@@ -1432,8 +1350,8 @@ export async function runBoundedExec(
|
|
|
1432
1350
|
await signalCleanup;
|
|
1433
1351
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
1434
1352
|
}
|
|
1435
|
-
|
|
1436
|
-
await
|
|
1353
|
+
queue.stop();
|
|
1354
|
+
await queue.idle();
|
|
1437
1355
|
await termination;
|
|
1438
1356
|
clearTimeout(boundTimer);
|
|
1439
1357
|
cancelCapture();
|
|
@@ -1488,9 +1406,16 @@ export async function runBoundedExec(
|
|
|
1488
1406
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1489
1407
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1490
1408
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1409
|
+
const killedForMemory = killedFromOutside({
|
|
1410
|
+
sentSignal: state.sentSignal,
|
|
1411
|
+
bounded: state.expiry !== null,
|
|
1412
|
+
exitSignal,
|
|
1413
|
+
code,
|
|
1414
|
+
});
|
|
1491
1415
|
return {
|
|
1492
1416
|
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1493
1417
|
terminalOutput: outputChunks.join(''),
|
|
1418
|
+
...(killedForMemory ? { killedForMemory: true } : {}),
|
|
1494
1419
|
};
|
|
1495
1420
|
}
|
|
1496
1421
|
|
|
@@ -132,4 +132,10 @@ async function treeResidentBytes(root: number): Promise<number> {
|
|
|
132
132
|
return total;
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
-
export const darwinMemory: HostMemory = {
|
|
135
|
+
export const darwinMemory: HostMemory = {
|
|
136
|
+
readRunMemory,
|
|
137
|
+
peakBytes: (maxRss) => maxRss,
|
|
138
|
+
treeResidentBytes,
|
|
139
|
+
// macOS has no cgroup, and its anonymous and shared pages are not read from a file the pool may open.
|
|
140
|
+
liveUsage: () => ({ ok: false, reason: 'macOS keeps no anonymous and shared memory of a run to read' }),
|
|
141
|
+
};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
1
2
|
import { readFile } from 'node:fs/promises';
|
|
2
3
|
import { posix } from 'node:path';
|
|
3
4
|
import { childPids } from '../../process-children.linux.js';
|
|
4
|
-
import type { HostMemory, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
|
|
5
|
+
import type { HostMemory, LiveUsageReading, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* Linux's memory mechanisms. The run's memory is the least of the host's `MemTotal` and every finite `memory.max` of
|
|
@@ -103,6 +104,40 @@ async function readText(path: string): Promise<FileRead> {
|
|
|
103
104
|
}
|
|
104
105
|
}
|
|
105
106
|
|
|
107
|
+
/** The memory a bound holds live: `memory.stat`'s `anon` and `shmem` beside a cgroup's `memory.max`, meminfo's `AnonPages` and `Shmem` for the host's. */
|
|
108
|
+
export function linuxLiveUsage(read: (path: string) => FileRead, bound: string): LiveUsageReading {
|
|
109
|
+
const cgroupLimit = '/memory.max';
|
|
110
|
+
if (bound.endsWith(cgroupLimit)) {
|
|
111
|
+
const file = `${bound.slice(0, -cgroupLimit.length)}/memory.stat`;
|
|
112
|
+
const stat = read(file);
|
|
113
|
+
if (!stat.ok) return { ok: false, reason: `${file}: ${stat.code}: ${stat.message}` };
|
|
114
|
+
const [anon, shmem] = [/^anon (\d+)$/m.exec(stat.text)?.[1], /^shmem (\d+)$/m.exec(stat.text)?.[1]];
|
|
115
|
+
return anon === undefined || shmem === undefined
|
|
116
|
+
? { ok: false, reason: `${file} names no anon and shmem in bytes` }
|
|
117
|
+
: { ok: true, usage: { anon: Number(anon), shmem: Number(shmem), source: file } };
|
|
118
|
+
}
|
|
119
|
+
if (bound === `${MEMINFO} MemTotal`) {
|
|
120
|
+
const info = read(MEMINFO);
|
|
121
|
+
if (!info.ok) return { ok: false, reason: `${MEMINFO}: ${info.code}: ${info.message}` };
|
|
122
|
+
const [anon, shmem] = [
|
|
123
|
+
/^AnonPages:\s+(\d+) kB$/m.exec(info.text)?.[1],
|
|
124
|
+
/^Shmem:\s+(\d+) kB$/m.exec(info.text)?.[1],
|
|
125
|
+
];
|
|
126
|
+
return anon === undefined || shmem === undefined
|
|
127
|
+
? { ok: false, reason: `${MEMINFO} names no AnonPages and Shmem in kB` }
|
|
128
|
+
: { ok: true, usage: { anon: Number(anon) * 1024, shmem: Number(shmem) * 1024, source: MEMINFO } };
|
|
129
|
+
}
|
|
130
|
+
return { ok: false, reason: `${bound} is no cgroup memory.max or meminfo bound` };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function readTextSync(path: string): FileRead {
|
|
134
|
+
try {
|
|
135
|
+
return { ok: true, text: readFileSync(path, 'utf8') };
|
|
136
|
+
} catch (error) {
|
|
137
|
+
const code = error instanceof Error && 'code' in error && typeof error.code === 'string' ? error.code : 'EIO';
|
|
138
|
+
return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
|
|
139
|
+
}
|
|
140
|
+
}
|
|
106
141
|
/** Bun before 1.4.0 reports Linux's `ru_maxrss` as the kernel does, in KiB; from 1.4.0 it reports bytes. */
|
|
107
142
|
function peakBytes(maxRss: number, bunVersion: string): number {
|
|
108
143
|
const version = /^(\d+)\.(\d+)\./.exec(bunVersion);
|
|
@@ -144,4 +179,9 @@ async function residentBytes(pid: number): Promise<number> {
|
|
|
144
179
|
return kib === undefined ? 0 : Number(kib) * 1024;
|
|
145
180
|
}
|
|
146
181
|
|
|
147
|
-
export const linuxMemory: HostMemory = {
|
|
182
|
+
export const linuxMemory: HostMemory = {
|
|
183
|
+
readRunMemory: () => linuxRunMemory(readText),
|
|
184
|
+
peakBytes,
|
|
185
|
+
treeResidentBytes,
|
|
186
|
+
liveUsage: (bound) => linuxLiveUsage(readTextSync, bound),
|
|
187
|
+
};
|
|
@@ -21,8 +21,26 @@ export interface HostMemory {
|
|
|
21
21
|
* listing and its read adds nothing, and a root that has exited holds nothing.
|
|
22
22
|
*/
|
|
23
23
|
treeResidentBytes(root: number): Promise<number>;
|
|
24
|
+
/**
|
|
25
|
+
* What the run's bound holds that the kernel cannot take back, read now: its anonymous and shared memory. `bound` is
|
|
26
|
+
* the `source` of the run's memory (`RunMemory`): the cgroup's `memory.max`, whose `memory.stat` is read, or the
|
|
27
|
+
* host's `MemTotal`, whose `AnonPages` and `Shmem` are. The pool says it on every grant, so what the leaves are not
|
|
28
|
+
* charged for (the baseline, a tmpfs) and what a suite holds beyond its record are measured, not guessed.
|
|
29
|
+
*/
|
|
30
|
+
liveUsage(bound: string): LiveUsageReading;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** What a bound holds live, in bytes, and the file it was read from. */
|
|
34
|
+
export interface LiveUsage {
|
|
35
|
+
readonly anon: number;
|
|
36
|
+
readonly shmem: number;
|
|
37
|
+
readonly source: string;
|
|
24
38
|
}
|
|
25
39
|
|
|
40
|
+
export type LiveUsageReading =
|
|
41
|
+
| { readonly ok: true; readonly usage: LiveUsage }
|
|
42
|
+
| { readonly ok: false; readonly reason: string };
|
|
43
|
+
|
|
26
44
|
/** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
|
|
27
45
|
export interface RunMemory {
|
|
28
46
|
readonly bytes: number;
|
|
@@ -75,3 +93,10 @@ export function readRunMemory(): Promise<RunMemoryReading> {
|
|
|
75
93
|
}
|
|
76
94
|
return HOST_MEMORY.readRunMemory();
|
|
77
95
|
}
|
|
96
|
+
|
|
97
|
+
/** The run's bound live on this platform, or why it cannot be read: a pool says it, never leaves it out. */
|
|
98
|
+
export function readLiveUsage(bound: string): LiveUsageReading {
|
|
99
|
+
return HOST_MEMORY === null
|
|
100
|
+
? { ok: false, reason: `${process.platform} has no reader of live memory` }
|
|
101
|
+
: HOST_MEMORY.liveUsage(bound);
|
|
102
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A run again after the kernel killed the command for memory (specs/cowshed/05_gateway.md, "Host scheduling"). Only
|
|
3
|
+
* when a pool lent the run its tokens and a SIGKILL that no bound of this executor and no signal it sent caused ended
|
|
4
|
+
* it: the command runs once more, alone in its run's pool, so what it was killed beside is not there to be killed with
|
|
5
|
+
* it. This decides whether a command runs a second time and how its two outputs join, never what it runs: the key
|
|
6
|
+
* leaves it out (README, "Bounded exec key").
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** How one run of the command ended. */
|
|
10
|
+
export interface RunOutcome {
|
|
11
|
+
success: boolean;
|
|
12
|
+
terminalOutput: string;
|
|
13
|
+
/**
|
|
14
|
+
* The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
|
|
15
|
+
* which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
|
|
16
|
+
*/
|
|
17
|
+
killedForMemory?: boolean;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** What a run read of how its command ended and what this executor did to it. */
|
|
21
|
+
export interface Ending {
|
|
22
|
+
/** This executor sent the command's process group a signal. */
|
|
23
|
+
readonly sentSignal: boolean;
|
|
24
|
+
/** A bound of this executor ended the run. */
|
|
25
|
+
readonly bounded: boolean;
|
|
26
|
+
readonly exitSignal: NodeJS.Signals | null;
|
|
27
|
+
/** The exit status, or the status a signal folds to. */
|
|
28
|
+
readonly code: number;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
|
|
33
|
+
* run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who sent
|
|
34
|
+
* it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
|
|
35
|
+
*/
|
|
36
|
+
export function killedFromOutside(ending: Ending): boolean {
|
|
37
|
+
return !ending.sentSignal && !ending.bounded && (ending.exitSignal === 'SIGKILL' || ending.code === 137);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Run the command; when the kernel killed it for memory and a pool lent it tokens, run it once more alone, and join the
|
|
42
|
+
* two runs' output with a note naming the task.
|
|
43
|
+
*/
|
|
44
|
+
export async function runUntilNotKilledForMemory(
|
|
45
|
+
run: (alone: boolean) => Promise<RunOutcome>,
|
|
46
|
+
pooled: boolean,
|
|
47
|
+
task: string | undefined,
|
|
48
|
+
): Promise<RunOutcome> {
|
|
49
|
+
const first = await run(false);
|
|
50
|
+
if (first.killedForMemory !== true || !pooled) {
|
|
51
|
+
return first;
|
|
52
|
+
}
|
|
53
|
+
const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${
|
|
54
|
+
task === undefined ? '' : `: ${task}`
|
|
55
|
+
}); running it once more, alone in its run's pool\n`;
|
|
56
|
+
process.stderr.write(note);
|
|
57
|
+
const again = await run(true);
|
|
58
|
+
return {
|
|
59
|
+
success: again.success,
|
|
60
|
+
terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
|
|
61
|
+
...(again.killedForMemory === true ? { killedForMemory: true } : {}),
|
|
62
|
+
};
|
|
63
|
+
}
|