@smoothbricks/nx-plugin 0.7.11 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -25
- package/dist/.build-inputs.json +25 -17
- package/dist/built-output-packages.d.ts +27 -0
- package/dist/built-output-packages.d.ts.map +1 -0
- package/dist/built-output-packages.js +97 -0
- package/dist/cross-check-policy.d.ts +13 -1
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +13 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +68 -0
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -0
- package/dist/executors/bounded-exec/admission-queue.js +131 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts +14 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-grant.js +18 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +26 -24
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +88 -72
- package/dist/executors/bounded-exec/executor.d.ts +5 -7
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +69 -133
- package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
- package/dist/executors/bounded-exec/host-memory.js +6 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
- package/dist/executors/bounded-exec/memory-retry.d.ts +39 -0
- package/dist/executors/bounded-exec/memory-retry.d.ts.map +1 -0
- package/dist/executors/bounded-exec/memory-retry.js +33 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts +43 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/parent-grant.js +37 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts +17 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-charge.js +20 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +44 -8
- package/dist/executors/bounded-exec/pool-environment.d.ts +25 -0
- package/dist/executors/bounded-exec/pool-environment.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-environment.js +34 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +5 -18
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +12 -16
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -21
- package/dist/managed-files/ci-workflow.d.ts +29 -5
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +111 -16
- package/dist/managed-files/context.d.ts.map +1 -1
- package/dist/managed-files/context.js +1 -0
- package/dist/managed-files/files.d.ts +3 -1
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +2 -0
- package/dist/managed-files/publish-workflow.d.ts +6 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/managed-files/publish-workflow.js +16 -11
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +48 -14
- package/dist/typecheck-route.d.ts +28 -0
- package/dist/typecheck-route.d.ts.map +1 -0
- package/dist/typecheck-route.js +109 -0
- package/dist/workspace-manifest.d.ts +50 -5
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/executors.json +10 -0
- package/managed/raw/patches/nx@24.0.3.patch +16 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +93 -8
- package/managed/raw/tooling/direnv/apple-developer.sh +8 -1
- package/managed/raw/tooling/direnv/compiler-variables.sh +1 -1
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +4 -2
- package/managed/raw/tooling/direnv/secret-references.ts +19 -5
- package/managed/raw/tooling/git-hooks/pre-push.sh +20 -10
- package/managed/templates/github/actions/forgejo-artifact.cjs +9 -9
- package/managed/templates/github/actions/setup-devenv/action.yml +53 -0
- package/package.json +2 -2
- package/src/built-output-packages.ts +125 -0
- package/src/cross-check-policy.ts +14 -1
- package/src/executors/bounded-exec/admission-queue.ts +163 -0
- package/src/executors/bounded-exec/cpu-grant.ts +32 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
- package/src/executors/bounded-exec/cpu-pool.ts +123 -58
- package/src/executors/bounded-exec/executor.ts +93 -168
- package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
- package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
- package/src/executors/bounded-exec/host-memory.ts +25 -0
- package/src/executors/bounded-exec/memory-retry.ts +63 -0
- package/src/executors/bounded-exec/parent-grant.ts +75 -0
- package/src/executors/bounded-exec/peak-charge.ts +25 -0
- package/src/executors/bounded-exec/pool-admission.ts +66 -11
- package/src/executors/bounded-exec/pool-environment.ts +45 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +5 -4
- package/src/executors/bounded-exec/verdict.ts +17 -25
- package/src/index.ts +70 -21
- package/src/managed-files/ci-workflow.ts +143 -16
- package/src/managed-files/context.ts +1 -0
- package/src/managed-files/files.ts +5 -0
- package/src/managed-files/publish-workflow.ts +26 -10
- package/src/owned-installation.ts +63 -14
- package/src/typecheck-route.ts +64 -0
- package/src/workspace-manifest.ts +56 -5
- package/test-timings.json +141 -137
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"executor.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/executor.ts"],"names":[],"mappings":"AAeA,OAAO,EAAwB,KAAK,SAAS,EAAE,MAAM,sBAAsB,CAAC;AAK5E,OAAO,EAAE,sBAAsB,EAAmB,MAAM,uBAAuB,CAAC;AAahF,OAAO,EAAqB,KAAK,UAAU,EAA8B,MAAM,mBAAmB,CAAC;AAEnG,OAAO,EAOL,KAAK,YAAY,EAClB,MAAM,oBAAoB,CAAC;AAG5B,OAAO,EAAgB,KAAK,gBAAgB,EAAwB,MAAM,uBAAuB,CAAC;AAElG,OAAO,EASL,aAAa,EAEd,MAAM,eAAe,CAAC;AAIvB,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,OAAO,EAEL,KAAK,eAAe,EAcrB,MAAM,cAAc,CAAC;AAiBtB,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,wFAAwF;IACxF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,eAAe,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACtD,uFAAuF;IACvF,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAED,qEAAqE;AACrE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,SAAS,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,IAAI,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;CAC1D;AAED,MAAM,MAAM,iBAAiB,GAAG,UAAU,CAAC;AAkB3C,MAAM,WAAW,iBAAiB;IAChC,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1D;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG,IAAI,CAC3B,aAAa,EACb,SAAS,GAAG,SAAS,GAAG,UAAU,GAAG,OAAO,GAAG,QAAQ,GAAG,gBAAgB,CAC3E,CAAC;AAQF;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACtC,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,SAAS,EAAE,MAAM,GAAG,SAAS,EAC7B,GAAG,EAAE,MAAM,EACX,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAOzB,GACA,MAAM,GAAG,SAAS,CAiBpB;AAED,MAAM,CAAC,OAAO,UAAU,mBAAmB,CACzC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,iBAAiB,CAAC,CAE5B;AAED;;;GAGG;AACH,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CAc5B;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,KAAK,UAAU,GAAG,IAAI,GACzD,OAAO,CAAC,iBAAiB,CAAC,CAsB5B;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,aAAa,GAAG,kBAAkB,CAiBxE;AAED;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,kBAAkB,EAC3B,OAAO,EAAE,kBAAkB,EAC3B,MAAM,EAAE,iBAAiB,EACzB,UAAU,EAAE,UAAU,GAAG,IAAI,EAC7B,SAAS,GAAE,gBAAgB,GAAG,IAAW,EACzC,kBAAkB,GAAE,OAAO,sBAA+C,EAC1E,KAAK,GAAE,YAAY,GAAG,IAAW,EACjC,SAAS,CAAC,EAAE,SAAS,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAe5B;AAylCD,wBAAgB,uBAAuB,IAAI,iBAAiB,CAM3D"}
|
|
@@ -7,23 +7,27 @@ import { createInterface } from 'node:readline';
|
|
|
7
7
|
import { Duplex, Readable } from 'node:stream';
|
|
8
8
|
import { workspaceDataDirectoryForWorkspace } from 'nx/src/utils/cache-directory.js';
|
|
9
9
|
import { DEADLINE_PATIENCE, describeMembersLeft, killProcessGroup, processGroupHasMembers, signalProcessGroup, } from '../../process-group.js';
|
|
10
|
-
import {
|
|
11
|
-
import {
|
|
10
|
+
import { AdmissionObservation } from './admission-queue.js';
|
|
11
|
+
import { earliestBound, stepBounds, UNARMED } from './bounds.js';
|
|
12
|
+
import { createCpuPool } from './cpu-pool.js';
|
|
12
13
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
13
14
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
14
15
|
import { PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
15
16
|
import { askedCores, MBX_GRANT_FD, mbxIdleGated, mbxLeaf, mbxPool, withCountedReservation, withMbxLeaf, } from './mbx-pool.js';
|
|
16
|
-
import { MBX_QUEUE_POLL_MS, snapshotMbxQueue } from './mbx-queue.js';
|
|
17
17
|
import { describeUnserved, RESERVATIONS_ENV, reserveCores, runEndpoint } from './mbx-reservations.js';
|
|
18
|
+
import { killedFromOutside, runUntilNotKilledForMemory } from './memory-retry.js';
|
|
18
19
|
import { createNoSpaceWatch } from './no-space.js';
|
|
19
20
|
import { COWSHED_EXECUTABLE_ENV, cowshedSubtreeOwner, HANDLE_FD, readAdmission, SUPERVISOR_SOCKET_ENV, } from './owned-subtree.js';
|
|
21
|
+
import { askParentPool, describeGrant } from './parent-grant.js';
|
|
22
|
+
import { readLastPeak } from './peak-charge.js';
|
|
23
|
+
import { CPU_POOL_ENV, readInheritedCpuPool } from './pool-environment.js';
|
|
20
24
|
import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
|
|
21
25
|
import { createHostCommands, describeCoverageGap, describeRamTempError, describeSignalled, LEFTOVER_GRACE_MS, PRODUCTION, RAM_TEMP_CAPACITY_BYTES, RamTempVolume, ramTempPaths, } from './ram-temp.js';
|
|
22
26
|
import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
|
|
23
27
|
import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
|
|
24
28
|
import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
|
|
25
29
|
import { createTestResultWriter } from './test-result-writer.js';
|
|
26
|
-
import { BUN_TEST, clearTaskDirectory,
|
|
30
|
+
import { BUN_TEST, clearTaskDirectory, describeNoSpace, JUNIT_FILE, JUNIT_REPORT_ENV, judgeOperationalFailure, judgeRun, readJunitReport, taskDirectory, withJunitReport, writeRecord, } from './verdict.js';
|
|
27
31
|
const DEFAULT_KILL_AFTER_MS = 10_000;
|
|
28
32
|
const REAP_AFTER_FORCE_KILL_MS = 2_000;
|
|
29
33
|
/** The pool descriptor: only a grant may give it, so neither the target's nor the inherited environment passes it on. */
|
|
@@ -142,10 +146,21 @@ export function recordContext(context) {
|
|
|
142
146
|
* command runs on over budget to its own exit, and the run says so.
|
|
143
147
|
*/
|
|
144
148
|
export async function runBoundedExec(options, context, killer, tempVolume, inherited = null, captureDiagnostics = captureHangDiagnostics, owner = null, queueHost) {
|
|
145
|
-
const cwd = resolveCwd(options.cwd, context.root);
|
|
146
149
|
const directory = context.record?.directory ?? null;
|
|
147
|
-
// The task's last recorded peak
|
|
150
|
+
// The task's last recorded peak, and the tokens it was lent when the peak was measured, are its charge in its parent's
|
|
151
|
+
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
152
|
+
// killed run replaced, not by the peak it reached before it was killed.
|
|
148
153
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
154
|
+
return runUntilNotKilledForMemory((alone) => runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
155
|
+
lastPeak,
|
|
156
|
+
alone,
|
|
157
|
+
}), inherited !== null, context.record?.task.id);
|
|
158
|
+
}
|
|
159
|
+
/** One run of the command, from the pool's grant to its record. */
|
|
160
|
+
async function runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, attempt) {
|
|
161
|
+
const cwd = resolveCwd(options.cwd, context.root);
|
|
162
|
+
const directory = context.record?.directory ?? null;
|
|
163
|
+
const { lastPeak } = attempt;
|
|
149
164
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
150
165
|
if (directory !== null) {
|
|
151
166
|
await clearTaskDirectory(directory);
|
|
@@ -164,6 +179,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
164
179
|
settled: false,
|
|
165
180
|
expiry: null,
|
|
166
181
|
forceKillNeeded: false,
|
|
182
|
+
sentSignal: false,
|
|
167
183
|
poolCloseError: null,
|
|
168
184
|
retireError: null,
|
|
169
185
|
noSpace: null,
|
|
@@ -230,8 +246,10 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
230
246
|
if (lastPeak !== null && !lastPeak.ok) {
|
|
231
247
|
return refuseUnstarted(`bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`);
|
|
232
248
|
}
|
|
233
|
-
// A task with no recorded peak is charged nothing (null) until it has one.
|
|
249
|
+
// A task with no recorded peak is charged nothing (null) until it has one. A peak is charged in proportion to the
|
|
250
|
+
// tokens the run is lent when that is more than the run that measured it was lent.
|
|
234
251
|
const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
|
|
252
|
+
const chargeParallelism = lastPeak?.ok === true ? lastPeak.parallelism : null;
|
|
235
253
|
let lease = null;
|
|
236
254
|
let ordinaryTemp = null;
|
|
237
255
|
// Before spawn there is nothing to retire. Once started, only the owner's terminal retirement permits deletion.
|
|
@@ -367,22 +385,16 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
367
385
|
const asked = Date.now();
|
|
368
386
|
// A pool that answered and refused is a live budget, and running beside it would count the run twice: the run fails
|
|
369
387
|
// instead. A pool whose endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
}
|
|
381
|
-
else {
|
|
382
|
-
parentRefusal = fromParent.reason;
|
|
383
|
-
parentWait = fromParent.wait;
|
|
384
|
-
}
|
|
385
|
-
}
|
|
388
|
+
const parent = await askParentPool(inherited, {
|
|
389
|
+
want: demand.want,
|
|
390
|
+
charge,
|
|
391
|
+
parallelism: chargeParallelism,
|
|
392
|
+
alone: attempt.alone,
|
|
393
|
+
task: context.record?.task.id,
|
|
394
|
+
});
|
|
395
|
+
for (const said of parent.said)
|
|
396
|
+
appendStderr(said);
|
|
397
|
+
const { grant, refusal: parentRefusal, wait: parentWait } = parent;
|
|
386
398
|
const waitedMs = Date.now() - asked;
|
|
387
399
|
let cpu;
|
|
388
400
|
let sized = { command: requested, env: {} };
|
|
@@ -393,9 +405,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
393
405
|
if (grant !== null) {
|
|
394
406
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
395
407
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
appendStderr(`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`);
|
|
408
|
+
for (const said of describeGrant(grant, { waitedMs, want: demand.want, runner: demand.kind, charge })) {
|
|
409
|
+
appendStderr(said);
|
|
399
410
|
}
|
|
400
411
|
}
|
|
401
412
|
else if (leaf?.kind === 'reserve') {
|
|
@@ -426,32 +437,15 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
426
437
|
})();
|
|
427
438
|
return cpuReleased;
|
|
428
439
|
};
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
/** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
|
|
439
|
-
let confirmedAdmissionAt = null;
|
|
440
|
-
// A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
|
|
441
|
-
const queueWait = () => mbxAdmitted || (lastQueueRequests.length === 0 && reservation === null)
|
|
442
|
-
? null
|
|
443
|
-
: {
|
|
444
|
-
waitedMs: admissionWaitMs(admissionWait, performance.now()),
|
|
445
|
-
requests: lastQueueRequests,
|
|
446
|
-
};
|
|
447
|
-
const queueAbort = new AbortController();
|
|
448
|
-
let queueTimer;
|
|
449
|
-
let queueSnapshot = null;
|
|
450
|
-
const stopQueueObservation = () => {
|
|
451
|
-
admissionWait = stepAdmissionWait(admissionWait, { kind: 'stop', at: performance.now() });
|
|
452
|
-
clearInterval(queueTimer);
|
|
453
|
-
queueAbort.abort();
|
|
454
|
-
};
|
|
440
|
+
const queue = new AdmissionObservation({
|
|
441
|
+
over: () => state.settled || state.expiry !== null,
|
|
442
|
+
pid: () => child.pid,
|
|
443
|
+
environment: () => mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
444
|
+
tell,
|
|
445
|
+
step: (event) => stepTo(event),
|
|
446
|
+
testing: () => isTesting(),
|
|
447
|
+
queueHost,
|
|
448
|
+
});
|
|
455
449
|
// Records the run's verdict when it has a task to record as. An `expiry` is set only when a bound ended the run.
|
|
456
450
|
// Observation and ownership failures remain operational even after valid queue evidence; cancellation alone does not.
|
|
457
451
|
const recordRun = async (exitCode, expiry, elapsedMs, operational, noSpace, lockWaits, peakBytes) => {
|
|
@@ -474,7 +468,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
474
468
|
appendStderr(`bounded-exec: ${error}\n`);
|
|
475
469
|
}
|
|
476
470
|
});
|
|
477
|
-
const admissionQueue = queueWait();
|
|
471
|
+
const admissionQueue = queue.queueWait(reservation !== null);
|
|
478
472
|
const verdict = operational || resultFailure !== null
|
|
479
473
|
? judgeOperationalFailure(exitCode, report)
|
|
480
474
|
: judgeRun({
|
|
@@ -745,7 +739,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
745
739
|
try {
|
|
746
740
|
// Drain an older poll, then take a fresh bounded snapshot before idle can end queued work. Total and fixture
|
|
747
741
|
// ceilings remain armed while this observation runs.
|
|
748
|
-
await
|
|
742
|
+
await queue.refresh(true);
|
|
749
743
|
}
|
|
750
744
|
finally {
|
|
751
745
|
checkingIdle = false;
|
|
@@ -776,70 +770,6 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
776
770
|
}
|
|
777
771
|
armBoundTimer();
|
|
778
772
|
};
|
|
779
|
-
let lastQueueDiagnostic = '';
|
|
780
|
-
let lastQueueDiagnosticAt = null;
|
|
781
|
-
let lastSnapshotDiagnostics = '';
|
|
782
|
-
const unconfirmAdmission = (detail) => {
|
|
783
|
-
if (state.settled || state.expiry !== null || queueAbort.signal.aborted)
|
|
784
|
-
return;
|
|
785
|
-
if (unconfirmedAdmission !== detail) {
|
|
786
|
-
tell(`bounded-exec: mbx admission snapshot failed: ${detail}; admission is unconfirmed, so this wait earns no idle credit until a snapshot succeeds\n`);
|
|
787
|
-
}
|
|
788
|
-
unconfirmedAdmission = detail;
|
|
789
|
-
lastQueueRequests = [];
|
|
790
|
-
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: performance.now(), waiting: false });
|
|
791
|
-
// The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
|
|
792
|
-
stepTo({ kind: 'admission-queue', at: confirmedAdmissionAt ?? Date.now(), waiting: false, testing: isTesting() });
|
|
793
|
-
};
|
|
794
|
-
const refreshQueue = async (fresh) => {
|
|
795
|
-
if (fresh && queueSnapshot !== null)
|
|
796
|
-
await queueSnapshot;
|
|
797
|
-
if (state.settled || state.expiry !== null || queueAbort.signal.aborted || child.pid === undefined)
|
|
798
|
-
return;
|
|
799
|
-
const pid = child.pid;
|
|
800
|
-
queueSnapshot ??= (async () => {
|
|
801
|
-
const snapshot = await snapshotMbxQueue(pid, mergeEnv(lease?.directory ?? ordinaryTemp, env), queueAbort.signal, queueHost);
|
|
802
|
-
if (state.settled || state.expiry !== null || queueAbort.signal.aborted)
|
|
803
|
-
return;
|
|
804
|
-
if (!snapshot.ok) {
|
|
805
|
-
unconfirmAdmission(snapshot.detail);
|
|
806
|
-
return;
|
|
807
|
-
}
|
|
808
|
-
const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
|
|
809
|
-
if (diagnostics !== '' && diagnostics !== lastSnapshotDiagnostics)
|
|
810
|
-
tell(`bounded-exec: ${diagnostics}\n`);
|
|
811
|
-
lastSnapshotDiagnostics = diagnostics;
|
|
812
|
-
const waiting = snapshot.requests.length > 0;
|
|
813
|
-
const observedAt = Date.now();
|
|
814
|
-
const observedWaitAt = performance.now();
|
|
815
|
-
admissionWait = stepAdmissionWait(admissionWait, { kind: 'snapshot', at: observedWaitAt, waiting });
|
|
816
|
-
unconfirmedAdmission = null;
|
|
817
|
-
confirmedAdmissionAt = observedAt;
|
|
818
|
-
if (waiting) {
|
|
819
|
-
lastQueueRequests = snapshot.requests;
|
|
820
|
-
const positions = JSON.stringify(snapshot.requests);
|
|
821
|
-
if (positions !== lastQueueDiagnostic ||
|
|
822
|
-
lastQueueDiagnosticAt === null ||
|
|
823
|
-
Date.now() - lastQueueDiagnosticAt >= HANG_CAPTURE_MS) {
|
|
824
|
-
const waitedMs = admissionWaitMs(admissionWait, observedWaitAt);
|
|
825
|
-
tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
|
|
826
|
-
lastQueueDiagnostic = positions;
|
|
827
|
-
lastQueueDiagnosticAt = Date.now();
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
else {
|
|
831
|
-
lastQueueDiagnostic = '';
|
|
832
|
-
}
|
|
833
|
-
stepTo({ kind: 'admission-queue', at: observedAt, waiting, testing: isTesting() });
|
|
834
|
-
})()
|
|
835
|
-
.catch((error) => {
|
|
836
|
-
unconfirmAdmission(error instanceof Error ? error.message : String(error));
|
|
837
|
-
})
|
|
838
|
-
.finally(() => {
|
|
839
|
-
queueSnapshot = null;
|
|
840
|
-
});
|
|
841
|
-
await queueSnapshot;
|
|
842
|
-
};
|
|
843
773
|
// Only the CHILD's own output is read for progress. The diagnostics below go
|
|
844
774
|
// through appendStderr as well, and re-arming on those would let a teardown
|
|
845
775
|
// message extend the very bound that just fired.
|
|
@@ -919,6 +849,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
919
849
|
return;
|
|
920
850
|
}
|
|
921
851
|
try {
|
|
852
|
+
state.sentSignal = true;
|
|
922
853
|
await killer.kill(pid, 'SIGTERM');
|
|
923
854
|
}
|
|
924
855
|
catch (error) {
|
|
@@ -936,9 +867,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
936
867
|
clearTimeout(boundTimer);
|
|
937
868
|
// Preserve explicit live membership before cancellation removes the queued process. A failed bounded observation
|
|
938
869
|
// leaves the admission unconfirmed, never a manufactured queue or admission.
|
|
939
|
-
signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() :
|
|
870
|
+
signalCleanup = (leaf?.kind !== 'cargo' ? Promise.resolve() : queue.refresh(false))
|
|
940
871
|
.finally(() => {
|
|
941
|
-
|
|
872
|
+
queue.stop();
|
|
942
873
|
return killChildTree();
|
|
943
874
|
})
|
|
944
875
|
.finally(releaseCpu)
|
|
@@ -968,12 +899,12 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
968
899
|
child.once('error', (error) => {
|
|
969
900
|
appendStderr(`${error.message}\n`);
|
|
970
901
|
state.settled = true;
|
|
971
|
-
|
|
902
|
+
queue.stop();
|
|
972
903
|
resolve({ code: 1, signal: null });
|
|
973
904
|
});
|
|
974
905
|
child.once(hooked === null ? 'exit' : 'close', (code, signal) => {
|
|
975
906
|
state.settled = true;
|
|
976
|
-
|
|
907
|
+
queue.stop();
|
|
977
908
|
resolve({ code, signal });
|
|
978
909
|
});
|
|
979
910
|
});
|
|
@@ -992,9 +923,9 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
992
923
|
appendStderr(bound.kind === 'idle'
|
|
993
924
|
? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${bound.tests.length > 0
|
|
994
925
|
? `still running: ${bound.tests.join(', ')}`
|
|
995
|
-
: 'the runner named no test still running'} (cwd=${cwd}): ${command}\n${
|
|
926
|
+
: 'the runner named no test still running'} (cwd=${cwd}): ${command}\n${queue.unconfirmed === null
|
|
996
927
|
? ''
|
|
997
|
-
: `The last mbx admission snapshot failed (${
|
|
928
|
+
: `The last mbx admission snapshot failed (${queue.unconfirmed}), so the wait it could not confirm earned no idle credit\n`}`
|
|
998
929
|
: bound.kind === 'fixture'
|
|
999
930
|
? `\nFixture command timed out after ${elapsedMs}ms (fixtureTimeoutMs=${bound.limitMs}, cwd=${cwd}): ${segment ?? command}\n`
|
|
1000
931
|
: `\nCommand timed out after ${elapsedMs}ms (timeoutMs=${bound.limitMs}, cwd=${cwd}): ${command}\n`);
|
|
@@ -1037,6 +968,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1037
968
|
// cannot end is said, and a failure of the probe or of a signal is shown.
|
|
1038
969
|
const forceKill = async (pid) => {
|
|
1039
970
|
try {
|
|
971
|
+
state.sentSignal = true;
|
|
1040
972
|
if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
|
|
1041
973
|
tell(`${describeMembersLeft(pid)}\n`);
|
|
1042
974
|
}
|
|
@@ -1116,7 +1048,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1116
1048
|
channel.end();
|
|
1117
1049
|
return;
|
|
1118
1050
|
}
|
|
1119
|
-
|
|
1051
|
+
queue.waiting();
|
|
1120
1052
|
tell(`cowshed: mbx reserve wait start cpus=${cpus} priority=${reservedLeaf.priority}\n`);
|
|
1121
1053
|
const held = reserveCores(endpoint, cpus, reservedLeaf.priority);
|
|
1122
1054
|
reservation = held;
|
|
@@ -1129,7 +1061,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1129
1061
|
// A run that ended while it waited gives the grant back with its CPU (`releaseCpu`).
|
|
1130
1062
|
if (state.settled || state.expiry !== null)
|
|
1131
1063
|
return;
|
|
1132
|
-
|
|
1064
|
+
queue.admitted = true;
|
|
1133
1065
|
tell(`cowshed: mbx reserve granted elapsed=${Date.now() - startedAt}ms priority=${reservedLeaf.priority}\n`);
|
|
1134
1066
|
channel.end(`${granted.cpus}\n`);
|
|
1135
1067
|
stepTo({ kind: 'granted', at: Date.now(), testing: isTesting(), fixture: !fixtureDone });
|
|
@@ -1148,7 +1080,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1148
1080
|
for (const byte of chunk) {
|
|
1149
1081
|
if (byte === 103) {
|
|
1150
1082
|
if (!said) {
|
|
1151
|
-
|
|
1083
|
+
queue.admitted = true;
|
|
1152
1084
|
said = true;
|
|
1153
1085
|
tell(`cowshed: mbx cargo granted elapsed=${Date.now() - startedAt}ms\n`);
|
|
1154
1086
|
}
|
|
@@ -1164,10 +1096,7 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1164
1096
|
}
|
|
1165
1097
|
});
|
|
1166
1098
|
// Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool.
|
|
1167
|
-
|
|
1168
|
-
void refreshQueue(false);
|
|
1169
|
-
}, MBX_QUEUE_POLL_MS);
|
|
1170
|
-
void refreshQueue(false);
|
|
1099
|
+
queue.poll();
|
|
1171
1100
|
}
|
|
1172
1101
|
}
|
|
1173
1102
|
// A grant's connection is a ledger entry, not the command's owner: the lending parent's exit ends it, and what runs
|
|
@@ -1188,8 +1117,8 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1188
1117
|
await signalCleanup;
|
|
1189
1118
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
1190
1119
|
}
|
|
1191
|
-
|
|
1192
|
-
await
|
|
1120
|
+
queue.stop();
|
|
1121
|
+
await queue.idle();
|
|
1193
1122
|
await termination;
|
|
1194
1123
|
clearTimeout(boundTimer);
|
|
1195
1124
|
cancelCapture();
|
|
@@ -1236,9 +1165,16 @@ export async function runBoundedExec(options, context, killer, tempVolume, inher
|
|
|
1236
1165
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1237
1166
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1238
1167
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1168
|
+
const killedForMemory = killedFromOutside({
|
|
1169
|
+
sentSignal: state.sentSignal,
|
|
1170
|
+
bounded: state.expiry !== null,
|
|
1171
|
+
exitSignal,
|
|
1172
|
+
code,
|
|
1173
|
+
});
|
|
1239
1174
|
return {
|
|
1240
1175
|
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1241
1176
|
terminalOutput: outputChunks.join(''),
|
|
1177
|
+
...(killedForMemory ? { killedForMemory: true } : {}),
|
|
1242
1178
|
};
|
|
1243
1179
|
}
|
|
1244
1180
|
export function createProcessTreeKiller() {
|
|
@@ -18,7 +18,27 @@ export interface HostMemory {
|
|
|
18
18
|
* listing and its read adds nothing, and a root that has exited holds nothing.
|
|
19
19
|
*/
|
|
20
20
|
treeResidentBytes(root: number): Promise<number>;
|
|
21
|
+
/**
|
|
22
|
+
* What the run's bound holds that the kernel cannot take back, read now: its anonymous and shared memory. `bound` is
|
|
23
|
+
* the `source` of the run's memory (`RunMemory`): the cgroup's `memory.max`, whose `memory.stat` is read, or the
|
|
24
|
+
* host's `MemTotal`, whose `AnonPages` and `Shmem` are. The pool says it on every grant, so what the leaves are not
|
|
25
|
+
* charged for (the baseline, a tmpfs) and what a suite holds beyond its record are measured, not guessed.
|
|
26
|
+
*/
|
|
27
|
+
liveUsage(bound: string): LiveUsageReading;
|
|
28
|
+
}
|
|
29
|
+
/** What a bound holds live, in bytes, and the file it was read from. */
|
|
30
|
+
export interface LiveUsage {
|
|
31
|
+
readonly anon: number;
|
|
32
|
+
readonly shmem: number;
|
|
33
|
+
readonly source: string;
|
|
21
34
|
}
|
|
35
|
+
export type LiveUsageReading = {
|
|
36
|
+
readonly ok: true;
|
|
37
|
+
readonly usage: LiveUsage;
|
|
38
|
+
} | {
|
|
39
|
+
readonly ok: false;
|
|
40
|
+
readonly reason: string;
|
|
41
|
+
};
|
|
22
42
|
/** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
|
|
23
43
|
export interface RunMemory {
|
|
24
44
|
readonly bytes: number;
|
|
@@ -51,4 +71,6 @@ export declare class RunMemoryUnreadable extends Error {
|
|
|
51
71
|
export declare const HOST_MEMORY: HostMemory | null;
|
|
52
72
|
/** The run's memory on this platform, or why it cannot be read. */
|
|
53
73
|
export declare function readRunMemory(): Promise<RunMemoryReading>;
|
|
74
|
+
/** The run's bound live on this platform, or why it cannot be read: a pool says it, never leaves it out. */
|
|
75
|
+
export declare function readLiveUsage(bound: string): LiveUsageReading;
|
|
54
76
|
//# sourceMappingURL=host-memory.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host-memory.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3C;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IACtD;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"host-memory.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3C;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IACtD;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACjD;;;;;OAKG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAAC;CAC5C;AAED,wEAAwE;AACxE,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAA;CAAE,GAChD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpD,0FAA0F;AAC1F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,4FAA4F;AAC5F,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,sBAAsB,CAAC;IACjF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAAC;AAE3D,sFAAsF;AACtF,qBAAa,mBAAoB,SAAQ,KAAK;IAChC,QAAQ,CAAC,KAAK,EAAE,cAAc;IAA1C,YAAqB,KAAK,EAAE,cAAc,EAKzC;CACF;AAID,kGAAkG;AAClG,eAAO,MAAM,WAAW,EAAE,UAAU,GAAG,IAA0C,CAAC;AAElF,mEAAmE;AACnE,wBAAgB,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAazD;AAED,4GAA4G;AAC5G,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAI7D"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host-memory.darwin.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.darwin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,kBAAkB,CAAC;AAqIrE,eAAO,MAAM,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"host-memory.darwin.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.darwin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,kBAAkB,CAAC;AAqIrE,eAAO,MAAM,YAAY,EAAE,UAM1B,CAAC"}
|
|
@@ -99,4 +99,10 @@ async function treeResidentBytes(root) {
|
|
|
99
99
|
}
|
|
100
100
|
return total;
|
|
101
101
|
}
|
|
102
|
-
export const darwinMemory = {
|
|
102
|
+
export const darwinMemory = {
|
|
103
|
+
readRunMemory,
|
|
104
|
+
peakBytes: (maxRss) => maxRss,
|
|
105
|
+
treeResidentBytes,
|
|
106
|
+
// macOS has no cgroup, and its anonymous and shared pages are not read from a file the pool may open.
|
|
107
|
+
liveUsage: () => ({ ok: false, reason: 'macOS keeps no anonymous and shared memory of a run to read' }),
|
|
108
|
+
};
|
|
@@ -27,3 +27,9 @@ export function readRunMemory() {
|
|
|
27
27
|
}
|
|
28
28
|
return HOST_MEMORY.readRunMemory();
|
|
29
29
|
}
|
|
30
|
+
/** The run's bound live on this platform, or why it cannot be read: a pool says it, never leaves it out. */
|
|
31
|
+
export function readLiveUsage(bound) {
|
|
32
|
+
return HOST_MEMORY === null
|
|
33
|
+
? { ok: false, reason: `${process.platform} has no reader of live memory` }
|
|
34
|
+
: HOST_MEMORY.liveUsage(bound);
|
|
35
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { HostMemory, RunMemoryReading } from './host-memory.js';
|
|
1
|
+
import type { HostMemory, LiveUsageReading, RunMemoryReading } from './host-memory.js';
|
|
2
2
|
/** A file's text, or the errno code and message its read failed with. */
|
|
3
3
|
export type FileRead = {
|
|
4
4
|
readonly ok: true;
|
|
@@ -10,5 +10,7 @@ export type FileRead = {
|
|
|
10
10
|
};
|
|
11
11
|
/** The run's memory, reading every file through `read`. */
|
|
12
12
|
export declare function linuxRunMemory(read: (path: string) => Promise<FileRead>): Promise<RunMemoryReading>;
|
|
13
|
+
/** The memory a bound holds live: `memory.stat`'s `anon` and `shmem` beside a cgroup's `memory.max`, meminfo's `AnonPages` and `Shmem` for the host's. */
|
|
14
|
+
export declare function linuxLiveUsage(read: (path: string) => FileRead, bound: string): LiveUsageReading;
|
|
13
15
|
export declare const linuxMemory: HostMemory;
|
|
14
16
|
//# sourceMappingURL=host-memory.linux.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host-memory.linux.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.linux.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"host-memory.linux.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.linux.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAA6B,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAclH,yEAAyE;AACzE,MAAM,MAAM,QAAQ,GAChB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AA2B5E,2DAA2D;AAC3D,wBAAsB,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA8CzG;AAWD,0JAA0J;AAC1J,wBAAgB,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAuBhG;AAmDD,eAAO,MAAM,WAAW,EAAE,UAKzB,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
1
2
|
import { readFile } from 'node:fs/promises';
|
|
2
3
|
import { posix } from 'node:path';
|
|
3
4
|
import { childPids } from '../../process-children.linux.js';
|
|
@@ -93,6 +94,42 @@ async function readText(path) {
|
|
|
93
94
|
return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
|
|
94
95
|
}
|
|
95
96
|
}
|
|
97
|
+
/** The memory a bound holds live: `memory.stat`'s `anon` and `shmem` beside a cgroup's `memory.max`, meminfo's `AnonPages` and `Shmem` for the host's. */
|
|
98
|
+
export function linuxLiveUsage(read, bound) {
|
|
99
|
+
const cgroupLimit = '/memory.max';
|
|
100
|
+
if (bound.endsWith(cgroupLimit)) {
|
|
101
|
+
const file = `${bound.slice(0, -cgroupLimit.length)}/memory.stat`;
|
|
102
|
+
const stat = read(file);
|
|
103
|
+
if (!stat.ok)
|
|
104
|
+
return { ok: false, reason: `${file}: ${stat.code}: ${stat.message}` };
|
|
105
|
+
const [anon, shmem] = [/^anon (\d+)$/m.exec(stat.text)?.[1], /^shmem (\d+)$/m.exec(stat.text)?.[1]];
|
|
106
|
+
return anon === undefined || shmem === undefined
|
|
107
|
+
? { ok: false, reason: `${file} names no anon and shmem in bytes` }
|
|
108
|
+
: { ok: true, usage: { anon: Number(anon), shmem: Number(shmem), source: file } };
|
|
109
|
+
}
|
|
110
|
+
if (bound === `${MEMINFO} MemTotal`) {
|
|
111
|
+
const info = read(MEMINFO);
|
|
112
|
+
if (!info.ok)
|
|
113
|
+
return { ok: false, reason: `${MEMINFO}: ${info.code}: ${info.message}` };
|
|
114
|
+
const [anon, shmem] = [
|
|
115
|
+
/^AnonPages:\s+(\d+) kB$/m.exec(info.text)?.[1],
|
|
116
|
+
/^Shmem:\s+(\d+) kB$/m.exec(info.text)?.[1],
|
|
117
|
+
];
|
|
118
|
+
return anon === undefined || shmem === undefined
|
|
119
|
+
? { ok: false, reason: `${MEMINFO} names no AnonPages and Shmem in kB` }
|
|
120
|
+
: { ok: true, usage: { anon: Number(anon) * 1024, shmem: Number(shmem) * 1024, source: MEMINFO } };
|
|
121
|
+
}
|
|
122
|
+
return { ok: false, reason: `${bound} is no cgroup memory.max or meminfo bound` };
|
|
123
|
+
}
|
|
124
|
+
function readTextSync(path) {
|
|
125
|
+
try {
|
|
126
|
+
return { ok: true, text: readFileSync(path, 'utf8') };
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
const code = error instanceof Error && 'code' in error && typeof error.code === 'string' ? error.code : 'EIO';
|
|
130
|
+
return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
|
|
131
|
+
}
|
|
132
|
+
}
|
|
96
133
|
/** Bun before 1.4.0 reports Linux's `ru_maxrss` as the kernel does, in KiB; from 1.4.0 it reports bytes. */
|
|
97
134
|
function peakBytes(maxRss, bunVersion) {
|
|
98
135
|
const version = /^(\d+)\.(\d+)\./.exec(bunVersion);
|
|
@@ -134,4 +171,9 @@ async function residentBytes(pid) {
|
|
|
134
171
|
const kib = /^VmRSS:\s+(\d+) kB$/m.exec(status)?.[1];
|
|
135
172
|
return kib === undefined ? 0 : Number(kib) * 1024;
|
|
136
173
|
}
|
|
137
|
-
export const linuxMemory = {
|
|
174
|
+
export const linuxMemory = {
|
|
175
|
+
readRunMemory: () => linuxRunMemory(readText),
|
|
176
|
+
peakBytes,
|
|
177
|
+
treeResidentBytes,
|
|
178
|
+
liveUsage: (bound) => linuxLiveUsage(readTextSync, bound),
|
|
179
|
+
};
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A run again after the kernel killed the command for memory (specs/cowshed/05_gateway.md, "Host scheduling"). Only
|
|
3
|
+
* when a pool lent the run its tokens and a SIGKILL that no bound of this executor and no signal it sent caused ended
|
|
4
|
+
* it: the command runs once more, alone in its run's pool, so what it was killed beside is not there to be killed with
|
|
5
|
+
* it. This decides whether a command runs a second time and how its two outputs join, never what it runs: the key
|
|
6
|
+
* leaves it out (README, "Bounded exec key").
|
|
7
|
+
*/
|
|
8
|
+
/** How one run of the command ended. */
|
|
9
|
+
export interface RunOutcome {
|
|
10
|
+
success: boolean;
|
|
11
|
+
terminalOutput: string;
|
|
12
|
+
/**
|
|
13
|
+
* The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
|
|
14
|
+
* which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
|
|
15
|
+
*/
|
|
16
|
+
killedForMemory?: boolean;
|
|
17
|
+
}
|
|
18
|
+
/** What a run read of how its command ended and what this executor did to it. */
|
|
19
|
+
export interface Ending {
|
|
20
|
+
/** This executor sent the command's process group a signal. */
|
|
21
|
+
readonly sentSignal: boolean;
|
|
22
|
+
/** A bound of this executor ended the run. */
|
|
23
|
+
readonly bounded: boolean;
|
|
24
|
+
readonly exitSignal: NodeJS.Signals | null;
|
|
25
|
+
/** The exit status, or the status a signal folds to. */
|
|
26
|
+
readonly code: number;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
|
|
30
|
+
* run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who sent
|
|
31
|
+
* it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
|
|
32
|
+
*/
|
|
33
|
+
export declare function killedFromOutside(ending: Ending): boolean;
|
|
34
|
+
/**
|
|
35
|
+
* Run the command; when the kernel killed it for memory and a pool lent it tokens, run it once more alone, and join the
|
|
36
|
+
* two runs' output with a note naming the task.
|
|
37
|
+
*/
|
|
38
|
+
export declare function runUntilNotKilledForMemory(run: (alone: boolean) => Promise<RunOutcome>, pooled: boolean, task: string | undefined): Promise<RunOutcome>;
|
|
39
|
+
//# sourceMappingURL=memory-retry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory-retry.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/memory-retry.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,wCAAwC;AACxC,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,cAAc,EAAE,MAAM,CAAC;IACvB;;;OAGG;IACH,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,iFAAiF;AACjF,MAAM,WAAW,MAAM;IACrB,+DAA+D;IAC/D,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,8CAA8C;IAC9C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;IAC3C,wDAAwD;IACxD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED;;;GAGG;AACH,wBAAsB,0BAA0B,CAC9C,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC,UAAU,CAAC,EAC5C,MAAM,EAAE,OAAO,EACf,IAAI,EAAE,MAAM,GAAG,SAAS,GACvB,OAAO,CAAC,UAAU,CAAC,CAerB"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A run again after the kernel killed the command for memory (specs/cowshed/05_gateway.md, "Host scheduling"). Only
|
|
3
|
+
* when a pool lent the run its tokens and a SIGKILL that no bound of this executor and no signal it sent caused ended
|
|
4
|
+
* it: the command runs once more, alone in its run's pool, so what it was killed beside is not there to be killed with
|
|
5
|
+
* it. This decides whether a command runs a second time and how its two outputs join, never what it runs: the key
|
|
6
|
+
* leaves it out (README, "Bounded exec key").
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
|
|
10
|
+
* run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who sent
|
|
11
|
+
* it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
|
|
12
|
+
*/
|
|
13
|
+
export function killedFromOutside(ending) {
|
|
14
|
+
return !ending.sentSignal && !ending.bounded && (ending.exitSignal === 'SIGKILL' || ending.code === 137);
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Run the command; when the kernel killed it for memory and a pool lent it tokens, run it once more alone, and join the
|
|
18
|
+
* two runs' output with a note naming the task.
|
|
19
|
+
*/
|
|
20
|
+
export async function runUntilNotKilledForMemory(run, pooled, task) {
|
|
21
|
+
const first = await run(false);
|
|
22
|
+
if (first.killedForMemory !== true || !pooled) {
|
|
23
|
+
return first;
|
|
24
|
+
}
|
|
25
|
+
const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${task === undefined ? '' : `: ${task}`}); running it once more, alone in its run's pool\n`;
|
|
26
|
+
process.stderr.write(note);
|
|
27
|
+
const again = await run(true);
|
|
28
|
+
return {
|
|
29
|
+
success: again.success,
|
|
30
|
+
terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
|
|
31
|
+
...(again.killedForMemory === true ? { killedForMemory: true } : {}),
|
|
32
|
+
};
|
|
33
|
+
}
|