@smoothbricks/nx-plugin 0.7.11 → 0.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -1
- package/dist/.build-inputs.json +9 -9
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +25 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +86 -23
- package/dist/executors/bounded-exec/executor.d.ts +5 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +44 -5
- package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
- package/dist/executors/bounded-exec/host-memory.js +6 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
- package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +44 -8
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +4 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +19 -2
- package/managed/raw/patches/nx@24.0.3.patch +16 -0
- package/package.json +1 -1
- package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
- package/src/executors/bounded-exec/cpu-pool.ts +121 -25
- package/src/executors/bounded-exec/executor.ts +71 -5
- package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
- package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
- package/src/executors/bounded-exec/host-memory.ts +25 -0
- package/src/executors/bounded-exec/pool-admission.ts +66 -11
- package/src/executors/bounded-exec/run-cpu-pool.ts +3 -3
- package/src/executors/bounded-exec/verdict.ts +25 -5
- package/test-timings.json +2 -0
|
@@ -7,6 +7,7 @@ import { promisify } from 'node:util';
|
|
|
7
7
|
import typia from 'typia';
|
|
8
8
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
9
9
|
import { holdLease, type Unleased } from './gateway-lease.js';
|
|
10
|
+
import { type LiveUsageReading, type RunMemory, readLiveUsage } from './host-memory.js';
|
|
10
11
|
import { type CpuLeaseProgress, type CpuLeaseQueued, CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
|
|
11
12
|
import { admit } from './pool-admission.js';
|
|
12
13
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
@@ -63,12 +64,16 @@ export type CloseOutcome = { ok: true } | { ok: false; error: string };
|
|
|
63
64
|
|
|
64
65
|
/**
|
|
65
66
|
* The one line a child sends: a count of whole tokens, at least one, and the bytes it is charged, its task's last
|
|
66
|
-
* recorded memory peak. A child whose task has no recorded peak
|
|
67
|
+
* recorded memory peak, with the tokens it was lent when that peak was measured. A child whose task has no recorded peak
|
|
68
|
+
* names none: it is charged nothing. A peak with no tokens is charged flat.
|
|
67
69
|
*/
|
|
68
70
|
interface CpuPoolRequest {
|
|
69
71
|
op: 'cpu-pool';
|
|
70
72
|
want: number & typia.tags.Type<'uint32'> & typia.tags.Minimum<1>;
|
|
71
73
|
memory?: number & typia.tags.Type<'uint64'>;
|
|
74
|
+
parallelism?: number & typia.tags.Type<'uint32'> & typia.tags.Minimum<1>;
|
|
75
|
+
task?: string & typia.tags.MaxLength<256>;
|
|
76
|
+
alone?: boolean;
|
|
72
77
|
}
|
|
73
78
|
|
|
74
79
|
/** A grant whose charge alone exceeds the run's memory, made while nothing else held memory: the run's memory. */
|
|
@@ -80,7 +85,7 @@ export interface Oversized {
|
|
|
80
85
|
type CpuPoolReply =
|
|
81
86
|
| CpuLeaseProgress
|
|
82
87
|
| CpuLeaseQueued
|
|
83
|
-
| { ok: true; lease: 'granted'; tokens: number; oversized?: Oversized }
|
|
88
|
+
| { ok: true; lease: 'granted'; tokens: number; oversized?: Oversized; charged?: number }
|
|
84
89
|
| { ok: false; code: 'invalid-request'; error: string };
|
|
85
90
|
|
|
86
91
|
/** Exactly a request's fields: any other field is an invalid request. */
|
|
@@ -109,7 +114,13 @@ const run = promisify(execFile);
|
|
|
109
114
|
type Phase =
|
|
110
115
|
| { readonly kind: 'reading' }
|
|
111
116
|
| { readonly kind: 'queued' }
|
|
112
|
-
| {
|
|
117
|
+
| {
|
|
118
|
+
readonly kind: 'holding';
|
|
119
|
+
readonly bytes: number;
|
|
120
|
+
readonly slot: boolean;
|
|
121
|
+
readonly memory: number;
|
|
122
|
+
readonly alone: boolean;
|
|
123
|
+
}
|
|
113
124
|
| { readonly kind: 'done' };
|
|
114
125
|
|
|
115
126
|
/** A child connection and what the pool owes it. Its grant, or its place in line, ends with the connection. */
|
|
@@ -118,11 +129,14 @@ interface Child {
|
|
|
118
129
|
phase: Phase;
|
|
119
130
|
}
|
|
120
131
|
|
|
121
|
-
/** A queued child, the count it asked for
|
|
132
|
+
/** A queued child, the count it asked for, the bytes it is charged, the tokens that charge was measured under, its task. */
|
|
122
133
|
interface Waiter {
|
|
123
134
|
readonly child: Child;
|
|
124
135
|
readonly want: number;
|
|
125
136
|
readonly memory: number;
|
|
137
|
+
readonly parallelism?: number | undefined;
|
|
138
|
+
readonly task?: string | undefined;
|
|
139
|
+
readonly alone?: boolean | undefined;
|
|
126
140
|
}
|
|
127
141
|
|
|
128
142
|
/** One framed reply line. The newline is the framing; every reply goes through here. */
|
|
@@ -143,34 +157,59 @@ export function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>):
|
|
|
143
157
|
}
|
|
144
158
|
|
|
145
159
|
/**
|
|
146
|
-
* Tokens a parent pool lent, held until released, or why there are none. `
|
|
147
|
-
*
|
|
148
|
-
*
|
|
160
|
+
* Tokens a parent pool lent, held until released, or why there are none. `charged` is the bytes the pool charged the
|
|
161
|
+
* grant, null from a pool that charges none. `oversized` names the run's memory when the child's charge alone exceeds
|
|
162
|
+
* it and it was granted because nothing else held memory. `closed` resolves when the grant's connection closes, however
|
|
163
|
+
* it closes, so a sub-pool built on the grant ends with it.
|
|
149
164
|
*/
|
|
150
165
|
export type TokenGrant =
|
|
151
|
-
| {
|
|
166
|
+
| {
|
|
167
|
+
granted: true;
|
|
168
|
+
tokens: number;
|
|
169
|
+
charged: number | null;
|
|
170
|
+
oversized: Oversized | null;
|
|
171
|
+
release: () => void;
|
|
172
|
+
closed: Promise<void>;
|
|
173
|
+
}
|
|
152
174
|
| Unleased;
|
|
153
175
|
|
|
154
176
|
const isOversized = typia.createEquals<Oversized>();
|
|
155
177
|
|
|
178
|
+
/** What a leaf tells the pool beyond its count and its charge. */
|
|
179
|
+
export interface AskContext {
|
|
180
|
+
/** The tokens the task was lent when its recorded peak was measured. */
|
|
181
|
+
readonly parallelism?: number | undefined;
|
|
182
|
+
/** The Nx task the leaf runs, named in the pool's line for the grant. */
|
|
183
|
+
readonly task?: string | undefined;
|
|
184
|
+
/** The kernel killed the command for memory: it is run again, granted only while nothing else holds a grant. */
|
|
185
|
+
readonly alone?: boolean | undefined;
|
|
186
|
+
}
|
|
187
|
+
|
|
156
188
|
/**
|
|
157
189
|
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
158
|
-
* it has none.
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
190
|
+
* it has none. `context.parallelism` is the tokens the task was lent when that peak was measured; the pool charges the
|
|
191
|
+
* peak in proportion to the tokens it lends now when that is more, and lends fewer when the run's memory holds no more.
|
|
192
|
+
* The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the child's local
|
|
193
|
+
* allocation: its connection is the grant, and closing it returns only that allocation. An absent endpoint grants
|
|
194
|
+
* nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission rather than run
|
|
195
|
+
* the child beside its parent's count.
|
|
162
196
|
*/
|
|
163
197
|
export async function takeParentCpuTokens(
|
|
164
198
|
parent: InheritedCpuPool,
|
|
165
199
|
want: number,
|
|
166
200
|
memory: number | null,
|
|
201
|
+
context: AskContext = {},
|
|
167
202
|
): Promise<TokenGrant> {
|
|
203
|
+
const { parallelism, task, alone } = context;
|
|
168
204
|
if (!Number.isSafeInteger(want) || want < 1) {
|
|
169
205
|
throw new RangeError(`a child asks its parent pool for a positive count, not ${want} (pool ${parent.tokens})`);
|
|
170
206
|
}
|
|
171
207
|
if (memory !== null && (!Number.isSafeInteger(memory) || memory < 0)) {
|
|
172
208
|
throw new RangeError(`a child is charged a whole number of bytes, not ${memory}`);
|
|
173
209
|
}
|
|
210
|
+
if (parallelism !== undefined && (!Number.isSafeInteger(parallelism) || parallelism < 1)) {
|
|
211
|
+
throw new RangeError(`a child's charge is measured under a positive count of tokens, not ${parallelism}`);
|
|
212
|
+
}
|
|
174
213
|
if (parent.socket.length === 0 || parent.socket.includes('\0')) {
|
|
175
214
|
return {
|
|
176
215
|
granted: false,
|
|
@@ -178,12 +217,18 @@ export async function takeParentCpuTokens(
|
|
|
178
217
|
reason: `the parent CPU pool's endpoint ${JSON.stringify(parent.socket)} is not a socket path`,
|
|
179
218
|
};
|
|
180
219
|
}
|
|
181
|
-
const request =
|
|
220
|
+
const request = {
|
|
221
|
+
op: 'cpu-pool',
|
|
222
|
+
want,
|
|
223
|
+
...(memory === null ? {} : { memory, ...(parallelism === undefined ? {} : { parallelism }) }),
|
|
224
|
+
...(task === undefined ? {} : { task }),
|
|
225
|
+
...(alone === true ? { alone: true } : {}),
|
|
226
|
+
};
|
|
182
227
|
const lease = await holdLease(parent.socket, request, 'the parent CPU pool', new CpuLeaseWait());
|
|
183
228
|
if (!lease.granted) {
|
|
184
229
|
return lease;
|
|
185
230
|
}
|
|
186
|
-
const { tokens, oversized } = lease.answer;
|
|
231
|
+
const { tokens, oversized, charged } = lease.answer;
|
|
187
232
|
if (typeof tokens !== 'number' || !Number.isInteger(tokens) || tokens < 1 || tokens > want) {
|
|
188
233
|
lease.release();
|
|
189
234
|
return {
|
|
@@ -200,7 +245,22 @@ export async function takeParentCpuTokens(
|
|
|
200
245
|
reason: `the parent CPU pool named its memory ${JSON.stringify(oversized)} in a grant, which is no byte count`,
|
|
201
246
|
};
|
|
202
247
|
}
|
|
203
|
-
|
|
248
|
+
if (charged !== undefined && !(typeof charged === 'number' && Number.isSafeInteger(charged) && charged >= 0)) {
|
|
249
|
+
lease.release();
|
|
250
|
+
return {
|
|
251
|
+
granted: false,
|
|
252
|
+
cause: 'other',
|
|
253
|
+
reason: `the parent CPU pool charged ${JSON.stringify(charged)} in a grant, which is no byte count`,
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
return {
|
|
257
|
+
granted: true,
|
|
258
|
+
tokens,
|
|
259
|
+
charged: charged ?? null,
|
|
260
|
+
oversized: oversized ?? null,
|
|
261
|
+
release: lease.release,
|
|
262
|
+
closed: lease.closed,
|
|
263
|
+
};
|
|
204
264
|
}
|
|
205
265
|
|
|
206
266
|
/** A thrown value's message. */
|
|
@@ -354,13 +414,15 @@ function watchFifo(
|
|
|
354
414
|
/**
|
|
355
415
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
356
416
|
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
357
|
-
* pool charges its grants against `memory
|
|
417
|
+
* pool charges its grants against `memory` and says each grant with what the run's bound holds live
|
|
418
|
+
* (`liveUsage`); a sub-pool charges none (null).
|
|
358
419
|
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
359
420
|
*/
|
|
360
421
|
export async function createCpuPool(
|
|
361
422
|
tokens: number,
|
|
362
423
|
closed: Promise<void>,
|
|
363
|
-
memory:
|
|
424
|
+
memory: RunMemory | null,
|
|
425
|
+
liveUsage: (bound: string) => LiveUsageReading = readLiveUsage,
|
|
364
426
|
): Promise<{ ok: true; pool: CpuPool } | { ok: false; error: string }> {
|
|
365
427
|
if (!Number.isSafeInteger(tokens) || tokens < 1) {
|
|
366
428
|
throw new RangeError(`a CPU pool holds a positive safe integer of tokens, not ${tokens}`);
|
|
@@ -389,6 +451,9 @@ export async function createCpuPool(
|
|
|
389
451
|
let slotLent = false;
|
|
390
452
|
// The bytes the children holding grants are charged, against `memory`.
|
|
391
453
|
let heldMemory = 0;
|
|
454
|
+
// The children holding a grant, and whether one of them runs alone.
|
|
455
|
+
let holders = 0;
|
|
456
|
+
let aloneHolder = false;
|
|
392
457
|
// The readability wait while the queue's head finds no token, else undefined.
|
|
393
458
|
let watch: FifoWatch | undefined;
|
|
394
459
|
let closing = false;
|
|
@@ -477,24 +542,39 @@ export async function createCpuPool(
|
|
|
477
542
|
const admission = admit(
|
|
478
543
|
queue,
|
|
479
544
|
inHand + (slotLent ? 0 : 1),
|
|
480
|
-
memory === null ? null : { capacity: memory, held: heldMemory },
|
|
545
|
+
memory === null ? null : { capacity: memory.bytes, held: heldMemory },
|
|
546
|
+
{ holders, alone: aloneHolder },
|
|
481
547
|
);
|
|
482
548
|
queue.splice(0, admission.grants.length);
|
|
483
|
-
for (const { request, tokens: granted, oversized } of admission.grants) {
|
|
549
|
+
for (const { request, tokens: granted, charge, oversized } of admission.grants) {
|
|
484
550
|
// Fifo bytes first, the implicit slot last.
|
|
485
551
|
const bytes = Math.min(granted, inHand);
|
|
486
552
|
inHand -= bytes;
|
|
487
553
|
const slot = granted > bytes;
|
|
488
554
|
if (slot) slotLent = true;
|
|
489
|
-
heldMemory +=
|
|
490
|
-
|
|
555
|
+
heldMemory += charge;
|
|
556
|
+
holders += 1;
|
|
557
|
+
if (request.alone === true) aloneHolder = true;
|
|
558
|
+
request.child.phase = { kind: 'holding', bytes, slot, memory: charge, alone: request.alone === true };
|
|
491
559
|
request.child.socket.write(
|
|
492
560
|
frame(
|
|
493
|
-
|
|
494
|
-
? { ok: true, lease: 'granted', tokens: granted
|
|
495
|
-
:
|
|
561
|
+
memory === null
|
|
562
|
+
? { ok: true, lease: 'granted', tokens: granted }
|
|
563
|
+
: oversized
|
|
564
|
+
? { ok: true, lease: 'granted', tokens: granted, charged: charge, oversized: { memory: memory.bytes } }
|
|
565
|
+
: { ok: true, lease: 'granted', tokens: granted, charged: charge },
|
|
496
566
|
),
|
|
497
567
|
);
|
|
568
|
+
|
|
569
|
+
if (memory !== null) {
|
|
570
|
+
const live = liveUsage(memory.source);
|
|
571
|
+
const held = live.ok
|
|
572
|
+
? `${live.usage.anon + live.usage.shmem} bytes (anon ${live.usage.anon}, shmem ${live.usage.shmem}, ${live.usage.source})`
|
|
573
|
+
: `unread (${live.reason})`;
|
|
574
|
+
process.stderr.write(
|
|
575
|
+
`bounded-exec: the CPU pool at ${socketPath} granted ${request.task ?? 'a leaf'} ${granted} of ${request.want} tokens charged ${charge} bytes: holders charged ${heldMemory} of ${memory.bytes} bytes, live anon+shmem ${held}\n`,
|
|
576
|
+
);
|
|
577
|
+
}
|
|
498
578
|
}
|
|
499
579
|
if (admission.waiting === 'cpu') {
|
|
500
580
|
watchForTokens();
|
|
@@ -530,6 +610,8 @@ export async function createCpuPool(
|
|
|
530
610
|
// A closing pool drains its fifo whole; until then the bytes go back for the next holder.
|
|
531
611
|
if (child.phase.slot) slotLent = false;
|
|
532
612
|
heldMemory -= child.phase.memory;
|
|
613
|
+
holders -= 1;
|
|
614
|
+
if (child.phase.alone) aloneHolder = false;
|
|
533
615
|
if (!closing) returnTokens(child.phase.bytes);
|
|
534
616
|
}
|
|
535
617
|
child.phase = { kind: 'done' };
|
|
@@ -590,7 +672,21 @@ export async function createCpuPool(
|
|
|
590
672
|
}
|
|
591
673
|
const request = parsed;
|
|
592
674
|
child.phase = { kind: 'queued' };
|
|
593
|
-
|
|
675
|
+
const waiter: Waiter = {
|
|
676
|
+
child,
|
|
677
|
+
want: request.want,
|
|
678
|
+
memory: request.memory ?? 0,
|
|
679
|
+
parallelism: request.parallelism,
|
|
680
|
+
task: request.task,
|
|
681
|
+
alone: request.alone,
|
|
682
|
+
};
|
|
683
|
+
if (request.alone === true) {
|
|
684
|
+
// A leaf the kernel killed has waited once already: it goes ahead of the leaves that have not run.
|
|
685
|
+
const at = queue.findIndex((waiting) => waiting.alone !== true);
|
|
686
|
+
queue.splice(at === -1 ? queue.length : at, 0, waiter);
|
|
687
|
+
} else {
|
|
688
|
+
queue.push(waiter);
|
|
689
|
+
}
|
|
594
690
|
socket.write(frame({ ok: true, lease: 'queued', server: process.pid }));
|
|
595
691
|
drain(0);
|
|
596
692
|
});
|
|
@@ -134,12 +134,19 @@ export interface NxTaskContext {
|
|
|
134
134
|
export interface BoundedExecResult {
|
|
135
135
|
success: boolean;
|
|
136
136
|
terminalOutput: string;
|
|
137
|
+
/**
|
|
138
|
+
* The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
|
|
139
|
+
* which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
|
|
140
|
+
*/
|
|
141
|
+
killedForMemory?: boolean;
|
|
137
142
|
}
|
|
138
143
|
|
|
139
144
|
interface RunState {
|
|
140
145
|
settled: boolean;
|
|
141
146
|
expiry: ExpiredBound | null;
|
|
142
147
|
forceKillNeeded: boolean;
|
|
148
|
+
/** This executor sent the command's process group a signal: a SIGKILL it died of is not from outside. */
|
|
149
|
+
sentSignal: boolean;
|
|
143
150
|
/** The sub-pool's close error when it did not close cleanly: the run fails. */
|
|
144
151
|
poolCloseError: string | null;
|
|
145
152
|
/** Why the command's subtree could not be retired, when the supervisor's client failed: the run fails. */
|
|
@@ -313,10 +320,55 @@ export async function runBoundedExec(
|
|
|
313
320
|
owner: SubtreeOwner | null = null,
|
|
314
321
|
queueHost?: Parameters<typeof snapshotMbxQueue>[3],
|
|
315
322
|
): Promise<BoundedExecResult> {
|
|
316
|
-
const cwd = resolveCwd(options.cwd, context.root);
|
|
317
323
|
const directory = context.record?.directory ?? null;
|
|
318
|
-
// The task's last recorded peak
|
|
324
|
+
// The task's last recorded peak, and the tokens it was lent when the peak was measured, are its charge in its parent's
|
|
325
|
+
// pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
|
|
326
|
+
// killed run replaced, not by the peak it reached before it was killed.
|
|
319
327
|
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
328
|
+
const run = (alone: boolean): Promise<BoundedExecResult> =>
|
|
329
|
+
runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
|
|
330
|
+
lastPeak,
|
|
331
|
+
alone,
|
|
332
|
+
});
|
|
333
|
+
const first = await run(false);
|
|
334
|
+
if (first.killedForMemory !== true || inherited === null) {
|
|
335
|
+
return first;
|
|
336
|
+
}
|
|
337
|
+
// The kernel killed the leaf: a SIGKILL this executor did not send. It runs once more, alone in its run's pool, so
|
|
338
|
+
// what it was killed beside is not there to be killed with it.
|
|
339
|
+
const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${
|
|
340
|
+
context.record === undefined ? '' : `: ${context.record.task.id}`
|
|
341
|
+
}); running it once more, alone in its run's pool\n`;
|
|
342
|
+
process.stderr.write(note);
|
|
343
|
+
const again = await run(true);
|
|
344
|
+
return {
|
|
345
|
+
success: again.success,
|
|
346
|
+
terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
|
|
347
|
+
...(again.killedForMemory === true ? { killedForMemory: true } : {}),
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** What a run of the command is told that the run before it was not: the task's charge, and whether it runs alone. */
|
|
352
|
+
interface Attempt {
|
|
353
|
+
readonly lastPeak: Awaited<ReturnType<typeof readLastPeak>> | null;
|
|
354
|
+
readonly alone: boolean;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/** One run of the command, from the pool's grant to its record. */
|
|
358
|
+
async function runCommand(
|
|
359
|
+
options: BoundedExecOptions,
|
|
360
|
+
context: BoundedExecContext,
|
|
361
|
+
killer: ProcessTreeKiller,
|
|
362
|
+
tempVolume: TempVolume | null,
|
|
363
|
+
inherited: InheritedCpuPool | null,
|
|
364
|
+
captureDiagnostics: typeof captureHangDiagnostics,
|
|
365
|
+
owner: SubtreeOwner | null,
|
|
366
|
+
queueHost: Parameters<typeof snapshotMbxQueue>[3] | undefined,
|
|
367
|
+
attempt: Attempt,
|
|
368
|
+
): Promise<BoundedExecResult> {
|
|
369
|
+
const cwd = resolveCwd(options.cwd, context.root);
|
|
370
|
+
const directory = context.record?.directory ?? null;
|
|
371
|
+
const { lastPeak } = attempt;
|
|
320
372
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
321
373
|
if (directory !== null) {
|
|
322
374
|
await clearTaskDirectory(directory);
|
|
@@ -336,6 +388,7 @@ export async function runBoundedExec(
|
|
|
336
388
|
settled: false,
|
|
337
389
|
expiry: null,
|
|
338
390
|
forceKillNeeded: false,
|
|
391
|
+
sentSignal: false,
|
|
339
392
|
poolCloseError: null,
|
|
340
393
|
retireError: null,
|
|
341
394
|
noSpace: null,
|
|
@@ -408,8 +461,10 @@ export async function runBoundedExec(
|
|
|
408
461
|
`bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`,
|
|
409
462
|
);
|
|
410
463
|
}
|
|
411
|
-
// A task with no recorded peak is charged nothing (null) until it has one.
|
|
464
|
+
// A task with no recorded peak is charged nothing (null) until it has one. A peak is charged in proportion to the
|
|
465
|
+
// tokens the run is lent when that is more than the run that measured it was lent.
|
|
412
466
|
const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
|
|
467
|
+
const chargeParallelism = lastPeak?.ok === true ? lastPeak.parallelism : null;
|
|
413
468
|
|
|
414
469
|
let lease: RamTempLease | null = null;
|
|
415
470
|
let ordinaryTemp: string | null = null;
|
|
@@ -557,7 +612,11 @@ export async function runBoundedExec(
|
|
|
557
612
|
let parentRefusal: string | null = null;
|
|
558
613
|
let parentWait: CpuWaitRow | undefined;
|
|
559
614
|
if (inherited !== null) {
|
|
560
|
-
const fromParent = await takeParentCpuTokens(inherited, demand.want, charge
|
|
615
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want, charge, {
|
|
616
|
+
parallelism: chargeParallelism ?? undefined,
|
|
617
|
+
alone: attempt.alone ? true : undefined,
|
|
618
|
+
task: context.record?.task.id,
|
|
619
|
+
});
|
|
561
620
|
if (fromParent.granted) {
|
|
562
621
|
grant = fromParent;
|
|
563
622
|
} else if (fromParent.cause === 'absent') {
|
|
@@ -578,7 +637,7 @@ export async function runBoundedExec(
|
|
|
578
637
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
579
638
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
580
639
|
appendStderr(
|
|
581
|
-
`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`,
|
|
640
|
+
`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}${grant.charged !== null && grant.charged !== charge ? ` charged=${grant.charged}` : ''}\n`,
|
|
582
641
|
);
|
|
583
642
|
if (grant.oversized !== null) {
|
|
584
643
|
appendStderr(
|
|
@@ -1131,6 +1190,7 @@ export async function runBoundedExec(
|
|
|
1131
1190
|
return;
|
|
1132
1191
|
}
|
|
1133
1192
|
try {
|
|
1193
|
+
state.sentSignal = true;
|
|
1134
1194
|
await killer.kill(pid, 'SIGTERM');
|
|
1135
1195
|
} catch (error) {
|
|
1136
1196
|
const reason = error instanceof Error ? error.message : String(error);
|
|
@@ -1280,6 +1340,7 @@ export async function runBoundedExec(
|
|
|
1280
1340
|
// cannot end is said, and a failure of the probe or of a signal is shown.
|
|
1281
1341
|
const forceKill = async (pid: number): Promise<void> => {
|
|
1282
1342
|
try {
|
|
1343
|
+
state.sentSignal = true;
|
|
1283
1344
|
if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
|
|
1284
1345
|
tell(`${describeMembersLeft(pid)}\n`);
|
|
1285
1346
|
}
|
|
@@ -1488,9 +1549,14 @@ export async function runBoundedExec(
|
|
|
1488
1549
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1489
1550
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1490
1551
|
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1552
|
+
// A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
|
|
1553
|
+
// run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who
|
|
1554
|
+
// sent it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
|
|
1555
|
+
const killedForMemory = !state.sentSignal && state.expiry === null && (exitSignal === 'SIGKILL' || code === 137);
|
|
1491
1556
|
return {
|
|
1492
1557
|
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1493
1558
|
terminalOutput: outputChunks.join(''),
|
|
1559
|
+
...(killedForMemory ? { killedForMemory: true } : {}),
|
|
1494
1560
|
};
|
|
1495
1561
|
}
|
|
1496
1562
|
|
|
@@ -132,4 +132,10 @@ async function treeResidentBytes(root: number): Promise<number> {
|
|
|
132
132
|
return total;
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
-
export const darwinMemory: HostMemory = {
|
|
135
|
+
export const darwinMemory: HostMemory = {
|
|
136
|
+
readRunMemory,
|
|
137
|
+
peakBytes: (maxRss) => maxRss,
|
|
138
|
+
treeResidentBytes,
|
|
139
|
+
// macOS has no cgroup, and its anonymous and shared pages are not read from a file the pool may open.
|
|
140
|
+
liveUsage: () => ({ ok: false, reason: 'macOS keeps no anonymous and shared memory of a run to read' }),
|
|
141
|
+
};
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
1
2
|
import { readFile } from 'node:fs/promises';
|
|
2
3
|
import { posix } from 'node:path';
|
|
3
4
|
import { childPids } from '../../process-children.linux.js';
|
|
4
|
-
import type { HostMemory, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
|
|
5
|
+
import type { HostMemory, LiveUsageReading, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
|
|
5
6
|
|
|
6
7
|
/**
|
|
7
8
|
* Linux's memory mechanisms. The run's memory is the least of the host's `MemTotal` and every finite `memory.max` of
|
|
@@ -103,6 +104,40 @@ async function readText(path: string): Promise<FileRead> {
|
|
|
103
104
|
}
|
|
104
105
|
}
|
|
105
106
|
|
|
107
|
+
/** The memory a bound holds live: `memory.stat`'s `anon` and `shmem` beside a cgroup's `memory.max`, meminfo's `AnonPages` and `Shmem` for the host's. */
|
|
108
|
+
export function linuxLiveUsage(read: (path: string) => FileRead, bound: string): LiveUsageReading {
|
|
109
|
+
const cgroupLimit = '/memory.max';
|
|
110
|
+
if (bound.endsWith(cgroupLimit)) {
|
|
111
|
+
const file = `${bound.slice(0, -cgroupLimit.length)}/memory.stat`;
|
|
112
|
+
const stat = read(file);
|
|
113
|
+
if (!stat.ok) return { ok: false, reason: `${file}: ${stat.code}: ${stat.message}` };
|
|
114
|
+
const [anon, shmem] = [/^anon (\d+)$/m.exec(stat.text)?.[1], /^shmem (\d+)$/m.exec(stat.text)?.[1]];
|
|
115
|
+
return anon === undefined || shmem === undefined
|
|
116
|
+
? { ok: false, reason: `${file} names no anon and shmem in bytes` }
|
|
117
|
+
: { ok: true, usage: { anon: Number(anon), shmem: Number(shmem), source: file } };
|
|
118
|
+
}
|
|
119
|
+
if (bound === `${MEMINFO} MemTotal`) {
|
|
120
|
+
const info = read(MEMINFO);
|
|
121
|
+
if (!info.ok) return { ok: false, reason: `${MEMINFO}: ${info.code}: ${info.message}` };
|
|
122
|
+
const [anon, shmem] = [
|
|
123
|
+
/^AnonPages:\s+(\d+) kB$/m.exec(info.text)?.[1],
|
|
124
|
+
/^Shmem:\s+(\d+) kB$/m.exec(info.text)?.[1],
|
|
125
|
+
];
|
|
126
|
+
return anon === undefined || shmem === undefined
|
|
127
|
+
? { ok: false, reason: `${MEMINFO} names no AnonPages and Shmem in kB` }
|
|
128
|
+
: { ok: true, usage: { anon: Number(anon) * 1024, shmem: Number(shmem) * 1024, source: MEMINFO } };
|
|
129
|
+
}
|
|
130
|
+
return { ok: false, reason: `${bound} is no cgroup memory.max or meminfo bound` };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function readTextSync(path: string): FileRead {
|
|
134
|
+
try {
|
|
135
|
+
return { ok: true, text: readFileSync(path, 'utf8') };
|
|
136
|
+
} catch (error) {
|
|
137
|
+
const code = error instanceof Error && 'code' in error && typeof error.code === 'string' ? error.code : 'EIO';
|
|
138
|
+
return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
|
|
139
|
+
}
|
|
140
|
+
}
|
|
106
141
|
/** Bun before 1.4.0 reports Linux's `ru_maxrss` as the kernel does, in KiB; from 1.4.0 it reports bytes. */
|
|
107
142
|
function peakBytes(maxRss: number, bunVersion: string): number {
|
|
108
143
|
const version = /^(\d+)\.(\d+)\./.exec(bunVersion);
|
|
@@ -144,4 +179,9 @@ async function residentBytes(pid: number): Promise<number> {
|
|
|
144
179
|
return kib === undefined ? 0 : Number(kib) * 1024;
|
|
145
180
|
}
|
|
146
181
|
|
|
147
|
-
export const linuxMemory: HostMemory = {
|
|
182
|
+
export const linuxMemory: HostMemory = {
|
|
183
|
+
readRunMemory: () => linuxRunMemory(readText),
|
|
184
|
+
peakBytes,
|
|
185
|
+
treeResidentBytes,
|
|
186
|
+
liveUsage: (bound) => linuxLiveUsage(readTextSync, bound),
|
|
187
|
+
};
|
|
@@ -21,8 +21,26 @@ export interface HostMemory {
|
|
|
21
21
|
* listing and its read adds nothing, and a root that has exited holds nothing.
|
|
22
22
|
*/
|
|
23
23
|
treeResidentBytes(root: number): Promise<number>;
|
|
24
|
+
/**
|
|
25
|
+
* What the run's bound holds that the kernel cannot take back, read now: its anonymous and shared memory. `bound` is
|
|
26
|
+
* the `source` of the run's memory (`RunMemory`): the cgroup's `memory.max`, whose `memory.stat` is read, or the
|
|
27
|
+
* host's `MemTotal`, whose `AnonPages` and `Shmem` are. The pool says it on every grant, so what the leaves are not
|
|
28
|
+
* charged for (the baseline, a tmpfs) and what a suite holds beyond its record are measured, not guessed.
|
|
29
|
+
*/
|
|
30
|
+
liveUsage(bound: string): LiveUsageReading;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** What a bound holds live, in bytes, and the file it was read from. */
|
|
34
|
+
export interface LiveUsage {
|
|
35
|
+
readonly anon: number;
|
|
36
|
+
readonly shmem: number;
|
|
37
|
+
readonly source: string;
|
|
24
38
|
}
|
|
25
39
|
|
|
40
|
+
export type LiveUsageReading =
|
|
41
|
+
| { readonly ok: true; readonly usage: LiveUsage }
|
|
42
|
+
| { readonly ok: false; readonly reason: string };
|
|
43
|
+
|
|
26
44
|
/** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
|
|
27
45
|
export interface RunMemory {
|
|
28
46
|
readonly bytes: number;
|
|
@@ -75,3 +93,10 @@ export function readRunMemory(): Promise<RunMemoryReading> {
|
|
|
75
93
|
}
|
|
76
94
|
return HOST_MEMORY.readRunMemory();
|
|
77
95
|
}
|
|
96
|
+
|
|
97
|
+
/** The run's bound live on this platform, or why it cannot be read: a pool says it, never leaves it out. */
|
|
98
|
+
export function readLiveUsage(bound: string): LiveUsageReading {
|
|
99
|
+
return HOST_MEMORY === null
|
|
100
|
+
? { ok: false, reason: `${process.platform} has no reader of live memory` }
|
|
101
|
+
: HOST_MEMORY.liveUsage(bound);
|
|
102
|
+
}
|