@smoothbricks/nx-plugin 0.7.5 → 0.7.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -3
- package/dist/.build-inputs.json +11 -5
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +3 -2
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +24 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +96 -52
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +98 -45
- package/dist/executors/bounded-exec/host-memory.d.ts +47 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts +3 -0
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.darwin.js +40 -0
- package/dist/executors/bounded-exec/host-memory.js +29 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +14 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -0
- package/dist/executors/bounded-exec/host-memory.linux.js +104 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts +55 -0
- package/dist/executors/bounded-exec/leaf-peak.d.ts.map +1 -0
- package/dist/executors/bounded-exec/leaf-peak.js +146 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts +2 -0
- package/dist/executors/bounded-exec/peak-waiter.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-waiter.js +53 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +38 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-admission.js +31 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +6 -4
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +20 -7
- package/dist/executors/bounded-exec/verdict.d.ts +18 -0
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +21 -1
- package/package.json +1 -1
- package/src/executors/bounded-exec/cpu-pool-thread.ts +4 -3
- package/src/executors/bounded-exec/cpu-pool.ts +111 -49
- package/src/executors/bounded-exec/executor.ts +103 -41
- package/src/executors/bounded-exec/host-memory.darwin.ts +44 -0
- package/src/executors/bounded-exec/host-memory.linux.ts +115 -0
- package/src/executors/bounded-exec/host-memory.ts +70 -0
- package/src/executors/bounded-exec/leaf-peak.ts +151 -0
- package/src/executors/bounded-exec/peak-waiter.ts +82 -0
- package/src/executors/bounded-exec/pool-admission.ts +64 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +20 -6
- package/src/executors/bounded-exec/verdict.ts +30 -1
- package/test-timings.json +134 -132
|
@@ -8,6 +8,7 @@ import typia from 'typia';
|
|
|
8
8
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
9
9
|
import { holdLease, type Unleased } from './gateway-lease.js';
|
|
10
10
|
import { type CpuLeaseProgress, type CpuLeaseQueued, CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
|
|
11
|
+
import { admit } from './pool-admission.js';
|
|
11
12
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
12
13
|
|
|
13
14
|
/**
|
|
@@ -23,6 +24,11 @@ import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
|
23
24
|
* lasts while its connection does, so a child that dies returns its tokens with its socket. A pool
|
|
24
25
|
* grants only from its own count, and a pool whose upstream grant has gone grants nothing.
|
|
25
26
|
*
|
|
27
|
+
* A run's own pool also charges memory (`pool-admission.ts`; specs/cowshed/05_gateway.md, "Without mbx, every Nx run
|
|
28
|
+
* budgets itself"). A request names its task's last recorded peak as `memory`, in bytes, and is granted only while
|
|
29
|
+
* the charges of the leaves holding grants and its own fit the run's memory; a request that names none is charged
|
|
30
|
+
* nothing, as mbx's omitted `--memory` is. A sub-pool charges no memory: its holder's charge covers its tree.
|
|
31
|
+
*
|
|
26
32
|
* The same count is a GNU make jobserver fifo beside the socket, mode 0600, which Cargo, its build
|
|
27
33
|
* scripts and every nested `cargo` read through `CARGO_MAKEFLAGS`/`MAKEFLAGS` (specs/cowshed/05_gateway.md,
|
|
28
34
|
* "Cargo reads its holder's jobserver"). A pool of `g` holds `g − 1` token bytes there; the holder's own
|
|
@@ -55,17 +61,26 @@ export interface CpuPool {
|
|
|
55
61
|
|
|
56
62
|
export type CloseOutcome = { ok: true } | { ok: false; error: string };
|
|
57
63
|
|
|
58
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* The one line a child sends: a count of whole tokens, at least one, and the bytes it is charged, its task's last
|
|
66
|
+
* recorded memory peak. A child whose task has no recorded peak names none: it is charged nothing.
|
|
67
|
+
*/
|
|
59
68
|
interface CpuPoolRequest {
|
|
60
69
|
op: 'cpu-pool';
|
|
61
70
|
want: number & typia.tags.Type<'uint32'> & typia.tags.Minimum<1>;
|
|
71
|
+
memory?: number & typia.tags.Type<'uint64'>;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** A grant whose charge alone exceeds the run's memory, made while nothing else held memory: the run's memory. */
|
|
75
|
+
export interface Oversized {
|
|
76
|
+
readonly memory: number & typia.tags.Type<'uint64'>;
|
|
62
77
|
}
|
|
63
78
|
|
|
64
79
|
/** The pool's answers, in `holdLease`'s framing. */
|
|
65
80
|
type CpuPoolReply =
|
|
66
81
|
| CpuLeaseProgress
|
|
67
82
|
| CpuLeaseQueued
|
|
68
|
-
| { ok: true; lease: 'granted'; tokens: number }
|
|
83
|
+
| { ok: true; lease: 'granted'; tokens: number; oversized?: Oversized }
|
|
69
84
|
| { ok: false; code: 'invalid-request'; error: string };
|
|
70
85
|
|
|
71
86
|
/** Exactly a request's fields: any other field is an invalid request. */
|
|
@@ -94,7 +109,7 @@ const run = promisify(execFile);
|
|
|
94
109
|
type Phase =
|
|
95
110
|
| { readonly kind: 'reading' }
|
|
96
111
|
| { readonly kind: 'queued' }
|
|
97
|
-
| { readonly kind: 'holding'; readonly bytes: number; readonly slot: boolean }
|
|
112
|
+
| { readonly kind: 'holding'; readonly bytes: number; readonly slot: boolean; readonly memory: number }
|
|
98
113
|
| { readonly kind: 'done' };
|
|
99
114
|
|
|
100
115
|
/** A child connection and what the pool owes it. Its grant, or its place in line, ends with the connection. */
|
|
@@ -103,10 +118,11 @@ interface Child {
|
|
|
103
118
|
phase: Phase;
|
|
104
119
|
}
|
|
105
120
|
|
|
106
|
-
/** A queued child
|
|
121
|
+
/** A queued child, the count it asked for and the bytes it is charged. */
|
|
107
122
|
interface Waiter {
|
|
108
123
|
readonly child: Child;
|
|
109
124
|
readonly want: number;
|
|
125
|
+
readonly memory: number;
|
|
110
126
|
}
|
|
111
127
|
|
|
112
128
|
/** One framed reply line. The newline is the framing; every reply goes through here. */
|
|
@@ -127,21 +143,34 @@ export function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>):
|
|
|
127
143
|
}
|
|
128
144
|
|
|
129
145
|
/**
|
|
130
|
-
* Tokens a parent pool lent, held until released, or why there are none. `
|
|
131
|
-
*
|
|
146
|
+
* Tokens a parent pool lent, held until released, or why there are none. `oversized` names the run's memory when the
|
|
147
|
+
* child's charge alone exceeds it and it was granted because nothing else held memory. `closed` resolves when the
|
|
148
|
+
* grant's connection closes, however it closes, so a sub-pool built on the grant ends with it.
|
|
132
149
|
*/
|
|
133
|
-
export type TokenGrant =
|
|
150
|
+
export type TokenGrant =
|
|
151
|
+
| { granted: true; tokens: number; oversized: Oversized | null; release: () => void; closed: Promise<void> }
|
|
152
|
+
| Unleased;
|
|
153
|
+
|
|
154
|
+
const isOversized = typia.createEquals<Oversized>();
|
|
134
155
|
|
|
135
156
|
/**
|
|
136
|
-
* Ask the immediate parent pool for `want` tokens
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
157
|
+
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
158
|
+
* it has none. The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the
|
|
159
|
+
* child's local allocation: its connection is the grant, and closing it returns only that allocation. An absent
|
|
160
|
+
* endpoint grants nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission
|
|
161
|
+
* rather than run the child beside its parent's count.
|
|
140
162
|
*/
|
|
141
|
-
export async function takeParentCpuTokens(
|
|
163
|
+
export async function takeParentCpuTokens(
|
|
164
|
+
parent: InheritedCpuPool,
|
|
165
|
+
want: number,
|
|
166
|
+
memory: number | null,
|
|
167
|
+
): Promise<TokenGrant> {
|
|
142
168
|
if (!Number.isSafeInteger(want) || want < 1) {
|
|
143
169
|
throw new RangeError(`a child asks its parent pool for a positive count, not ${want} (pool ${parent.tokens})`);
|
|
144
170
|
}
|
|
171
|
+
if (memory !== null && (!Number.isSafeInteger(memory) || memory < 0)) {
|
|
172
|
+
throw new RangeError(`a child is charged a whole number of bytes, not ${memory}`);
|
|
173
|
+
}
|
|
145
174
|
if (parent.socket.length === 0 || parent.socket.includes('\0')) {
|
|
146
175
|
return {
|
|
147
176
|
granted: false,
|
|
@@ -149,11 +178,12 @@ export async function takeParentCpuTokens(parent: InheritedCpuPool, want: number
|
|
|
149
178
|
reason: `the parent CPU pool's endpoint ${JSON.stringify(parent.socket)} is not a socket path`,
|
|
150
179
|
};
|
|
151
180
|
}
|
|
152
|
-
const
|
|
181
|
+
const request = memory === null ? { op: 'cpu-pool', want } : { op: 'cpu-pool', want, memory };
|
|
182
|
+
const lease = await holdLease(parent.socket, request, 'the parent CPU pool', new CpuLeaseWait());
|
|
153
183
|
if (!lease.granted) {
|
|
154
184
|
return lease;
|
|
155
185
|
}
|
|
156
|
-
const tokens = lease.answer
|
|
186
|
+
const { tokens, oversized } = lease.answer;
|
|
157
187
|
if (typeof tokens !== 'number' || !Number.isInteger(tokens) || tokens < 1 || tokens > want) {
|
|
158
188
|
lease.release();
|
|
159
189
|
return {
|
|
@@ -162,7 +192,15 @@ export async function takeParentCpuTokens(parent: InheritedCpuPool, want: number
|
|
|
162
192
|
reason: `the parent CPU pool granted ${JSON.stringify(tokens)} tokens for ${want}`,
|
|
163
193
|
};
|
|
164
194
|
}
|
|
165
|
-
|
|
195
|
+
if (oversized !== undefined && !isOversized(oversized)) {
|
|
196
|
+
lease.release();
|
|
197
|
+
return {
|
|
198
|
+
granted: false,
|
|
199
|
+
cause: 'other',
|
|
200
|
+
reason: `the parent CPU pool named its memory ${JSON.stringify(oversized)} in a grant, which is no byte count`,
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
return { granted: true, tokens, oversized: oversized ?? null, release: lease.release, closed: lease.closed };
|
|
166
204
|
}
|
|
167
205
|
|
|
168
206
|
/** A thrown value's message. */
|
|
@@ -315,12 +353,14 @@ function watchFifo(
|
|
|
315
353
|
|
|
316
354
|
/**
|
|
317
355
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
318
|
-
* grants, ends its child connections, drains its jobserver fifo and removes its directory.
|
|
319
|
-
*
|
|
356
|
+
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
357
|
+
* pool charges its grants against `memory`, the run's memory in bytes; a sub-pool charges none (null).
|
|
358
|
+
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
320
359
|
*/
|
|
321
360
|
export async function createCpuPool(
|
|
322
361
|
tokens: number,
|
|
323
362
|
closed: Promise<void>,
|
|
363
|
+
memory: number | null,
|
|
324
364
|
): Promise<{ ok: true; pool: CpuPool } | { ok: false; error: string }> {
|
|
325
365
|
if (!Number.isSafeInteger(tokens) || tokens < 1) {
|
|
326
366
|
throw new RangeError(`a CPU pool holds a positive safe integer of tokens, not ${tokens}`);
|
|
@@ -347,6 +387,8 @@ export async function createCpuPool(
|
|
|
347
387
|
const scratch = Buffer.allocUnsafe(tokens);
|
|
348
388
|
// The holder's implicit first token, while a socket child holds it.
|
|
349
389
|
let slotLent = false;
|
|
390
|
+
// The bytes the children holding grants are charged, against `memory`.
|
|
391
|
+
let heldMemory = 0;
|
|
350
392
|
// The readability wait while the queue's head finds no token, else undefined.
|
|
351
393
|
let watch: FifoWatch | undefined;
|
|
352
394
|
let closing = false;
|
|
@@ -404,46 +446,62 @@ export async function createCpuPool(
|
|
|
404
446
|
}
|
|
405
447
|
};
|
|
406
448
|
|
|
407
|
-
// FIFO: the head takes `min(want, available)` as one grant once any token is available,
|
|
408
|
-
//
|
|
409
|
-
//
|
|
410
|
-
// child needs. Whatever bytes are in hand when no head can
|
|
449
|
+
// FIFO (`admit`): the head takes `min(want, available)` as one grant once any token is available and, in a run's
|
|
450
|
+
// pool, its charge fits beside the holders'; available is the fifo bytes it can read without blocking plus the
|
|
451
|
+
// implicit slot while no child holds it. `arrived` is bytes the readability wait already took. A waiting child holds
|
|
452
|
+
// nothing, so no child waits while it holds tokens another child needs. Whatever bytes are in hand when no head can
|
|
453
|
+
// use them go straight back to the fifo. A head that waits for memory watches the fifo for nothing: only a holder's
|
|
454
|
+
// close frees memory, and that close drains again.
|
|
411
455
|
function drain(arrived: number): void {
|
|
456
|
+
if (closing) {
|
|
457
|
+
stopWatching();
|
|
458
|
+
return;
|
|
459
|
+
}
|
|
460
|
+
// A child that half-closed or failed while queued leaves the line. Its `close` finds it gone from the queue.
|
|
461
|
+
for (let at = queue.length - 1; at >= 0; at -= 1) {
|
|
462
|
+
if (queue[at]?.child.socket.writable === false) queue.splice(at, 1);
|
|
463
|
+
}
|
|
412
464
|
let inHand = arrived;
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
}
|
|
423
|
-
if (inHand < waiter.want) {
|
|
424
|
-
try {
|
|
425
|
-
inHand += readTokens(jobserver, scratch, waiter.want - inHand);
|
|
426
|
-
} catch (error) {
|
|
427
|
-
fail(`its jobserver fifo could not be read: ${describe(error)}`);
|
|
428
|
-
return;
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
const granted = Math.min(waiter.want, inHand + (slotLent ? 0 : 1));
|
|
432
|
-
if (granted === 0) {
|
|
433
|
-
watchForTokens();
|
|
465
|
+
const wanted = Math.min(
|
|
466
|
+
queue.reduce((sum, waiter) => sum + waiter.want, 0),
|
|
467
|
+
scratch.length,
|
|
468
|
+
);
|
|
469
|
+
if (inHand < wanted) {
|
|
470
|
+
try {
|
|
471
|
+
inHand += readTokens(jobserver, scratch, wanted - inHand);
|
|
472
|
+
} catch (error) {
|
|
473
|
+
fail(`its jobserver fifo could not be read: ${describe(error)}`);
|
|
434
474
|
return;
|
|
435
475
|
}
|
|
436
|
-
|
|
476
|
+
}
|
|
477
|
+
const admission = admit(
|
|
478
|
+
queue,
|
|
479
|
+
inHand + (slotLent ? 0 : 1),
|
|
480
|
+
memory === null ? null : { capacity: memory, held: heldMemory },
|
|
481
|
+
);
|
|
482
|
+
queue.splice(0, admission.grants.length);
|
|
483
|
+
for (const { request, tokens: granted, oversized } of admission.grants) {
|
|
437
484
|
// Fifo bytes first, the implicit slot last.
|
|
438
485
|
const bytes = Math.min(granted, inHand);
|
|
439
486
|
inHand -= bytes;
|
|
440
487
|
const slot = granted > bytes;
|
|
441
488
|
if (slot) slotLent = true;
|
|
442
|
-
|
|
443
|
-
|
|
489
|
+
heldMemory += request.memory;
|
|
490
|
+
request.child.phase = { kind: 'holding', bytes, slot, memory: request.memory };
|
|
491
|
+
request.child.socket.write(
|
|
492
|
+
frame(
|
|
493
|
+
oversized && memory !== null
|
|
494
|
+
? { ok: true, lease: 'granted', tokens: granted, oversized: { memory } }
|
|
495
|
+
: { ok: true, lease: 'granted', tokens: granted },
|
|
496
|
+
),
|
|
497
|
+
);
|
|
498
|
+
}
|
|
499
|
+
if (admission.waiting === 'cpu') {
|
|
500
|
+
watchForTokens();
|
|
501
|
+
} else {
|
|
502
|
+
stopWatching();
|
|
444
503
|
}
|
|
445
|
-
|
|
446
|
-
if (!closing) returnTokens(inHand);
|
|
504
|
+
returnTokens(inHand);
|
|
447
505
|
}
|
|
448
506
|
|
|
449
507
|
const refuse = (child: Child, error: string): void => {
|
|
@@ -471,6 +529,7 @@ export async function createCpuPool(
|
|
|
471
529
|
} else if (child.phase.kind === 'holding') {
|
|
472
530
|
// A closing pool drains its fifo whole; until then the bytes go back for the next holder.
|
|
473
531
|
if (child.phase.slot) slotLent = false;
|
|
532
|
+
heldMemory -= child.phase.memory;
|
|
474
533
|
if (!closing) returnTokens(child.phase.bytes);
|
|
475
534
|
}
|
|
476
535
|
child.phase = { kind: 'done' };
|
|
@@ -523,12 +582,15 @@ export async function createCpuPool(
|
|
|
523
582
|
parsed = undefined;
|
|
524
583
|
}
|
|
525
584
|
if (!isRequest(parsed)) {
|
|
526
|
-
refuse(
|
|
585
|
+
refuse(
|
|
586
|
+
child,
|
|
587
|
+
'a request is {"op":"cpu-pool","want":<a whole number of tokens, at least 1>,"memory"?:<bytes it is charged>}',
|
|
588
|
+
);
|
|
527
589
|
return;
|
|
528
590
|
}
|
|
529
591
|
const request = parsed;
|
|
530
592
|
child.phase = { kind: 'queued' };
|
|
531
|
-
queue.push({ child, want: request.want });
|
|
593
|
+
queue.push({ child, want: request.want, memory: request.memory ?? 0 });
|
|
532
594
|
socket.write(frame({ ok: true, lease: 'queued', server: process.pid }));
|
|
533
595
|
drain(0);
|
|
534
596
|
});
|
|
@@ -36,6 +36,7 @@ import {
|
|
|
36
36
|
import { GATEWAY_SOCKET } from './gateway-lease.js';
|
|
37
37
|
import type { HangDiagnostics } from './hang-diagnostics.js';
|
|
38
38
|
import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
39
|
+
import { type LeafPeak, PEAK_FD, peakWaiter, readWaiterReport, underWaiter } from './leaf-peak.js';
|
|
39
40
|
import type { CpuWaitRow } from './lease-wait.js';
|
|
40
41
|
import type { LockWait } from './lock-wait.js';
|
|
41
42
|
import {
|
|
@@ -92,6 +93,7 @@ import {
|
|
|
92
93
|
judgeRun,
|
|
93
94
|
type NoSpaceSighting,
|
|
94
95
|
readJunitReport,
|
|
96
|
+
readLastPeak,
|
|
95
97
|
taskDirectory,
|
|
96
98
|
withJunitReport,
|
|
97
99
|
writeRecord,
|
|
@@ -319,6 +321,8 @@ export async function runBoundedExec(
|
|
|
319
321
|
): Promise<BoundedExecResult> {
|
|
320
322
|
const cwd = resolveCwd(options.cwd, context.root);
|
|
321
323
|
const directory = context.record?.directory ?? null;
|
|
324
|
+
// The task's last recorded peak is its charge in its parent's pool, read before the clear below removes it.
|
|
325
|
+
const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
|
|
322
326
|
// A previous run's verdict and report go first, so neither can be read as this run's.
|
|
323
327
|
if (directory !== null) {
|
|
324
328
|
await clearTaskDirectory(directory);
|
|
@@ -366,6 +370,20 @@ export async function runBoundedExec(
|
|
|
366
370
|
// Nobody is left to tell.
|
|
367
371
|
}
|
|
368
372
|
};
|
|
373
|
+
// A run refused before its command exists says why and records an operational failure, and no peak: nothing ran.
|
|
374
|
+
const refuseUnstarted = async (message: string): Promise<BoundedExecResult> => {
|
|
375
|
+
appendStderr(message);
|
|
376
|
+
if (context.record !== undefined) {
|
|
377
|
+
const { task, directory: recordDirectory } = context.record;
|
|
378
|
+
await writeRecord(recordDirectory, {
|
|
379
|
+
task: task.id,
|
|
380
|
+
hash: task.hash,
|
|
381
|
+
verdict: judgeOperationalFailure(1, null),
|
|
382
|
+
peakBytes: null,
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
return { success: false, terminalOutput: outputChunks.join('') };
|
|
386
|
+
};
|
|
369
387
|
|
|
370
388
|
// The fixture pair names one segment: a fixture with no build budget, or a budget with no fixture, is a
|
|
371
389
|
// target that does not say what it runs, so it does not run. Units are counted at the end of that segment.
|
|
@@ -374,14 +392,9 @@ export async function runBoundedExec(
|
|
|
374
392
|
(options.unitsCommand !== undefined && options.fixtureCommand === undefined) ||
|
|
375
393
|
(fixture !== null && !(fixture.timeoutMs > 0))
|
|
376
394
|
) {
|
|
377
|
-
|
|
395
|
+
return refuseUnstarted(
|
|
378
396
|
'bounded-exec: fixtureCommand and a positive fixtureTimeoutMs must be named together, and unitsCommand only with them; refusing to run the command\n',
|
|
379
397
|
);
|
|
380
|
-
if (context.record !== undefined) {
|
|
381
|
-
const { task, directory: recordDirectory } = context.record;
|
|
382
|
-
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
383
|
-
}
|
|
384
|
-
return { success: false, terminalOutput: outputChunks.join('') };
|
|
385
398
|
}
|
|
386
399
|
|
|
387
400
|
// A leg that runs bun runs the Bun its workspace declares, or it does not run. This comes before the RAM lease and the
|
|
@@ -394,13 +407,15 @@ export async function runBoundedExec(
|
|
|
394
407
|
pathSource: options.env?.PATH === undefined ? 'the inherited environment' : "the target's env",
|
|
395
408
|
});
|
|
396
409
|
if (runtime !== null) {
|
|
397
|
-
|
|
398
|
-
if (context.record !== undefined) {
|
|
399
|
-
const { task, directory: recordDirectory } = context.record;
|
|
400
|
-
await writeRecord(recordDirectory, { task: task.id, hash: task.hash, verdict: judgeOperationalFailure(1, null) });
|
|
401
|
-
}
|
|
402
|
-
return { success: false, terminalOutput: outputChunks.join('') };
|
|
410
|
+
return refuseUnstarted(describeRuntimePinRefusal(runtime, context.record?.task.id));
|
|
403
411
|
}
|
|
412
|
+
if (lastPeak !== null && !lastPeak.ok) {
|
|
413
|
+
return refuseUnstarted(
|
|
414
|
+
`bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`,
|
|
415
|
+
);
|
|
416
|
+
}
|
|
417
|
+
// A task with no recorded peak is charged nothing (null) until it has one.
|
|
418
|
+
const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
|
|
404
419
|
|
|
405
420
|
let lease: RamTempLease | null = null;
|
|
406
421
|
let ordinaryTemp: string | null = null;
|
|
@@ -548,7 +563,7 @@ export async function runBoundedExec(
|
|
|
548
563
|
let parentRefusal: string | null = null;
|
|
549
564
|
let parentWait: CpuWaitRow | undefined;
|
|
550
565
|
if (inherited !== null) {
|
|
551
|
-
const fromParent = await takeParentCpuTokens(inherited, demand.want);
|
|
566
|
+
const fromParent = await takeParentCpuTokens(inherited, demand.want, charge);
|
|
552
567
|
if (fromParent.granted) {
|
|
553
568
|
grant = fromParent;
|
|
554
569
|
} else if (fromParent.cause === 'absent') {
|
|
@@ -569,8 +584,13 @@ export async function runBoundedExec(
|
|
|
569
584
|
sized = sizeRunner(demand, requested, grant.tokens);
|
|
570
585
|
cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
|
|
571
586
|
appendStderr(
|
|
572
|
-
`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind}\n`,
|
|
587
|
+
`cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`,
|
|
573
588
|
);
|
|
589
|
+
if (grant.oversized !== null) {
|
|
590
|
+
appendStderr(
|
|
591
|
+
`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
|
|
592
|
+
);
|
|
593
|
+
}
|
|
574
594
|
} else if (leaf?.kind === 'reserve') {
|
|
575
595
|
sized = { command: sizeRunnerToGrant(leaf.demand, requested), env: {} };
|
|
576
596
|
} else if (leaf !== null) {
|
|
@@ -633,6 +653,7 @@ export async function runBoundedExec(
|
|
|
633
653
|
operational: boolean,
|
|
634
654
|
noSpace: NoSpaceSighting | null,
|
|
635
655
|
lockWaits: readonly LockWait[],
|
|
656
|
+
peakBytes: number | null,
|
|
636
657
|
): Promise<void> => {
|
|
637
658
|
if (context.record === undefined) {
|
|
638
659
|
return;
|
|
@@ -678,6 +699,7 @@ export async function runBoundedExec(
|
|
|
678
699
|
verdict,
|
|
679
700
|
...(cpu ? { cpu } : {}),
|
|
680
701
|
...(parentWait ? { cpuWait: parentWait } : {}),
|
|
702
|
+
peakBytes,
|
|
681
703
|
});
|
|
682
704
|
};
|
|
683
705
|
// A run refused before its command starts says why, gives back its grant and RAM, and records the failure.
|
|
@@ -686,7 +708,7 @@ export async function runBoundedExec(
|
|
|
686
708
|
await releaseCpu();
|
|
687
709
|
// The command never started, so its lease holds nothing of its own to keep.
|
|
688
710
|
await releaseLease(false);
|
|
689
|
-
await recordRun(1, null, Date.now() - asked, false, null, []);
|
|
711
|
+
await recordRun(1, null, Date.now() - asked, false, null, [], null);
|
|
690
712
|
return { success: false, terminalOutput: outputChunks.join('') };
|
|
691
713
|
};
|
|
692
714
|
if (parentRefusal !== null) {
|
|
@@ -704,7 +726,7 @@ export async function runBoundedExec(
|
|
|
704
726
|
// A granted run serves a sub-pool of its own grant, so its descendants draw from that and never from above it.
|
|
705
727
|
// A pool that cannot be set up refuses the run: no granted command runs beside its grant.
|
|
706
728
|
if (grant !== null) {
|
|
707
|
-
const created = await createCpuPool(grant.tokens, grant.closed);
|
|
729
|
+
const created = await createCpuPool(grant.tokens, grant.closed, null);
|
|
708
730
|
if (!created.ok) {
|
|
709
731
|
return refuse(`cowshed: cpu pool unavailable: ${created.error}; refusing to run the command\n`);
|
|
710
732
|
}
|
|
@@ -756,38 +778,67 @@ export async function runBoundedExec(
|
|
|
756
778
|
const grantSlots: IOType[] = leaf !== null ? ['ignore', 'pipe'] : [];
|
|
757
779
|
const rootGrantSlot: IOType[] = leaf !== null ? ['pipe'] : [];
|
|
758
780
|
const root = owner?.rootArgv(shellCommand);
|
|
781
|
+
// The command's own descriptors past stderr, from fd 3: the lifecycle channel, the subtree admission, mbx's grant.
|
|
782
|
+
const descriptors: IOType[] =
|
|
783
|
+
root === undefined
|
|
784
|
+
? hooked === null && leaf === null
|
|
785
|
+
? []
|
|
786
|
+
: [hooked === null ? 'ignore' : 'pipe', ...grantSlots]
|
|
787
|
+
: [hooked === null ? 'ignore' : 'pipe', 'pipe', ...rootGrantSlot];
|
|
788
|
+
// A run that records its verdict records its command's peak too: the command runs under a waiter that reports it.
|
|
789
|
+
const waiter = context.record === undefined ? null : peakWaiter();
|
|
790
|
+
const measured =
|
|
791
|
+
waiter?.kind === 'waiter'
|
|
792
|
+
? underWaiter(waiter.runtime, root?.file ?? '/bin/sh', root?.args ?? ['-c', shellCommand], descriptors)
|
|
793
|
+
: null;
|
|
759
794
|
let child: ChildProcess;
|
|
760
795
|
const results = directory === null ? null : await createTestResultWriter(directory, { cwd, root: context.root });
|
|
761
796
|
if (results !== null && !results.ok) return refuse(`bounded-exec: ${results.error}\n`);
|
|
762
797
|
resultWriter = results?.ok ? results.writer : null;
|
|
763
798
|
try {
|
|
764
799
|
// fd 3 is the runner lifecycle's duplex socket when the command is hooked; the subtree admission takes fd 4, which
|
|
765
|
-
// cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant
|
|
800
|
+
// cowshed closes fd 4 before exec; fd 5 is a Cargo leaf's mbx admission stream or a reserved leaf's ask and grant;
|
|
801
|
+
// the waiter, when there is one, passes them on and reports the command's peak on fd 6.
|
|
766
802
|
child =
|
|
767
|
-
|
|
768
|
-
? spawn(
|
|
803
|
+
measured !== null
|
|
804
|
+
? spawn(measured.file, measured.args, {
|
|
769
805
|
cwd,
|
|
770
806
|
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
771
|
-
|
|
772
|
-
stdio:
|
|
773
|
-
hooked === null && leaf === null
|
|
774
|
-
? 'pipe'
|
|
775
|
-
: ['pipe', 'pipe', 'pipe', hooked === null ? 'ignore' : 'pipe', ...grantSlots],
|
|
807
|
+
stdio: measured.stdio,
|
|
776
808
|
detached: process.platform !== 'win32',
|
|
777
809
|
windowsHide: true,
|
|
778
810
|
})
|
|
779
|
-
:
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
811
|
+
: root === undefined
|
|
812
|
+
? spawn(shellCommand, [], {
|
|
813
|
+
cwd,
|
|
814
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
815
|
+
shell: true,
|
|
816
|
+
stdio: descriptors.length === 0 ? 'pipe' : ['pipe', 'pipe', 'pipe', ...descriptors],
|
|
817
|
+
detached: process.platform !== 'win32',
|
|
818
|
+
windowsHide: true,
|
|
819
|
+
})
|
|
820
|
+
: spawn(root.file, root.args, {
|
|
821
|
+
cwd,
|
|
822
|
+
env: mergeEnv(lease?.directory ?? ordinaryTemp, env),
|
|
823
|
+
stdio: ['pipe', 'pipe', 'pipe', ...descriptors],
|
|
824
|
+
detached: process.platform !== 'win32',
|
|
825
|
+
windowsHide: true,
|
|
826
|
+
});
|
|
786
827
|
} catch (error) {
|
|
787
828
|
// Nothing started, so nothing runs under the grant: it goes back here, with the RAM lease.
|
|
788
829
|
return refuse(`cowshed: could not start the command: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
789
830
|
}
|
|
790
831
|
subtreeRetired = false;
|
|
832
|
+
const reportChannel = child.stdio.at(PEAK_FD);
|
|
833
|
+
const waiterReport =
|
|
834
|
+
measured === null ? null : readWaiterReport(reportChannel instanceof Readable ? reportChannel : null);
|
|
835
|
+
// The leaf's runner is its command: under a waiter, which leads the group, the child the waiter names; else the
|
|
836
|
+
// group's leader itself. A hang snapshot names and samples it (specs/cowshed/08_testing.md).
|
|
837
|
+
const runnerPid: Promise<number | null> = waiterReport?.started ?? Promise.resolve(child.pid ?? null);
|
|
838
|
+
let runner: number | null = null;
|
|
839
|
+
void runnerPid.then((pid) => {
|
|
840
|
+
runner = pid;
|
|
841
|
+
});
|
|
791
842
|
if (root !== undefined) {
|
|
792
843
|
const channel = child.stdio[HANDLE_FD];
|
|
793
844
|
const leased = lease;
|
|
@@ -847,22 +898,24 @@ export async function runBoundedExec(
|
|
|
847
898
|
captureTimer = setTimeout(
|
|
848
899
|
() => {
|
|
849
900
|
if (state.settled || state.expiry !== null || idleCapture !== capture) return;
|
|
850
|
-
const
|
|
851
|
-
if (
|
|
901
|
+
const group = child.pid;
|
|
902
|
+
if (group === undefined) {
|
|
852
903
|
idleCapture.result = { kind: 'failed', detail: 'runner PID unavailable; no holder conclusion' };
|
|
853
904
|
return;
|
|
854
905
|
}
|
|
855
906
|
idleCapture.result = { kind: 'pending', startedAtMs: Date.now() };
|
|
856
|
-
void
|
|
857
|
-
.then(() =>
|
|
858
|
-
|
|
907
|
+
void runnerPid
|
|
908
|
+
.then((pid) => {
|
|
909
|
+
if (pid === null)
|
|
910
|
+
throw new Error('runner PID unavailable: its waiter named no command; no holder conclusion');
|
|
911
|
+
return captureDiagnostics({
|
|
859
912
|
pid,
|
|
860
|
-
pgid: process.platform === 'win32' ? null :
|
|
913
|
+
pgid: process.platform === 'win32' ? null : group,
|
|
861
914
|
expiresAtMs: capture.expiresAtMs,
|
|
862
915
|
deadlineMs: Math.max(0, Math.min(HANG_CAPTURE_MS, capture.expiresAtMs - Date.now())),
|
|
863
916
|
signal: capture.controller.signal,
|
|
864
|
-
})
|
|
865
|
-
)
|
|
917
|
+
});
|
|
918
|
+
})
|
|
866
919
|
.then(
|
|
867
920
|
(snapshot) => {
|
|
868
921
|
if (idleCapture === capture) idleCapture.result = { kind: 'captured', snapshot };
|
|
@@ -1115,6 +1168,7 @@ export async function runBoundedExec(
|
|
|
1115
1168
|
state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null,
|
|
1116
1169
|
state.noSpace === null ? null : await state.noSpace,
|
|
1117
1170
|
state.lockWaits,
|
|
1171
|
+
null,
|
|
1118
1172
|
),
|
|
1119
1173
|
)
|
|
1120
1174
|
.finally(() => process.kill(process.pid, signal));
|
|
@@ -1183,7 +1237,7 @@ export async function runBoundedExec(
|
|
|
1183
1237
|
const observed = idleCapture?.result;
|
|
1184
1238
|
state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
|
|
1185
1239
|
appendStderr(
|
|
1186
|
-
`Hang diagnostics: runner pid=${
|
|
1240
|
+
`Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${
|
|
1187
1241
|
process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
|
|
1188
1242
|
} expiryAtMs=${Date.now()} scheduledExpiryAtMs=${idleCapture?.expiresAtMs ?? 'unavailable'}\n`,
|
|
1189
1243
|
);
|
|
@@ -1420,10 +1474,18 @@ export async function runBoundedExec(
|
|
|
1420
1474
|
await releaseLease(code !== 0 || state.expiry !== null || state.poolCloseError !== null || lifecycleFailure !== null);
|
|
1421
1475
|
releaseReservation();
|
|
1422
1476
|
|
|
1477
|
+
// The command's peak, as its waiter reported it once both ended; a peak that was not measured is said, never zero.
|
|
1478
|
+
const reported: LeafPeak | null =
|
|
1479
|
+
waiterReport === null ? (waiter?.kind === 'unmeasured' ? waiter : null) : await waiterReport.peak;
|
|
1480
|
+
if (reported?.kind === 'unmeasured') {
|
|
1481
|
+
appendStderr(`bounded-exec: this run's memory peak was not measured (${reported.reason}); its record names none\n`);
|
|
1482
|
+
}
|
|
1483
|
+
const peakBytes = reported?.kind === 'measured' ? reported.bytes : null;
|
|
1484
|
+
|
|
1423
1485
|
// Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
|
|
1424
1486
|
// branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
|
|
1425
1487
|
const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
|
|
1426
|
-
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits);
|
|
1488
|
+
await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
|
|
1427
1489
|
return {
|
|
1428
1490
|
success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
|
|
1429
1491
|
terminalOutput: outputChunks.join(''),
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { promisify } from 'node:util';
|
|
3
|
+
import type { HostMemory, RunMemoryReading } from './host-memory.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* macOS's memory mechanisms. A run's memory is the host's, `hw.memsize`: macOS confines no job to a memory limit of
|
|
7
|
+
* its own. Bun reports a leaf's `ru_maxrss` in bytes, as macOS does.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
const run = promisify(execFile);
|
|
11
|
+
const HW_MEMSIZE = 'hw.memsize';
|
|
12
|
+
|
|
13
|
+
async function readRunMemory(): Promise<RunMemoryReading> {
|
|
14
|
+
let stdout: string;
|
|
15
|
+
try {
|
|
16
|
+
({ stdout } = await run('sysctl', ['-n', HW_MEMSIZE]));
|
|
17
|
+
} catch (error) {
|
|
18
|
+
return {
|
|
19
|
+
ok: false,
|
|
20
|
+
error: {
|
|
21
|
+
operation: 'read-run-memory',
|
|
22
|
+
code: 'unreadable',
|
|
23
|
+
subject: HW_MEMSIZE,
|
|
24
|
+
cause: error instanceof Error ? error.message : String(error),
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
const value = stdout.trim();
|
|
29
|
+
const bytes = Number(value);
|
|
30
|
+
if (!/^\d+$/.test(value) || !Number.isSafeInteger(bytes) || bytes === 0) {
|
|
31
|
+
return {
|
|
32
|
+
ok: false,
|
|
33
|
+
error: {
|
|
34
|
+
operation: 'read-run-memory',
|
|
35
|
+
code: 'malformed',
|
|
36
|
+
subject: HW_MEMSIZE,
|
|
37
|
+
cause: `sysctl printed ${JSON.stringify(value)}, not a byte count`,
|
|
38
|
+
},
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
return { ok: true, memory: { bytes, source: HW_MEMSIZE } };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export const darwinMemory: HostMemory = { readRunMemory, peakBytes: (maxRss) => maxRss };
|