@smoothbricks/nx-plugin 0.7.10 → 0.7.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +30 -21
  2. package/dist/.build-inputs.json +9 -9
  3. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
  4. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  5. package/dist/executors/bounded-exec/cpu-pool.d.ts +25 -10
  6. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  7. package/dist/executors/bounded-exec/cpu-pool.js +86 -23
  8. package/dist/executors/bounded-exec/executor.d.ts +5 -0
  9. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  10. package/dist/executors/bounded-exec/executor.js +44 -5
  11. package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
  12. package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
  13. package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
  14. package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
  15. package/dist/executors/bounded-exec/host-memory.js +6 -0
  16. package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
  17. package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
  18. package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
  19. package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
  20. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
  21. package/dist/executors/bounded-exec/pool-admission.js +44 -8
  22. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  23. package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
  24. package/dist/executors/bounded-exec/verdict.d.ts +4 -2
  25. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  26. package/dist/executors/bounded-exec/verdict.js +19 -2
  27. package/managed/raw/patches/nx@24.0.3.patch +16 -0
  28. package/managed/raw/patches/ttsc@0.34.1.patch +189 -2
  29. package/package.json +1 -1
  30. package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
  31. package/src/executors/bounded-exec/cpu-pool.ts +121 -25
  32. package/src/executors/bounded-exec/executor.ts +71 -5
  33. package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
  34. package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
  35. package/src/executors/bounded-exec/host-memory.ts +25 -0
  36. package/src/executors/bounded-exec/pool-admission.ts +66 -11
  37. package/src/executors/bounded-exec/run-cpu-pool.ts +3 -3
  38. package/src/executors/bounded-exec/verdict.ts +25 -5
  39. package/test-timings.json +2 -0
@@ -1,5 +1,6 @@
1
1
  import { parentPort, workerData } from 'node:worker_threads';
2
2
  import { type CloseOutcome, createCpuPool } from './cpu-pool.js';
3
+ import type { RunMemory } from './host-memory.js';
3
4
 
4
5
  /**
5
6
  * The thread an Nx run's CPU pool runs on (`run-cpu-pool.ts`; specs/cowshed/05_gateway.md, "Every Nx run budgets
@@ -9,14 +10,14 @@ import { type CloseOutcome, createCpuPool } from './cpu-pool.js';
9
10
  * jobserver fifo and heartbeat share an event loop that runs nothing else, so the heartbeat proves that the actor which
10
11
  * admits and grants is turning.
11
12
  *
12
- * The thread is started with the pool's count and the run's memory. It says once that the pool serves, with the environment a task inherits,
13
- * or why it cannot; the host's one message is its close, after which the thread says how the pool closed.
13
+ * The thread is started with the pool's count and the run's memory. It says once that the pool serves, with the environment a
14
+ * task inherits, or why it cannot; the host's one message is its close, after which the thread says how the pool closed.
14
15
  */
15
16
 
16
- /** What the thread is started with: the run's count, and the run's memory in bytes its grants are charged against. */
17
+ /** What the thread is started with: the run's count, and the run's memory its grants are charged against. */
17
18
  export interface CpuPoolThreadData {
18
19
  readonly tokens: number;
19
- readonly memory: number;
20
+ readonly memory: RunMemory;
20
21
  }
21
22
 
22
23
  /** What the thread says, in order: the pool serves or failed; after the host's close, how the pool closed. */
@@ -7,6 +7,7 @@ import { promisify } from 'node:util';
7
7
  import typia from 'typia';
8
8
  import { ownedSocketDirectory } from '../../testing.js';
9
9
  import { holdLease, type Unleased } from './gateway-lease.js';
10
+ import { type LiveUsageReading, type RunMemory, readLiveUsage } from './host-memory.js';
10
11
  import { type CpuLeaseProgress, type CpuLeaseQueued, CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
11
12
  import { admit } from './pool-admission.js';
12
13
  import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
@@ -63,12 +64,16 @@ export type CloseOutcome = { ok: true } | { ok: false; error: string };
63
64
 
64
65
  /**
65
66
  * The one line a child sends: a count of whole tokens, at least one, and the bytes it is charged, its task's last
66
- * recorded memory peak. A child whose task has no recorded peak names none: it is charged nothing.
67
+ * recorded memory peak, with the tokens it was lent when that peak was measured. A child whose task has no recorded peak
68
+ * names none: it is charged nothing. A peak with no tokens is charged flat.
67
69
  */
68
70
  interface CpuPoolRequest {
69
71
  op: 'cpu-pool';
70
72
  want: number & typia.tags.Type<'uint32'> & typia.tags.Minimum<1>;
71
73
  memory?: number & typia.tags.Type<'uint64'>;
74
+ parallelism?: number & typia.tags.Type<'uint32'> & typia.tags.Minimum<1>;
75
+ task?: string & typia.tags.MaxLength<256>;
76
+ alone?: boolean;
72
77
  }
73
78
 
74
79
  /** A grant whose charge alone exceeds the run's memory, made while nothing else held memory: the run's memory. */
@@ -80,7 +85,7 @@ export interface Oversized {
80
85
  type CpuPoolReply =
81
86
  | CpuLeaseProgress
82
87
  | CpuLeaseQueued
83
- | { ok: true; lease: 'granted'; tokens: number; oversized?: Oversized }
88
+ | { ok: true; lease: 'granted'; tokens: number; oversized?: Oversized; charged?: number }
84
89
  | { ok: false; code: 'invalid-request'; error: string };
85
90
 
86
91
  /** Exactly a request's fields: any other field is an invalid request. */
@@ -109,7 +114,13 @@ const run = promisify(execFile);
109
114
  type Phase =
110
115
  | { readonly kind: 'reading' }
111
116
  | { readonly kind: 'queued' }
112
- | { readonly kind: 'holding'; readonly bytes: number; readonly slot: boolean; readonly memory: number }
117
+ | {
118
+ readonly kind: 'holding';
119
+ readonly bytes: number;
120
+ readonly slot: boolean;
121
+ readonly memory: number;
122
+ readonly alone: boolean;
123
+ }
113
124
  | { readonly kind: 'done' };
114
125
 
115
126
  /** A child connection and what the pool owes it. Its grant, or its place in line, ends with the connection. */
@@ -118,11 +129,14 @@ interface Child {
118
129
  phase: Phase;
119
130
  }
120
131
 
121
- /** A queued child, the count it asked for and the bytes it is charged. */
132
+ /** A queued child, the count it asked for, the bytes it is charged, the tokens that charge was measured under, its task. */
122
133
  interface Waiter {
123
134
  readonly child: Child;
124
135
  readonly want: number;
125
136
  readonly memory: number;
137
+ readonly parallelism?: number | undefined;
138
+ readonly task?: string | undefined;
139
+ readonly alone?: boolean | undefined;
126
140
  }
127
141
 
128
142
  /** One framed reply line. The newline is the framing; every reply goes through here. */
@@ -143,34 +157,59 @@ export function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>):
143
157
  }
144
158
 
145
159
  /**
146
- * Tokens a parent pool lent, held until released, or why there are none. `oversized` names the run's memory when the
147
- * child's charge alone exceeds it and it was granted because nothing else held memory. `closed` resolves when the
148
- * grant's connection closes, however it closes, so a sub-pool built on the grant ends with it.
160
+ * Tokens a parent pool lent, held until released, or why there are none. `charged` is the bytes the pool charged the
161
+ * grant, null from a pool that charges none. `oversized` names the run's memory when the child's charge alone exceeds
162
+ * it and it was granted because nothing else held memory. `closed` resolves when the grant's connection closes, however
163
+ * it closes, so a sub-pool built on the grant ends with it.
149
164
  */
150
165
  export type TokenGrant =
151
- | { granted: true; tokens: number; oversized: Oversized | null; release: () => void; closed: Promise<void> }
166
+ | {
167
+ granted: true;
168
+ tokens: number;
169
+ charged: number | null;
170
+ oversized: Oversized | null;
171
+ release: () => void;
172
+ closed: Promise<void>;
173
+ }
152
174
  | Unleased;
153
175
 
154
176
  const isOversized = typia.createEquals<Oversized>();
155
177
 
178
+ /** What a leaf tells the pool beyond its count and its charge. */
179
+ export interface AskContext {
180
+ /** The tokens the task was lent when its recorded peak was measured. */
181
+ readonly parallelism?: number | undefined;
182
+ /** The Nx task the leaf runs, named in the pool's line for the grant. */
183
+ readonly task?: string | undefined;
184
+ /** The kernel killed the command for memory: it is run again, granted only while nothing else holds a grant. */
185
+ readonly alone?: boolean | undefined;
186
+ }
187
+
156
188
  /**
157
189
  * Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
158
- * it has none. The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the
159
- * child's local allocation: its connection is the grant, and closing it returns only that allocation. An absent
160
- * endpoint grants nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission
161
- * rather than run the child beside its parent's count.
190
+ * it has none. `context.parallelism` is the tokens the task was lent when that peak was measured; the pool charges the
191
+ * peak in proportion to the tokens it lends now when that is more, and lends fewer when the run's memory holds no more.
192
+ * The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the child's local
193
+ * allocation: its connection is the grant, and closing it returns only that allocation. An absent endpoint grants
194
+ * nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission rather than run
195
+ * the child beside its parent's count.
162
196
  */
163
197
  export async function takeParentCpuTokens(
164
198
  parent: InheritedCpuPool,
165
199
  want: number,
166
200
  memory: number | null,
201
+ context: AskContext = {},
167
202
  ): Promise<TokenGrant> {
203
+ const { parallelism, task, alone } = context;
168
204
  if (!Number.isSafeInteger(want) || want < 1) {
169
205
  throw new RangeError(`a child asks its parent pool for a positive count, not ${want} (pool ${parent.tokens})`);
170
206
  }
171
207
  if (memory !== null && (!Number.isSafeInteger(memory) || memory < 0)) {
172
208
  throw new RangeError(`a child is charged a whole number of bytes, not ${memory}`);
173
209
  }
210
+ if (parallelism !== undefined && (!Number.isSafeInteger(parallelism) || parallelism < 1)) {
211
+ throw new RangeError(`a child's charge is measured under a positive count of tokens, not ${parallelism}`);
212
+ }
174
213
  if (parent.socket.length === 0 || parent.socket.includes('\0')) {
175
214
  return {
176
215
  granted: false,
@@ -178,12 +217,18 @@ export async function takeParentCpuTokens(
178
217
  reason: `the parent CPU pool's endpoint ${JSON.stringify(parent.socket)} is not a socket path`,
179
218
  };
180
219
  }
181
- const request = memory === null ? { op: 'cpu-pool', want } : { op: 'cpu-pool', want, memory };
220
+ const request = {
221
+ op: 'cpu-pool',
222
+ want,
223
+ ...(memory === null ? {} : { memory, ...(parallelism === undefined ? {} : { parallelism }) }),
224
+ ...(task === undefined ? {} : { task }),
225
+ ...(alone === true ? { alone: true } : {}),
226
+ };
182
227
  const lease = await holdLease(parent.socket, request, 'the parent CPU pool', new CpuLeaseWait());
183
228
  if (!lease.granted) {
184
229
  return lease;
185
230
  }
186
- const { tokens, oversized } = lease.answer;
231
+ const { tokens, oversized, charged } = lease.answer;
187
232
  if (typeof tokens !== 'number' || !Number.isInteger(tokens) || tokens < 1 || tokens > want) {
188
233
  lease.release();
189
234
  return {
@@ -200,7 +245,22 @@ export async function takeParentCpuTokens(
200
245
  reason: `the parent CPU pool named its memory ${JSON.stringify(oversized)} in a grant, which is no byte count`,
201
246
  };
202
247
  }
203
- return { granted: true, tokens, oversized: oversized ?? null, release: lease.release, closed: lease.closed };
248
+ if (charged !== undefined && !(typeof charged === 'number' && Number.isSafeInteger(charged) && charged >= 0)) {
249
+ lease.release();
250
+ return {
251
+ granted: false,
252
+ cause: 'other',
253
+ reason: `the parent CPU pool charged ${JSON.stringify(charged)} in a grant, which is no byte count`,
254
+ };
255
+ }
256
+ return {
257
+ granted: true,
258
+ tokens,
259
+ charged: charged ?? null,
260
+ oversized: oversized ?? null,
261
+ release: lease.release,
262
+ closed: lease.closed,
263
+ };
204
264
  }
205
265
 
206
266
  /** A thrown value's message. */
@@ -354,13 +414,15 @@ function watchFifo(
354
414
  /**
355
415
  * Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
356
416
  * grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
357
- * pool charges its grants against `memory`, the run's memory in bytes; a sub-pool charges none (null).
417
+ * pool charges its grants against `memory` and says each grant with what the run's bound holds live
418
+ * (`liveUsage`); a sub-pool charges none (null).
358
419
  * Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
359
420
  */
360
421
  export async function createCpuPool(
361
422
  tokens: number,
362
423
  closed: Promise<void>,
363
- memory: number | null,
424
+ memory: RunMemory | null,
425
+ liveUsage: (bound: string) => LiveUsageReading = readLiveUsage,
364
426
  ): Promise<{ ok: true; pool: CpuPool } | { ok: false; error: string }> {
365
427
  if (!Number.isSafeInteger(tokens) || tokens < 1) {
366
428
  throw new RangeError(`a CPU pool holds a positive safe integer of tokens, not ${tokens}`);
@@ -389,6 +451,9 @@ export async function createCpuPool(
389
451
  let slotLent = false;
390
452
  // The bytes the children holding grants are charged, against `memory`.
391
453
  let heldMemory = 0;
454
+ // The children holding a grant, and whether one of them runs alone.
455
+ let holders = 0;
456
+ let aloneHolder = false;
392
457
  // The readability wait while the queue's head finds no token, else undefined.
393
458
  let watch: FifoWatch | undefined;
394
459
  let closing = false;
@@ -477,24 +542,39 @@ export async function createCpuPool(
477
542
  const admission = admit(
478
543
  queue,
479
544
  inHand + (slotLent ? 0 : 1),
480
- memory === null ? null : { capacity: memory, held: heldMemory },
545
+ memory === null ? null : { capacity: memory.bytes, held: heldMemory },
546
+ { holders, alone: aloneHolder },
481
547
  );
482
548
  queue.splice(0, admission.grants.length);
483
- for (const { request, tokens: granted, oversized } of admission.grants) {
549
+ for (const { request, tokens: granted, charge, oversized } of admission.grants) {
484
550
  // Fifo bytes first, the implicit slot last.
485
551
  const bytes = Math.min(granted, inHand);
486
552
  inHand -= bytes;
487
553
  const slot = granted > bytes;
488
554
  if (slot) slotLent = true;
489
- heldMemory += request.memory;
490
- request.child.phase = { kind: 'holding', bytes, slot, memory: request.memory };
555
+ heldMemory += charge;
556
+ holders += 1;
557
+ if (request.alone === true) aloneHolder = true;
558
+ request.child.phase = { kind: 'holding', bytes, slot, memory: charge, alone: request.alone === true };
491
559
  request.child.socket.write(
492
560
  frame(
493
- oversized && memory !== null
494
- ? { ok: true, lease: 'granted', tokens: granted, oversized: { memory } }
495
- : { ok: true, lease: 'granted', tokens: granted },
561
+ memory === null
562
+ ? { ok: true, lease: 'granted', tokens: granted }
563
+ : oversized
564
+ ? { ok: true, lease: 'granted', tokens: granted, charged: charge, oversized: { memory: memory.bytes } }
565
+ : { ok: true, lease: 'granted', tokens: granted, charged: charge },
496
566
  ),
497
567
  );
568
+
569
+ if (memory !== null) {
570
+ const live = liveUsage(memory.source);
571
+ const held = live.ok
572
+ ? `${live.usage.anon + live.usage.shmem} bytes (anon ${live.usage.anon}, shmem ${live.usage.shmem}, ${live.usage.source})`
573
+ : `unread (${live.reason})`;
574
+ process.stderr.write(
575
+ `bounded-exec: the CPU pool at ${socketPath} granted ${request.task ?? 'a leaf'} ${granted} of ${request.want} tokens charged ${charge} bytes: holders charged ${heldMemory} of ${memory.bytes} bytes, live anon+shmem ${held}\n`,
576
+ );
577
+ }
498
578
  }
499
579
  if (admission.waiting === 'cpu') {
500
580
  watchForTokens();
@@ -530,6 +610,8 @@ export async function createCpuPool(
530
610
  // A closing pool drains its fifo whole; until then the bytes go back for the next holder.
531
611
  if (child.phase.slot) slotLent = false;
532
612
  heldMemory -= child.phase.memory;
613
+ holders -= 1;
614
+ if (child.phase.alone) aloneHolder = false;
533
615
  if (!closing) returnTokens(child.phase.bytes);
534
616
  }
535
617
  child.phase = { kind: 'done' };
@@ -590,7 +672,21 @@ export async function createCpuPool(
590
672
  }
591
673
  const request = parsed;
592
674
  child.phase = { kind: 'queued' };
593
- queue.push({ child, want: request.want, memory: request.memory ?? 0 });
675
+ const waiter: Waiter = {
676
+ child,
677
+ want: request.want,
678
+ memory: request.memory ?? 0,
679
+ parallelism: request.parallelism,
680
+ task: request.task,
681
+ alone: request.alone,
682
+ };
683
+ if (request.alone === true) {
684
+ // A leaf the kernel killed has waited once already: it goes ahead of the leaves that have not run.
685
+ const at = queue.findIndex((waiting) => waiting.alone !== true);
686
+ queue.splice(at === -1 ? queue.length : at, 0, waiter);
687
+ } else {
688
+ queue.push(waiter);
689
+ }
594
690
  socket.write(frame({ ok: true, lease: 'queued', server: process.pid }));
595
691
  drain(0);
596
692
  });
@@ -134,12 +134,19 @@ export interface NxTaskContext {
134
134
  export interface BoundedExecResult {
135
135
  success: boolean;
136
136
  terminalOutput: string;
137
+ /**
138
+ * The command died of a SIGKILL that no bound of this executor and no signal it sent caused: the kernel's OOM killer,
139
+ * which sends it to a leaf in a run's cgroup. Set on the last run of the command only when it was killed again.
140
+ */
141
+ killedForMemory?: boolean;
137
142
  }
138
143
 
139
144
  interface RunState {
140
145
  settled: boolean;
141
146
  expiry: ExpiredBound | null;
142
147
  forceKillNeeded: boolean;
148
+ /** This executor sent the command's process group a signal: a SIGKILL it died of is not from outside. */
149
+ sentSignal: boolean;
143
150
  /** The sub-pool's close error when it did not close cleanly: the run fails. */
144
151
  poolCloseError: string | null;
145
152
  /** Why the command's subtree could not be retired, when the supervisor's client failed: the run fails. */
@@ -313,10 +320,55 @@ export async function runBoundedExec(
313
320
  owner: SubtreeOwner | null = null,
314
321
  queueHost?: Parameters<typeof snapshotMbxQueue>[3],
315
322
  ): Promise<BoundedExecResult> {
316
- const cwd = resolveCwd(options.cwd, context.root);
317
323
  const directory = context.record?.directory ?? null;
318
- // The task's last recorded peak is its charge in its parent's pool, read before the clear below removes it.
324
+ // The task's last recorded peak, and the tokens it was lent when the peak was measured, are its charge in its parent's
325
+ // pool, read once before a run's clear removes them: a command run again after a kill is charged by the record the
326
+ // killed run replaced, not by the peak it reached before it was killed.
319
327
  const lastPeak = directory !== null && inherited !== null ? await readLastPeak(directory) : null;
328
+ const run = (alone: boolean): Promise<BoundedExecResult> =>
329
+ runCommand(options, context, killer, tempVolume, inherited, captureDiagnostics, owner, queueHost, {
330
+ lastPeak,
331
+ alone,
332
+ });
333
+ const first = await run(false);
334
+ if (first.killedForMemory !== true || inherited === null) {
335
+ return first;
336
+ }
337
+ // The kernel killed the leaf: a SIGKILL this executor did not send. It runs once more, alone in its run's pool, so
338
+ // what it was killed beside is not there to be killed with it.
339
+ const note = `cowshed: the kernel killed the command for memory (a SIGKILL this executor did not send${
340
+ context.record === undefined ? '' : `: ${context.record.task.id}`
341
+ }); running it once more, alone in its run's pool\n`;
342
+ process.stderr.write(note);
343
+ const again = await run(true);
344
+ return {
345
+ success: again.success,
346
+ terminalOutput: `${first.terminalOutput}${note}${again.terminalOutput}`,
347
+ ...(again.killedForMemory === true ? { killedForMemory: true } : {}),
348
+ };
349
+ }
350
+
351
+ /** What a run of the command is told that the run before it was not: the task's charge, and whether it runs alone. */
352
+ interface Attempt {
353
+ readonly lastPeak: Awaited<ReturnType<typeof readLastPeak>> | null;
354
+ readonly alone: boolean;
355
+ }
356
+
357
+ /** One run of the command, from the pool's grant to its record. */
358
+ async function runCommand(
359
+ options: BoundedExecOptions,
360
+ context: BoundedExecContext,
361
+ killer: ProcessTreeKiller,
362
+ tempVolume: TempVolume | null,
363
+ inherited: InheritedCpuPool | null,
364
+ captureDiagnostics: typeof captureHangDiagnostics,
365
+ owner: SubtreeOwner | null,
366
+ queueHost: Parameters<typeof snapshotMbxQueue>[3] | undefined,
367
+ attempt: Attempt,
368
+ ): Promise<BoundedExecResult> {
369
+ const cwd = resolveCwd(options.cwd, context.root);
370
+ const directory = context.record?.directory ?? null;
371
+ const { lastPeak } = attempt;
320
372
  // A previous run's verdict and report go first, so neither can be read as this run's.
321
373
  if (directory !== null) {
322
374
  await clearTaskDirectory(directory);
@@ -336,6 +388,7 @@ export async function runBoundedExec(
336
388
  settled: false,
337
389
  expiry: null,
338
390
  forceKillNeeded: false,
391
+ sentSignal: false,
339
392
  poolCloseError: null,
340
393
  retireError: null,
341
394
  noSpace: null,
@@ -408,8 +461,10 @@ export async function runBoundedExec(
408
461
  `bounded-exec: ${lastPeak.error}; it is this task's memory charge, so the command does not run\n`,
409
462
  );
410
463
  }
411
- // A task with no recorded peak is charged nothing (null) until it has one.
464
+ // A task with no recorded peak is charged nothing (null) until it has one. A peak is charged in proportion to the
465
+ // tokens the run is lent when that is more than the run that measured it was lent.
412
466
  const charge = lastPeak?.ok === true ? lastPeak.peakBytes : null;
467
+ const chargeParallelism = lastPeak?.ok === true ? lastPeak.parallelism : null;
413
468
 
414
469
  let lease: RamTempLease | null = null;
415
470
  let ordinaryTemp: string | null = null;
@@ -557,7 +612,11 @@ export async function runBoundedExec(
557
612
  let parentRefusal: string | null = null;
558
613
  let parentWait: CpuWaitRow | undefined;
559
614
  if (inherited !== null) {
560
- const fromParent = await takeParentCpuTokens(inherited, demand.want, charge);
615
+ const fromParent = await takeParentCpuTokens(inherited, demand.want, charge, {
616
+ parallelism: chargeParallelism ?? undefined,
617
+ alone: attempt.alone ? true : undefined,
618
+ task: context.record?.task.id,
619
+ });
561
620
  if (fromParent.granted) {
562
621
  grant = fromParent;
563
622
  } else if (fromParent.cause === 'absent') {
@@ -578,7 +637,7 @@ export async function runBoundedExec(
578
637
  sized = sizeRunner(demand, requested, grant.tokens);
579
638
  cpu = { tokens: grant.tokens, want: demand.want, waitedMs };
580
639
  appendStderr(
581
- `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}\n`,
640
+ `cowshed: cpu-pool wait done elapsed=${waitedMs}ms status=ok tokens=${grant.tokens}/${demand.want} runner=${demand.kind} memory=${charge ?? 'none'}${grant.charged !== null && grant.charged !== charge ? ` charged=${grant.charged}` : ''}\n`,
582
641
  );
583
642
  if (grant.oversized !== null) {
584
643
  appendStderr(
@@ -1131,6 +1190,7 @@ export async function runBoundedExec(
1131
1190
  return;
1132
1191
  }
1133
1192
  try {
1193
+ state.sentSignal = true;
1134
1194
  await killer.kill(pid, 'SIGTERM');
1135
1195
  } catch (error) {
1136
1196
  const reason = error instanceof Error ? error.message : String(error);
@@ -1280,6 +1340,7 @@ export async function runBoundedExec(
1280
1340
  // cannot end is said, and a failure of the probe or of a signal is shown.
1281
1341
  const forceKill = async (pid: number): Promise<void> => {
1282
1342
  try {
1343
+ state.sentSignal = true;
1283
1344
  if (await killProcessGroup(pid, DEADLINE_PATIENCE, (group) => killer.kill(group, 'SIGKILL'))) {
1284
1345
  tell(`${describeMembersLeft(pid)}\n`);
1285
1346
  }
@@ -1488,9 +1549,14 @@ export async function runBoundedExec(
1488
1549
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1489
1550
  const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1490
1551
  await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
1552
+ // A SIGKILL this executor did not send, to a command that no bound of this executor ended, came from outside it. In a
1553
+ // run's cgroup the one that sends it to a leaf is the kernel's OOM killer. The exit code alone says nothing of who
1554
+ // sent it: a bound's own force-kill ends in 137 too, and is why `sentSignal` is read first.
1555
+ const killedForMemory = !state.sentSignal && state.expiry === null && (exitSignal === 'SIGKILL' || code === 137);
1491
1556
  return {
1492
1557
  success: !operationalFailure && resultFailure === null && state.expiry === null && code === 0,
1493
1558
  terminalOutput: outputChunks.join(''),
1559
+ ...(killedForMemory ? { killedForMemory: true } : {}),
1494
1560
  };
1495
1561
  }
1496
1562
 
@@ -132,4 +132,10 @@ async function treeResidentBytes(root: number): Promise<number> {
132
132
  return total;
133
133
  }
134
134
 
135
- export const darwinMemory: HostMemory = { readRunMemory, peakBytes: (maxRss) => maxRss, treeResidentBytes };
135
+ export const darwinMemory: HostMemory = {
136
+ readRunMemory,
137
+ peakBytes: (maxRss) => maxRss,
138
+ treeResidentBytes,
139
+ // macOS has no cgroup, and its anonymous and shared pages are not read from a file the pool may open.
140
+ liveUsage: () => ({ ok: false, reason: 'macOS keeps no anonymous and shared memory of a run to read' }),
141
+ };
@@ -1,7 +1,8 @@
1
+ import { readFileSync } from 'node:fs';
1
2
  import { readFile } from 'node:fs/promises';
2
3
  import { posix } from 'node:path';
3
4
  import { childPids } from '../../process-children.linux.js';
4
- import type { HostMemory, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
5
+ import type { HostMemory, LiveUsageReading, RunMemory, RunMemoryError, RunMemoryReading } from './host-memory.js';
5
6
 
6
7
  /**
7
8
  * Linux's memory mechanisms. The run's memory is the least of the host's `MemTotal` and every finite `memory.max` of
@@ -103,6 +104,40 @@ async function readText(path: string): Promise<FileRead> {
103
104
  }
104
105
  }
105
106
 
107
+ /** The memory a bound holds live: `memory.stat`'s `anon` and `shmem` beside a cgroup's `memory.max`, meminfo's `AnonPages` and `Shmem` for the host's. */
108
+ export function linuxLiveUsage(read: (path: string) => FileRead, bound: string): LiveUsageReading {
109
+ const cgroupLimit = '/memory.max';
110
+ if (bound.endsWith(cgroupLimit)) {
111
+ const file = `${bound.slice(0, -cgroupLimit.length)}/memory.stat`;
112
+ const stat = read(file);
113
+ if (!stat.ok) return { ok: false, reason: `${file}: ${stat.code}: ${stat.message}` };
114
+ const [anon, shmem] = [/^anon (\d+)$/m.exec(stat.text)?.[1], /^shmem (\d+)$/m.exec(stat.text)?.[1]];
115
+ return anon === undefined || shmem === undefined
116
+ ? { ok: false, reason: `${file} names no anon and shmem in bytes` }
117
+ : { ok: true, usage: { anon: Number(anon), shmem: Number(shmem), source: file } };
118
+ }
119
+ if (bound === `${MEMINFO} MemTotal`) {
120
+ const info = read(MEMINFO);
121
+ if (!info.ok) return { ok: false, reason: `${MEMINFO}: ${info.code}: ${info.message}` };
122
+ const [anon, shmem] = [
123
+ /^AnonPages:\s+(\d+) kB$/m.exec(info.text)?.[1],
124
+ /^Shmem:\s+(\d+) kB$/m.exec(info.text)?.[1],
125
+ ];
126
+ return anon === undefined || shmem === undefined
127
+ ? { ok: false, reason: `${MEMINFO} names no AnonPages and Shmem in kB` }
128
+ : { ok: true, usage: { anon: Number(anon) * 1024, shmem: Number(shmem) * 1024, source: MEMINFO } };
129
+ }
130
+ return { ok: false, reason: `${bound} is no cgroup memory.max or meminfo bound` };
131
+ }
132
+
133
+ function readTextSync(path: string): FileRead {
134
+ try {
135
+ return { ok: true, text: readFileSync(path, 'utf8') };
136
+ } catch (error) {
137
+ const code = error instanceof Error && 'code' in error && typeof error.code === 'string' ? error.code : 'EIO';
138
+ return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
139
+ }
140
+ }
106
141
  /** Bun before 1.4.0 reports Linux's `ru_maxrss` as the kernel does, in KiB; from 1.4.0 it reports bytes. */
107
142
  function peakBytes(maxRss: number, bunVersion: string): number {
108
143
  const version = /^(\d+)\.(\d+)\./.exec(bunVersion);
@@ -144,4 +179,9 @@ async function residentBytes(pid: number): Promise<number> {
144
179
  return kib === undefined ? 0 : Number(kib) * 1024;
145
180
  }
146
181
 
147
- export const linuxMemory: HostMemory = { readRunMemory: () => linuxRunMemory(readText), peakBytes, treeResidentBytes };
182
+ export const linuxMemory: HostMemory = {
183
+ readRunMemory: () => linuxRunMemory(readText),
184
+ peakBytes,
185
+ treeResidentBytes,
186
+ liveUsage: (bound) => linuxLiveUsage(readTextSync, bound),
187
+ };
@@ -21,8 +21,26 @@ export interface HostMemory {
21
21
  * listing and its read adds nothing, and a root that has exited holds nothing.
22
22
  */
23
23
  treeResidentBytes(root: number): Promise<number>;
24
+ /**
25
+ * What the run's bound holds that the kernel cannot take back, read now: its anonymous and shared memory. `bound` is
26
+ * the `source` of the run's memory (`RunMemory`): the cgroup's `memory.max`, whose `memory.stat` is read, or the
27
+ * host's `MemTotal`, whose `AnonPages` and `Shmem` are. The pool says it on every grant, so what the leaves are not
28
+ * charged for (the baseline, a tmpfs) and what a suite holds beyond its record are measured, not guessed.
29
+ */
30
+ liveUsage(bound: string): LiveUsageReading;
31
+ }
32
+
33
+ /** What a bound holds live, in bytes, and the file it was read from. */
34
+ export interface LiveUsage {
35
+ readonly anon: number;
36
+ readonly shmem: number;
37
+ readonly source: string;
24
38
  }
25
39
 
40
+ export type LiveUsageReading =
41
+ | { readonly ok: true; readonly usage: LiveUsage }
42
+ | { readonly ok: false; readonly reason: string };
43
+
26
44
  /** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
27
45
  export interface RunMemory {
28
46
  readonly bytes: number;
@@ -75,3 +93,10 @@ export function readRunMemory(): Promise<RunMemoryReading> {
75
93
  }
76
94
  return HOST_MEMORY.readRunMemory();
77
95
  }
96
+
97
+ /** The run's bound live on this platform, or why it cannot be read: a pool says it, never leaves it out. */
98
+ export function readLiveUsage(bound: string): LiveUsageReading {
99
+ return HOST_MEMORY === null
100
+ ? { ok: false, reason: `${process.platform} has no reader of live memory` }
101
+ : HOST_MEMORY.liveUsage(bound);
102
+ }