@smoothbricks/nx-plugin 0.7.18 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -99
- package/dist/.build-inputs.json +35 -29
- package/dist/built-output-packages.js +1 -1
- package/dist/cargo-closure-input.js +1 -1
- package/dist/cargo-source-hash.d.ts +3 -21
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +4 -91
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +2 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +7 -5
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +14 -11
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +207 -69
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +3 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +66 -48
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +28 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +30 -9
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +8 -5
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +187 -283
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress.d.ts +3 -0
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +2 -2
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +26 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +26 -5
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +105 -73
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/task-results.js +2 -1
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1179 -133
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +105 -11
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/package.json +67 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/cargo-closure-input.ts +1 -1
- package/src/cargo-source-hash.ts +16 -132
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +1 -2
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +28 -15
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +280 -106
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +70 -52
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +36 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +213 -182
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress.ts +37 -2
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +2 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +39 -6
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +112 -78
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/workspace-config-policy.ts +161 -43
- package/test-timings.json +157 -147
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -6,10 +6,11 @@ import { join } from 'node:path';
|
|
|
6
6
|
import { promisify } from 'node:util';
|
|
7
7
|
import typia from 'typia';
|
|
8
8
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
9
|
+
import type { CaseCore } from './case-gate.js';
|
|
9
10
|
import { holdLease, type Unleased } from './gateway-lease.js';
|
|
10
11
|
import { type LiveUsageReading, type RunMemory, readLiveUsage } from './host-memory.js';
|
|
11
12
|
import { type CpuLeaseProgress, type CpuLeaseQueued, CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
|
|
12
|
-
import { admit } from './pool-admission.js';
|
|
13
|
+
import { admit, IdleBesideStarved, place } from './pool-admission.js';
|
|
13
14
|
import { type InheritedCpuPool, poolEnvironment } from './pool-environment.js';
|
|
14
15
|
|
|
15
16
|
/**
|
|
@@ -50,6 +51,36 @@ export interface CpuPool {
|
|
|
50
51
|
close(): Promise<CloseOutcome>;
|
|
51
52
|
}
|
|
52
53
|
|
|
54
|
+
/** What a pool says when it closes: the token-time idle beside starved holds, and the grants that made them. */
|
|
55
|
+
export interface PoolSummary {
|
|
56
|
+
/** Token-milliseconds no socket grant held while a hold was starved (pool-admission.ts, `IdleBesideStarved`). */
|
|
57
|
+
readonly idleBesideStarvedMs: number;
|
|
58
|
+
/** Leaves granted fewer tokens than they asked, whether gated or not. */
|
|
59
|
+
readonly starved: number;
|
|
60
|
+
/** Lent tokens granted, one per test case. */
|
|
61
|
+
readonly lent: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** When a pool lends: its clock, and how long after a leaf's grant returns it grants no lent request. */
|
|
65
|
+
export interface Lending {
|
|
66
|
+
readonly settleMs: number;
|
|
67
|
+
readonly now: () => number;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The settle time (specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets itself"): Nx starts a run's next
|
|
72
|
+
* task when one ends, and that task's leaf takes what its predecessor freed before any case is lent it.
|
|
73
|
+
*/
|
|
74
|
+
const LEND_SETTLE_MS = 3_000;
|
|
75
|
+
const LENDING: Lending = { settleMs: LEND_SETTLE_MS, now: () => performance.now() };
|
|
76
|
+
|
|
77
|
+
/** The line a run's pool says of `summary` when it closes; null for a pool that starved no leaf and lent nothing. */
|
|
78
|
+
export function describeIdle(summary: PoolSummary): string | null {
|
|
79
|
+
return summary.starved === 0 && summary.lent === 0
|
|
80
|
+
? null
|
|
81
|
+
: `cowshed: cpu-pool idle beside_starved_core_s=${(summary.idleBesideStarvedMs / 1000).toFixed(3)} starved=${summary.starved} lent=${summary.lent}\n`;
|
|
82
|
+
}
|
|
83
|
+
|
|
53
84
|
export type CloseOutcome = { ok: true } | { ok: false; error: string };
|
|
54
85
|
|
|
55
86
|
/**
|
|
@@ -64,6 +95,12 @@ interface CpuPoolRequest {
|
|
|
64
95
|
parallelism?: number & typia.tags.Type<'uint32'> & typia.tags.Minimum<1>;
|
|
65
96
|
task?: string & typia.tags.MaxLength<256>;
|
|
66
97
|
alone?: boolean;
|
|
98
|
+
/** One token for one test case of a leaf that holds a grant: granted only while no other request waits. */
|
|
99
|
+
lend?: boolean;
|
|
100
|
+
/** A lent request's leaf's predicted work left, in ms. */
|
|
101
|
+
left?: number & typia.tags.Type<'uint64'>;
|
|
102
|
+
/** A leaf whose cases each ask for a token of their own (`lend`): what it lacks of its ask is not idle beside it. */
|
|
103
|
+
gated?: boolean;
|
|
67
104
|
}
|
|
68
105
|
|
|
69
106
|
/** A grant whose charge alone exceeds the run's memory, made while nothing else held memory: the run's memory. */
|
|
@@ -110,6 +147,9 @@ type Phase =
|
|
|
110
147
|
readonly slot: boolean;
|
|
111
148
|
readonly memory: number;
|
|
112
149
|
readonly alone: boolean;
|
|
150
|
+
readonly tokens: number;
|
|
151
|
+
readonly starved: boolean;
|
|
152
|
+
readonly lend: boolean;
|
|
113
153
|
}
|
|
114
154
|
| { readonly kind: 'done' };
|
|
115
155
|
|
|
@@ -127,6 +167,9 @@ interface Waiter {
|
|
|
127
167
|
readonly parallelism?: number | undefined;
|
|
128
168
|
readonly task?: string | undefined;
|
|
129
169
|
readonly alone?: boolean | undefined;
|
|
170
|
+
readonly lend?: boolean | undefined;
|
|
171
|
+
readonly left?: number | undefined;
|
|
172
|
+
readonly gated?: boolean | undefined;
|
|
130
173
|
}
|
|
131
174
|
|
|
132
175
|
/** One framed reply line. The newline is the framing; every reply goes through here. */
|
|
@@ -159,6 +202,8 @@ export interface AskContext {
|
|
|
159
202
|
readonly task?: string | undefined;
|
|
160
203
|
/** The kernel killed the command for memory: it is run again, granted only while nothing else holds a grant. */
|
|
161
204
|
readonly alone?: boolean | undefined;
|
|
205
|
+
/** Its cases each ask the pool for a token of their own. */
|
|
206
|
+
readonly gated?: boolean | undefined;
|
|
162
207
|
}
|
|
163
208
|
|
|
164
209
|
/**
|
|
@@ -176,7 +221,7 @@ export async function takeParentCpuTokens(
|
|
|
176
221
|
memory: number | null,
|
|
177
222
|
context: AskContext = {},
|
|
178
223
|
): Promise<TokenGrant> {
|
|
179
|
-
const { parallelism, task, alone } = context;
|
|
224
|
+
const { parallelism, task, alone, gated } = context;
|
|
180
225
|
if (!Number.isSafeInteger(want) || want < 1) {
|
|
181
226
|
throw new RangeError(`a child asks its parent pool for a positive count, not ${want} (pool ${parent.tokens})`);
|
|
182
227
|
}
|
|
@@ -199,6 +244,7 @@ export async function takeParentCpuTokens(
|
|
|
199
244
|
...(memory === null ? {} : { memory, ...(parallelism === undefined ? {} : { parallelism }) }),
|
|
200
245
|
...(task === undefined ? {} : { task }),
|
|
201
246
|
...(alone === true ? { alone: true } : {}),
|
|
247
|
+
...(gated === true ? { gated: true } : {}),
|
|
202
248
|
};
|
|
203
249
|
const lease = await holdLease(parent.socket, request, 'the parent CPU pool', new CpuLeaseWait());
|
|
204
250
|
if (!lease.granted) {
|
|
@@ -239,6 +285,38 @@ export async function takeParentCpuTokens(
|
|
|
239
285
|
};
|
|
240
286
|
}
|
|
241
287
|
|
|
288
|
+
/**
|
|
289
|
+
* One token of `parent` asked for one test case of a leaf that holds a grant of it, named `task` in the request
|
|
290
|
+
* (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold). The pool grants it only while no other
|
|
291
|
+
* request waits. `release` gives the token back, or withdraws the request while it waits. A pool that refuses, falls
|
|
292
|
+
* silent or is gone grants nothing: the case waits for a token of its leaf's own grant.
|
|
293
|
+
*/
|
|
294
|
+
export function lendParentToken(parent: InheritedCpuPool, task: string, left: number): CaseCore {
|
|
295
|
+
const withdrawn = new AbortController();
|
|
296
|
+
const lease = holdLease(
|
|
297
|
+
parent.socket,
|
|
298
|
+
{ op: 'cpu-pool', want: 1, lend: true, left: Math.max(0, Math.round(left)), task: task.slice(0, MAX_TASK_LENGTH) },
|
|
299
|
+
'the parent CPU pool',
|
|
300
|
+
new CpuLeaseWait(),
|
|
301
|
+
undefined,
|
|
302
|
+
withdrawn.signal,
|
|
303
|
+
);
|
|
304
|
+
let released = false;
|
|
305
|
+
return {
|
|
306
|
+
granted: lease.then((held) => held.granted && !released),
|
|
307
|
+
release: () => {
|
|
308
|
+
if (released) return;
|
|
309
|
+
released = true;
|
|
310
|
+
withdrawn.abort();
|
|
311
|
+
void lease.then((held) => {
|
|
312
|
+
if (held.granted) held.release();
|
|
313
|
+
});
|
|
314
|
+
},
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/** A request's `task` is at most this long (`CpuPoolRequest`). */
|
|
319
|
+
const MAX_TASK_LENGTH = 256;
|
|
242
320
|
/** A thrown value's message. */
|
|
243
321
|
const describe = (error: unknown): string => (error instanceof Error ? error.message : String(error));
|
|
244
322
|
|
|
@@ -399,7 +477,8 @@ export async function createCpuPool(
|
|
|
399
477
|
closed: Promise<void>,
|
|
400
478
|
memory: RunMemory | null,
|
|
401
479
|
liveUsage: (bound: string) => LiveUsageReading = readLiveUsage,
|
|
402
|
-
|
|
480
|
+
lending: Lending = LENDING,
|
|
481
|
+
): Promise<{ ok: true; pool: CpuPool; summary: () => PoolSummary } | { ok: false; error: string }> {
|
|
403
482
|
if (!Number.isSafeInteger(tokens) || tokens < 1) {
|
|
404
483
|
throw new RangeError(`a CPU pool holds a positive safe integer of tokens, not ${tokens}`);
|
|
405
484
|
}
|
|
@@ -430,6 +509,15 @@ export async function createCpuPool(
|
|
|
430
509
|
// The children holding a grant, and whether one of them runs alone.
|
|
431
510
|
let holders = 0;
|
|
432
511
|
let aloneHolder = false;
|
|
512
|
+
// The tokens socket grants hold, the leaves that hold fewer than they asked and take no gate, and what the pool says
|
|
513
|
+
// of both when it closes.
|
|
514
|
+
let heldTokens = 0;
|
|
515
|
+
let starvedHolders = 0;
|
|
516
|
+
let partialGrants = 0;
|
|
517
|
+
let lentGrants = 0;
|
|
518
|
+
const idle = new IdleBesideStarved(lending.now());
|
|
519
|
+
// A leaf return arms this one-shot window. A later leaf return replaces it; a case return does not.
|
|
520
|
+
let settle: NodeJS.Timeout | undefined;
|
|
433
521
|
// The readability wait while the queue's head finds no token, else undefined.
|
|
434
522
|
let watch: FifoWatch | undefined;
|
|
435
523
|
let closing = false;
|
|
@@ -520,6 +608,7 @@ export async function createCpuPool(
|
|
|
520
608
|
inHand + (slotLent ? 0 : 1),
|
|
521
609
|
memory === null ? null : { capacity: memory.bytes, held: heldMemory },
|
|
522
610
|
{ holders, alone: aloneHolder },
|
|
611
|
+
settle === undefined,
|
|
523
612
|
);
|
|
524
613
|
queue.splice(0, admission.grants.length);
|
|
525
614
|
for (const { request, tokens: granted, charge, oversized } of admission.grants) {
|
|
@@ -530,8 +619,23 @@ export async function createCpuPool(
|
|
|
530
619
|
if (slot) slotLent = true;
|
|
531
620
|
heldMemory += charge;
|
|
532
621
|
holders += 1;
|
|
622
|
+
heldTokens += granted;
|
|
623
|
+
const partial = granted < request.want;
|
|
624
|
+
const starved = partial && request.gated !== true;
|
|
625
|
+
if (partial) partialGrants += 1;
|
|
626
|
+
if (starved) starvedHolders += 1;
|
|
627
|
+
if (request.lend === true) lentGrants += 1;
|
|
533
628
|
if (request.alone === true) aloneHolder = true;
|
|
534
|
-
request.child.phase = {
|
|
629
|
+
request.child.phase = {
|
|
630
|
+
kind: 'holding',
|
|
631
|
+
bytes,
|
|
632
|
+
slot,
|
|
633
|
+
memory: charge,
|
|
634
|
+
alone: request.alone === true,
|
|
635
|
+
tokens: granted,
|
|
636
|
+
starved,
|
|
637
|
+
lend: request.lend === true,
|
|
638
|
+
};
|
|
535
639
|
request.child.socket.write(
|
|
536
640
|
frame(
|
|
537
641
|
memory === null
|
|
@@ -542,7 +646,7 @@ export async function createCpuPool(
|
|
|
542
646
|
),
|
|
543
647
|
);
|
|
544
648
|
|
|
545
|
-
if (memory !== null) {
|
|
649
|
+
if (memory !== null && request.lend !== true) {
|
|
546
650
|
const live = liveUsage(memory.source);
|
|
547
651
|
const held = live.ok
|
|
548
652
|
? `${live.usage.anon + live.usage.shmem} bytes (anon ${live.usage.anon}, shmem ${live.usage.shmem}, ${live.usage.source})`
|
|
@@ -558,6 +662,7 @@ export async function createCpuPool(
|
|
|
558
662
|
stopWatching();
|
|
559
663
|
}
|
|
560
664
|
returnTokens(inHand);
|
|
665
|
+
idle.observe(lending.now(), tokens - heldTokens, starvedHolders > 0 || queue.at(-1)?.lend === true);
|
|
561
666
|
}
|
|
562
667
|
|
|
563
668
|
const refuse = (child: Child, error: string): void => {
|
|
@@ -587,6 +692,17 @@ export async function createCpuPool(
|
|
|
587
692
|
if (child.phase.slot) slotLent = false;
|
|
588
693
|
heldMemory -= child.phase.memory;
|
|
589
694
|
holders -= 1;
|
|
695
|
+
heldTokens -= child.phase.tokens;
|
|
696
|
+
if (child.phase.starved) starvedHolders -= 1;
|
|
697
|
+
// What a leaf returns is its successor's first: Nx starts the run's next task as this one ends.
|
|
698
|
+
if (!closing && !child.phase.lend && lending.settleMs > 0) {
|
|
699
|
+
clearTimeout(settle);
|
|
700
|
+
settle = setTimeout(() => {
|
|
701
|
+
settle = undefined;
|
|
702
|
+
drain(0);
|
|
703
|
+
}, lending.settleMs);
|
|
704
|
+
settle.unref();
|
|
705
|
+
}
|
|
590
706
|
if (child.phase.alone) aloneHolder = false;
|
|
591
707
|
if (!closing) returnTokens(child.phase.bytes);
|
|
592
708
|
}
|
|
@@ -646,6 +762,13 @@ export async function createCpuPool(
|
|
|
646
762
|
);
|
|
647
763
|
return;
|
|
648
764
|
}
|
|
765
|
+
if (parsed.lend === true && (parsed.want !== 1 || parsed.memory !== undefined || parsed.alone === true)) {
|
|
766
|
+
refuse(
|
|
767
|
+
child,
|
|
768
|
+
'a lent request is one token for one test case: {"op":"cpu-pool","want":1,"lend":true,"task"?:<its case>}',
|
|
769
|
+
);
|
|
770
|
+
return;
|
|
771
|
+
}
|
|
649
772
|
const request = parsed;
|
|
650
773
|
child.phase = { kind: 'queued' };
|
|
651
774
|
const waiter: Waiter = {
|
|
@@ -655,14 +778,11 @@ export async function createCpuPool(
|
|
|
655
778
|
parallelism: request.parallelism,
|
|
656
779
|
task: request.task,
|
|
657
780
|
alone: request.alone,
|
|
781
|
+
lend: request.lend,
|
|
782
|
+
left: request.left,
|
|
783
|
+
gated: request.gated,
|
|
658
784
|
};
|
|
659
|
-
|
|
660
|
-
// A leaf the kernel killed has waited once already: it goes ahead of the leaves that have not run.
|
|
661
|
-
const at = queue.findIndex((waiting) => waiting.alone !== true);
|
|
662
|
-
queue.splice(at === -1 ? queue.length : at, 0, waiter);
|
|
663
|
-
} else {
|
|
664
|
-
queue.push(waiter);
|
|
665
|
-
}
|
|
785
|
+
queue.splice(place(queue, waiter), 0, waiter);
|
|
666
786
|
socket.write(frame({ ok: true, lease: 'queued', server: process.pid }));
|
|
667
787
|
drain(0);
|
|
668
788
|
});
|
|
@@ -676,6 +796,8 @@ export async function createCpuPool(
|
|
|
676
796
|
shut ??= (async (): Promise<CloseOutcome> => {
|
|
677
797
|
closing = true;
|
|
678
798
|
clearInterval(heartbeat);
|
|
799
|
+
clearTimeout(settle);
|
|
800
|
+
idle.observe(lending.now(), 0, false);
|
|
679
801
|
stopWatching();
|
|
680
802
|
await started;
|
|
681
803
|
for (const socket of children) {
|
|
@@ -773,5 +895,10 @@ export async function createCpuPool(
|
|
|
773
895
|
environment: poolEnvironment(socketPath, tokens, fifoPath),
|
|
774
896
|
close: shutdown,
|
|
775
897
|
},
|
|
898
|
+
summary: () => ({
|
|
899
|
+
idleBesideStarvedMs: idle.tokenMs(lending.now()),
|
|
900
|
+
starved: partialGrants,
|
|
901
|
+
lent: lentGrants,
|
|
902
|
+
}),
|
|
776
903
|
};
|
|
777
904
|
}
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
import typia from 'typia';
|
|
2
|
+
import { REAP_MS } from './lease-wait.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The life of an Nx run's mbx reservation endpoint in the run that started it (mbx-reservations.ts;
|
|
6
|
+
* specs/cowshed/05_gateway.md, "Parallelism inside a job"), as a state machine over what the run can observe of its
|
|
7
|
+
* child. Pure: no I/O, no clock and no timer. The shell maps the child's and the probe's events into `EndpointEvent`s
|
|
8
|
+
* and executes the `EndpointEffect`s each step answers, in their order; it decides nothing.
|
|
9
|
+
*
|
|
10
|
+
* The run waits in four phases, each for events only:
|
|
11
|
+
*
|
|
12
|
+
* - `starting`, for the child's first line on its standard output, or its end. `listening` opens the probe; any other
|
|
13
|
+
* line is refused and the child killed. A child whose standard output ends without a line can say none: its start
|
|
14
|
+
* is refused, by how it ended when it ends within the reap bound, and it is killed when it does not.
|
|
15
|
+
* - `probing`, for the endpoint's answer to `{"type":"capabilities"}` on the probe's connection, or that connection's
|
|
16
|
+
* end: whether the installed mbx's shim gates test cases (05_gateway.md, "A case starts on a core"). The start is
|
|
17
|
+
* then answered served, which publishes the socket to the run's tasks. A child that ends first is refused by how it
|
|
18
|
+
* ended: no task is told of an endpoint that is gone.
|
|
19
|
+
* - `serving`, for the run's end. A child that ends first is `gone`, and the run's end says how.
|
|
20
|
+
* - `ending`, for the child's end once a pipe of it closed (its standard input by the run, its standard output by
|
|
21
|
+
* itself) or it was killed, with the reap timer armed.
|
|
22
|
+
*
|
|
23
|
+
* The start is answered once (`serve` or `refuse`) and a served endpoint's end once (`conclude`); `ended` takes no
|
|
24
|
+
* event.
|
|
25
|
+
*/
|
|
26
|
+
export type EndpointLife =
|
|
27
|
+
| { readonly phase: 'starting'; readonly out: string; readonly said: string; readonly pid: number | undefined }
|
|
28
|
+
| { readonly phase: 'probing'; readonly reply: string; readonly said: string; readonly pid: number | undefined }
|
|
29
|
+
| { readonly phase: 'serving'; readonly said: string; readonly pid: number | undefined }
|
|
30
|
+
| { readonly phase: 'gone'; readonly ended: Ended }
|
|
31
|
+
| {
|
|
32
|
+
readonly phase: 'ending';
|
|
33
|
+
readonly pid: number | undefined;
|
|
34
|
+
readonly said: string;
|
|
35
|
+
/** Whether the child was sent SIGKILL; otherwise a pipe of it closed and its end is awaited. */
|
|
36
|
+
readonly killed: boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Why its start is refused; null for a served endpoint ending with its run. An unkilled child that ends is
|
|
39
|
+
* refused by how it ended instead, which says more.
|
|
40
|
+
*/
|
|
41
|
+
readonly refusing: string | null;
|
|
42
|
+
}
|
|
43
|
+
| { readonly phase: 'ended' };
|
|
44
|
+
|
|
45
|
+
/** How a served endpoint ended, and whether that was its ordinary end: status 0 with nothing said, unkilled. */
|
|
46
|
+
export interface Ended {
|
|
47
|
+
readonly how: string;
|
|
48
|
+
readonly clean: boolean;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** What the shell observes. `run-end` is the run's own: its `postTasksExecution`, or a held run's question answered. */
|
|
52
|
+
export type EndpointEvent =
|
|
53
|
+
| { readonly type: 'stdout'; readonly text: string }
|
|
54
|
+
| { readonly type: 'stdout-end' }
|
|
55
|
+
| { readonly type: 'stderr'; readonly text: string }
|
|
56
|
+
/** Node's `error` of the child: it could not be spawned. */
|
|
57
|
+
| { readonly type: 'error'; readonly message: string }
|
|
58
|
+
/** Node's `close` of the child: it has exited and its pipes have ended. */
|
|
59
|
+
| { readonly type: 'close'; readonly code: number | null; readonly signal: string | null }
|
|
60
|
+
| { readonly type: 'probe'; readonly text: string }
|
|
61
|
+
/** The probe's connection ended or failed. */
|
|
62
|
+
| { readonly type: 'probe-end' }
|
|
63
|
+
| { readonly type: 'run-end' }
|
|
64
|
+
/** The reap timer the last `arm` started has run out. */
|
|
65
|
+
| { readonly type: 'expired' };
|
|
66
|
+
|
|
67
|
+
/** What the shell does. `refuse` and `conclude` remove the socket's directory before they answer. */
|
|
68
|
+
export type EndpointEffect =
|
|
69
|
+
/** Connect to the endpoint's socket and send it `{"type":"capabilities"}`. */
|
|
70
|
+
| { readonly do: 'probe' }
|
|
71
|
+
/** End the probe's connection. */
|
|
72
|
+
| { readonly do: 'hang-up' }
|
|
73
|
+
/** Answer the start: the endpoint serves, and its mbx gates test cases or does not. */
|
|
74
|
+
| { readonly do: 'serve'; readonly gates: boolean }
|
|
75
|
+
/** Answer the start: no endpoint, and why. */
|
|
76
|
+
| { readonly do: 'refuse'; readonly why: string }
|
|
77
|
+
| { readonly do: 'close-stdin' }
|
|
78
|
+
| { readonly do: 'kill' }
|
|
79
|
+
|
|
80
|
+
/** Stop the child and its pipes from keeping the run alive, without signalling it again. */
|
|
81
|
+
| { readonly do: 'unref-child' }
|
|
82
|
+
| { readonly do: 'arm'; readonly ms: number }
|
|
83
|
+
| { readonly do: 'disarm' }
|
|
84
|
+
/** Answer the run's end. */
|
|
85
|
+
| { readonly do: 'conclude'; readonly ended: Ended };
|
|
86
|
+
|
|
87
|
+
export type EndpointStep = readonly [EndpointLife, readonly EndpointEffect[]];
|
|
88
|
+
|
|
89
|
+
/** What is kept of the endpoint's stderr to say why it ended: its last bytes. */
|
|
90
|
+
const SAID_BYTES = 4096;
|
|
91
|
+
|
|
92
|
+
/** The life of a child just spawned; `pid` is what names it should it not end when killed. */
|
|
93
|
+
export function spawned(pid: number | undefined): EndpointLife {
|
|
94
|
+
return { phase: 'starting', out: '', said: '', pid };
|
|
95
|
+
}
|
|
96
|
+
const ENDED: EndpointLife = { phase: 'ended' };
|
|
97
|
+
|
|
98
|
+
type CapabilitiesReply = { type: 'capabilities'; test_gate: boolean };
|
|
99
|
+
const parseCapabilities = typia.json.createValidateParse<CapabilitiesReply>();
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Whether the mbx that answered `line` to `{"type":"capabilities"}` gates test cases. One that does not know the
|
|
103
|
+
* request answers an error, which gates nothing.
|
|
104
|
+
*/
|
|
105
|
+
/**
|
|
106
|
+
* Whether the mbx that answered `line` to `{"type":"capabilities"}` gates test cases. One that does not know the
|
|
107
|
+
* request answers an error, which gates nothing, and so does a line that is no JSON: typia's parse validates what
|
|
108
|
+
* `JSON.parse` returns and throws what it throws.
|
|
109
|
+
*/
|
|
110
|
+
export function gatesTests(line: string): boolean {
|
|
111
|
+
try {
|
|
112
|
+
const reply = parseCapabilities(line);
|
|
113
|
+
return reply.success && reply.data.test_gate;
|
|
114
|
+
} catch {
|
|
115
|
+
return false;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** `text` up to its first line break, or null while it holds none. */
|
|
120
|
+
function firstLine(text: string): string | null {
|
|
121
|
+
const end = text.indexOf('\n');
|
|
122
|
+
return end === -1 ? null : text.slice(0, end);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function saying(said: string, text: string): string {
|
|
126
|
+
return (said + text).slice(-SAID_BYTES);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function closed(said: string, code: number | null, signal: string | null): Ended {
|
|
130
|
+
const last = said.trim().split('\n').at(-1) ?? '';
|
|
131
|
+
return {
|
|
132
|
+
how: `mbx reservations ended ${signal === null ? `with status ${code}` : `on ${signal}`}${last === '' ? '' : `: ${last}`}`,
|
|
133
|
+
clean: signal === null && code === 0 && last === '',
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function unspawned(message: string): Ended {
|
|
138
|
+
return { how: `mbx reservations could not run: ${message}`, clean: false };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** How the child ended, when `event` is its end. */
|
|
142
|
+
function endOf(said: string, event: EndpointEvent): Ended | null {
|
|
143
|
+
if (event.type === 'close') return closed(said, event.code, event.signal);
|
|
144
|
+
if (event.type === 'error') return unspawned(event.message);
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** The next state of `life` at `event`, and what the shell does about it. */
|
|
149
|
+
export function endpointStep(life: EndpointLife, event: EndpointEvent): EndpointStep {
|
|
150
|
+
switch (life.phase) {
|
|
151
|
+
case 'starting':
|
|
152
|
+
return starting(life, event);
|
|
153
|
+
case 'probing':
|
|
154
|
+
return probing(life, event);
|
|
155
|
+
case 'serving':
|
|
156
|
+
return serving(life, event);
|
|
157
|
+
case 'gone':
|
|
158
|
+
return event.type === 'run-end' ? [ENDED, [{ do: 'conclude', ended: life.ended }]] : [life, []];
|
|
159
|
+
case 'ending':
|
|
160
|
+
return ending(life, event);
|
|
161
|
+
case 'ended':
|
|
162
|
+
return [life, []];
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function unanswered(phase: string): never {
|
|
167
|
+
throw new Error(
|
|
168
|
+
`the run's end reached its endpoint while ${phase}: the run holds no endpoint before its start is answered`,
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function starting(life: EndpointLife & { phase: 'starting' }, event: EndpointEvent): EndpointStep {
|
|
173
|
+
const ended = endOf(life.said, event);
|
|
174
|
+
if (ended !== null) return [ENDED, [{ do: 'refuse', why: ended.how }]];
|
|
175
|
+
switch (event.type) {
|
|
176
|
+
case 'stderr':
|
|
177
|
+
return [{ ...life, said: saying(life.said, event.text) }, []];
|
|
178
|
+
case 'stdout': {
|
|
179
|
+
const out = life.out + event.text;
|
|
180
|
+
const line = firstLine(out);
|
|
181
|
+
if (line === null) return [{ ...life, out }, []];
|
|
182
|
+
if (line === 'listening') {
|
|
183
|
+
return [{ phase: 'probing', reply: '', said: life.said, pid: life.pid }, [{ do: 'probe' }]];
|
|
184
|
+
}
|
|
185
|
+
return [
|
|
186
|
+
{
|
|
187
|
+
phase: 'ending',
|
|
188
|
+
pid: life.pid,
|
|
189
|
+
said: life.said,
|
|
190
|
+
killed: true,
|
|
191
|
+
refusing: `mbx reservations said ${JSON.stringify(line)}, not that it was listening`,
|
|
192
|
+
},
|
|
193
|
+
[{ do: 'kill' }, { do: 'arm', ms: REAP_MS }],
|
|
194
|
+
];
|
|
195
|
+
}
|
|
196
|
+
case 'stdout-end': {
|
|
197
|
+
const unsaid = `mbx reservations closed its standard output ${life.out === '' ? '' : `after ${JSON.stringify(life.out)}, `}without saying it was listening`;
|
|
198
|
+
return [
|
|
199
|
+
{ phase: 'ending', pid: life.pid, said: life.said, killed: false, refusing: unsaid },
|
|
200
|
+
[{ do: 'arm', ms: REAP_MS }],
|
|
201
|
+
];
|
|
202
|
+
}
|
|
203
|
+
case 'run-end':
|
|
204
|
+
return unanswered(life.phase);
|
|
205
|
+
default:
|
|
206
|
+
return [life, []];
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function probing(life: EndpointLife & { phase: 'probing' }, event: EndpointEvent): EndpointStep {
|
|
211
|
+
const served: EndpointLife = { phase: 'serving', said: life.said, pid: life.pid };
|
|
212
|
+
const ended = endOf(life.said, event);
|
|
213
|
+
if (ended !== null) return [ENDED, [{ do: 'hang-up' }, { do: 'refuse', why: ended.how }]];
|
|
214
|
+
switch (event.type) {
|
|
215
|
+
case 'stderr':
|
|
216
|
+
return [{ ...life, said: saying(life.said, event.text) }, []];
|
|
217
|
+
case 'probe': {
|
|
218
|
+
const reply = life.reply + event.text;
|
|
219
|
+
const line = firstLine(reply);
|
|
220
|
+
if (line === null) return [{ ...life, reply }, []];
|
|
221
|
+
return [served, [{ do: 'hang-up' }, { do: 'serve', gates: gatesTests(line) }]];
|
|
222
|
+
}
|
|
223
|
+
case 'probe-end':
|
|
224
|
+
return [served, [{ do: 'serve', gates: false }]];
|
|
225
|
+
case 'run-end':
|
|
226
|
+
return unanswered(life.phase);
|
|
227
|
+
default:
|
|
228
|
+
return [life, []];
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function serving(life: EndpointLife & { phase: 'serving' }, event: EndpointEvent): EndpointStep {
|
|
233
|
+
const ended = endOf(life.said, event);
|
|
234
|
+
if (ended !== null) return [{ phase: 'gone', ended }, []];
|
|
235
|
+
switch (event.type) {
|
|
236
|
+
case 'stderr':
|
|
237
|
+
return [{ ...life, said: saying(life.said, event.text) }, []];
|
|
238
|
+
case 'run-end':
|
|
239
|
+
return [
|
|
240
|
+
{ phase: 'ending', said: life.said, pid: life.pid, killed: false, refusing: null },
|
|
241
|
+
[{ do: 'close-stdin' }, { do: 'arm', ms: REAP_MS }],
|
|
242
|
+
];
|
|
243
|
+
default:
|
|
244
|
+
return [life, []];
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function ending(life: EndpointLife & { phase: 'ending' }, event: EndpointEvent): EndpointStep {
|
|
249
|
+
const ended = endOf(life.said, event);
|
|
250
|
+
if (ended !== null) {
|
|
251
|
+
if (life.refusing !== null) {
|
|
252
|
+
return [ENDED, [{ do: 'disarm' }, { do: 'refuse', why: life.killed ? life.refusing : ended.how }]];
|
|
253
|
+
}
|
|
254
|
+
const outlived: Ended = life.killed
|
|
255
|
+
? { how: `${ended.how}, killed after it outlived its standard input by ${REAP_MS}ms`, clean: false }
|
|
256
|
+
: ended;
|
|
257
|
+
return [ENDED, [{ do: 'disarm' }, { do: 'conclude', ended: outlived }]];
|
|
258
|
+
}
|
|
259
|
+
switch (event.type) {
|
|
260
|
+
case 'stderr':
|
|
261
|
+
return [{ ...life, said: saying(life.said, event.text) }, []];
|
|
262
|
+
case 'expired': {
|
|
263
|
+
// A closed pipe did not end it within the reap bound: kill it, and wait that bound once more.
|
|
264
|
+
if (!life.killed) {
|
|
265
|
+
const refusing = life.refusing === null ? null : `${life.refusing}, and had not ended ${REAP_MS}ms later`;
|
|
266
|
+
return [{ ...life, killed: true, refusing }, [{ do: 'kill' }, { do: 'arm', ms: REAP_MS }]];
|
|
267
|
+
}
|
|
268
|
+
// Neither did the kill: it is stuck in the kernel, or something it started holds its pipes. The wait ends and
|
|
269
|
+
// says so; the child is left as it is (05_gateway.md, "A child's exit and its output are reap-class waits").
|
|
270
|
+
const left = `${life.pid === undefined ? 'it' : `pid ${life.pid}`} had not ended ${REAP_MS}ms after it was killed and is left as it is`;
|
|
271
|
+
if (life.refusing !== null)
|
|
272
|
+
return [ENDED, [{ do: 'unref-child' }, { do: 'refuse', why: `${life.refusing}; ${left}` }]];
|
|
273
|
+
const how = `mbx reservations outlived its standard input by ${REAP_MS}ms; ${left}`;
|
|
274
|
+
return [ENDED, [{ do: 'unref-child' }, { do: 'conclude', ended: { how, clean: false } }]];
|
|
275
|
+
}
|
|
276
|
+
default:
|
|
277
|
+
return [life, []];
|
|
278
|
+
}
|
|
279
|
+
}
|