@smoothbricks/nx-plugin 0.7.11 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -25
- package/dist/.build-inputs.json +25 -17
- package/dist/built-output-packages.d.ts +27 -0
- package/dist/built-output-packages.d.ts.map +1 -0
- package/dist/built-output-packages.js +97 -0
- package/dist/cross-check-policy.d.ts +13 -1
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +13 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +68 -0
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -0
- package/dist/executors/bounded-exec/admission-queue.js +131 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts +14 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-grant.js +18 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +26 -24
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +88 -72
- package/dist/executors/bounded-exec/executor.d.ts +5 -7
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +69 -133
- package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
- package/dist/executors/bounded-exec/host-memory.js +6 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
- package/dist/executors/bounded-exec/memory-retry.d.ts +39 -0
- package/dist/executors/bounded-exec/memory-retry.d.ts.map +1 -0
- package/dist/executors/bounded-exec/memory-retry.js +33 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts +43 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/parent-grant.js +37 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts +17 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-charge.js +20 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +44 -8
- package/dist/executors/bounded-exec/pool-environment.d.ts +25 -0
- package/dist/executors/bounded-exec/pool-environment.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-environment.js +34 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +5 -18
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +12 -16
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -21
- package/dist/managed-files/ci-workflow.d.ts +29 -5
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +111 -16
- package/dist/managed-files/context.d.ts.map +1 -1
- package/dist/managed-files/context.js +1 -0
- package/dist/managed-files/files.d.ts +3 -1
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +2 -0
- package/dist/managed-files/publish-workflow.d.ts +6 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/managed-files/publish-workflow.js +16 -11
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +48 -14
- package/dist/typecheck-route.d.ts +28 -0
- package/dist/typecheck-route.d.ts.map +1 -0
- package/dist/typecheck-route.js +109 -0
- package/dist/workspace-manifest.d.ts +50 -5
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/executors.json +10 -0
- package/managed/raw/patches/nx@24.0.3.patch +16 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +93 -8
- package/managed/raw/tooling/direnv/apple-developer.sh +8 -1
- package/managed/raw/tooling/direnv/compiler-variables.sh +1 -1
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +4 -2
- package/managed/raw/tooling/direnv/secret-references.ts +19 -5
- package/managed/raw/tooling/git-hooks/pre-push.sh +20 -10
- package/managed/templates/github/actions/forgejo-artifact.cjs +9 -9
- package/managed/templates/github/actions/setup-devenv/action.yml +53 -0
- package/package.json +2 -2
- package/src/built-output-packages.ts +125 -0
- package/src/cross-check-policy.ts +14 -1
- package/src/executors/bounded-exec/admission-queue.ts +163 -0
- package/src/executors/bounded-exec/cpu-grant.ts +32 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
- package/src/executors/bounded-exec/cpu-pool.ts +123 -58
- package/src/executors/bounded-exec/executor.ts +93 -168
- package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
- package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
- package/src/executors/bounded-exec/host-memory.ts +25 -0
- package/src/executors/bounded-exec/memory-retry.ts +63 -0
- package/src/executors/bounded-exec/parent-grant.ts +75 -0
- package/src/executors/bounded-exec/peak-charge.ts +25 -0
- package/src/executors/bounded-exec/pool-admission.ts +66 -11
- package/src/executors/bounded-exec/pool-environment.ts +45 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +5 -4
- package/src/executors/bounded-exec/verdict.ts +17 -25
- package/src/index.ts +70 -21
- package/src/managed-files/ci-workflow.ts +143 -16
- package/src/managed-files/context.ts +1 -0
- package/src/managed-files/files.ts +5 -0
- package/src/managed-files/publish-workflow.ts +26 -10
- package/src/owned-installation.ts +63 -14
- package/src/typecheck-route.ts +64 -0
- package/src/workspace-manifest.ts +56 -5
- package/test-timings.json +141 -137
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { admissionWaitMs, type BoundEvent, NO_ADMISSION_WAIT, stepAdmissionWait } from './bounds.js';
|
|
2
|
+
import { HANG_CAPTURE_MS } from './hang-diagnostics.js';
|
|
3
|
+
import { MBX_QUEUE_POLL_MS, type MbxQueuedRequest, snapshotMbxQueue } from './mbx-queue.js';
|
|
4
|
+
import { type AdmissionQueueWait, describeAdmissionQueue } from './verdict.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* What a Cargo or reserved leaf waits in: mbx's admission queue. The snapshots of it are said on stderr and decide
|
|
8
|
+
* whether the wait is credited against the idle bound, so they can fail a run and never change a command that passes:
|
|
9
|
+
* the key leaves this out (README, "Bounded exec key"). The bounds themselves (bounds.ts) stay keyed.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
/** The mbx host a snapshot runs `mbx` through; the real one unless a test injects another. */
|
|
13
|
+
export type QueueHost = Parameters<typeof snapshotMbxQueue>[3];
|
|
14
|
+
|
|
15
|
+
/** What the run tells its observation of itself. */
|
|
16
|
+
export interface ObservationHost {
|
|
17
|
+
/** The run is over: it settled or a bound expired. Nothing more is observed or credited. */
|
|
18
|
+
over(): boolean;
|
|
19
|
+
/** The command's pid, once it is spawned. */
|
|
20
|
+
pid(): number | undefined;
|
|
21
|
+
/** The environment the command runs with, through which mbx finds its session. */
|
|
22
|
+
environment(): NodeJS.ProcessEnv;
|
|
23
|
+
/** Told, never thrown. */
|
|
24
|
+
tell(text: string): void;
|
|
25
|
+
/** The bounds' next event. */
|
|
26
|
+
step(event: BoundEvent): void;
|
|
27
|
+
/** Whether the command is running a test. */
|
|
28
|
+
testing(): boolean;
|
|
29
|
+
readonly queueHost: QueueHost | undefined;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** One run's view of the admission queue its leaf waits in. */
|
|
33
|
+
export class AdmissionObservation {
|
|
34
|
+
/** mbx granted the leaf its cores. */
|
|
35
|
+
admitted = false;
|
|
36
|
+
private wait = NO_ADMISSION_WAIT;
|
|
37
|
+
private requests: readonly MbxQueuedRequest[] = [];
|
|
38
|
+
/**
|
|
39
|
+
* Why the last admission snapshot failed, while the one after it has not succeeded; null otherwise. A snapshot only
|
|
40
|
+
* decides whether waiting on mbx admission is credited against the idle bound, so a failed one withholds that credit
|
|
41
|
+
* and ends nothing: the bounds decide, and an idle expiry names it.
|
|
42
|
+
*/
|
|
43
|
+
private failure: string | null = null;
|
|
44
|
+
/** When the last successful snapshot completed (`Date.now()`): credit runs to it and no further. */
|
|
45
|
+
private confirmedAt: number | null = null;
|
|
46
|
+
private readonly abort = new AbortController();
|
|
47
|
+
private timer: NodeJS.Timeout | undefined;
|
|
48
|
+
private snapshot: Promise<void> | null = null;
|
|
49
|
+
private lastDiagnostic = '';
|
|
50
|
+
private lastDiagnosticAt: number | null = null;
|
|
51
|
+
private lastSnapshotDiagnostics = '';
|
|
52
|
+
|
|
53
|
+
constructor(private readonly host: ObservationHost) {}
|
|
54
|
+
|
|
55
|
+
/** Why the last snapshot failed, while the one after it has not succeeded. */
|
|
56
|
+
get unconfirmed(): string | null {
|
|
57
|
+
return this.failure;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* A reserved leaf's own request on its run's endpoint is its admission wait; a Cargo leaf's comes from mbx's report.
|
|
62
|
+
* `reserved` is whether the leaf holds a reservation of its own.
|
|
63
|
+
*/
|
|
64
|
+
queueWait(reserved: boolean): AdmissionQueueWait | null {
|
|
65
|
+
return this.admitted || (this.requests.length === 0 && !reserved)
|
|
66
|
+
? null
|
|
67
|
+
: { waitedMs: admissionWaitMs(this.wait, performance.now()), requests: this.requests };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** A reserved leaf's request began to wait on its run's endpoint. */
|
|
71
|
+
waiting(): void {
|
|
72
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: performance.now(), waiting: true });
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool: poll it. */
|
|
76
|
+
poll(): void {
|
|
77
|
+
this.timer = setInterval(() => {
|
|
78
|
+
void this.refresh(false);
|
|
79
|
+
}, MBX_QUEUE_POLL_MS);
|
|
80
|
+
void this.refresh(false);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** The run ended: the wait stops counting and no snapshot is taken or credited again. */
|
|
84
|
+
stop(): void {
|
|
85
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'stop', at: performance.now() });
|
|
86
|
+
clearInterval(this.timer);
|
|
87
|
+
this.abort.abort();
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** The snapshot in flight, if any, has finished. */
|
|
91
|
+
async idle(): Promise<void> {
|
|
92
|
+
await this.snapshot;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
private unconfirm(detail: string): void {
|
|
96
|
+
if (this.host.over() || this.abort.signal.aborted) return;
|
|
97
|
+
if (this.failure !== detail) {
|
|
98
|
+
this.host.tell(
|
|
99
|
+
`bounded-exec: mbx admission snapshot failed: ${detail}; admission is unconfirmed, so this wait earns no idle credit until a snapshot succeeds\n`,
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
this.failure = detail;
|
|
103
|
+
this.requests = [];
|
|
104
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: performance.now(), waiting: false });
|
|
105
|
+
// The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
|
|
106
|
+
this.host.step({
|
|
107
|
+
kind: 'admission-queue',
|
|
108
|
+
at: this.confirmedAt ?? Date.now(),
|
|
109
|
+
waiting: false,
|
|
110
|
+
testing: this.host.testing(),
|
|
111
|
+
});
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Take a snapshot; `fresh` drains an older poll first. */
|
|
115
|
+
async refresh(fresh: boolean): Promise<void> {
|
|
116
|
+
if (fresh && this.snapshot !== null) await this.snapshot;
|
|
117
|
+
const pid = this.host.pid();
|
|
118
|
+
if (this.host.over() || this.abort.signal.aborted || pid === undefined) return;
|
|
119
|
+
this.snapshot ??= (async () => {
|
|
120
|
+
const snapshot = await snapshotMbxQueue(pid, this.host.environment(), this.abort.signal, this.host.queueHost);
|
|
121
|
+
if (this.host.over() || this.abort.signal.aborted) return;
|
|
122
|
+
if (!snapshot.ok) {
|
|
123
|
+
this.unconfirm(snapshot.detail);
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
const diagnostics = snapshot.diagnostics?.join('; ') ?? '';
|
|
127
|
+
if (diagnostics !== '' && diagnostics !== this.lastSnapshotDiagnostics) {
|
|
128
|
+
this.host.tell(`bounded-exec: ${diagnostics}\n`);
|
|
129
|
+
}
|
|
130
|
+
this.lastSnapshotDiagnostics = diagnostics;
|
|
131
|
+
const waiting = snapshot.requests.length > 0;
|
|
132
|
+
const observedAt = Date.now();
|
|
133
|
+
const observedWaitAt = performance.now();
|
|
134
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: observedWaitAt, waiting });
|
|
135
|
+
this.failure = null;
|
|
136
|
+
this.confirmedAt = observedAt;
|
|
137
|
+
if (waiting) {
|
|
138
|
+
this.requests = snapshot.requests;
|
|
139
|
+
const positions = JSON.stringify(snapshot.requests);
|
|
140
|
+
if (
|
|
141
|
+
positions !== this.lastDiagnostic ||
|
|
142
|
+
this.lastDiagnosticAt === null ||
|
|
143
|
+
Date.now() - this.lastDiagnosticAt >= HANG_CAPTURE_MS
|
|
144
|
+
) {
|
|
145
|
+
const waitedMs = admissionWaitMs(this.wait, observedWaitAt);
|
|
146
|
+
this.host.tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
|
|
147
|
+
this.lastDiagnostic = positions;
|
|
148
|
+
this.lastDiagnosticAt = Date.now();
|
|
149
|
+
}
|
|
150
|
+
} else {
|
|
151
|
+
this.lastDiagnostic = '';
|
|
152
|
+
}
|
|
153
|
+
this.host.step({ kind: 'admission-queue', at: observedAt, waiting, testing: this.host.testing() });
|
|
154
|
+
})()
|
|
155
|
+
.catch((error: unknown) => {
|
|
156
|
+
this.unconfirm(error instanceof Error ? error.message : String(error));
|
|
157
|
+
})
|
|
158
|
+
.finally(() => {
|
|
159
|
+
this.snapshot = null;
|
|
160
|
+
});
|
|
161
|
+
await this.snapshot;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The CPU grant a run's record names (verdict.ts keeps the record; this is what it says of the pool). The tokens are the
|
|
3
|
+
* parallelism the record's peak was measured under, which the task's next run reads (peak-charge.ts) to charge that
|
|
4
|
+
* peak in proportion to the parallelism it is lent. It is scheduling metadata: a record that names none, or names one
|
|
5
|
+
* that is malformed, changes no verdict a passing run records.
|
|
6
|
+
*/
|
|
7
|
+
export interface CpuGrantRecord {
|
|
8
|
+
readonly tokens: number;
|
|
9
|
+
readonly want: number;
|
|
10
|
+
readonly waitedMs: number;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** What a record's `cpu` field holds: no grant, a grant, or something that is no grant. */
|
|
14
|
+
export function parseCpuGrant(value: unknown): CpuGrantRecord | undefined | 'malformed' {
|
|
15
|
+
if (value === undefined) {
|
|
16
|
+
return undefined;
|
|
17
|
+
}
|
|
18
|
+
if (typeof value !== 'object' || value === null) {
|
|
19
|
+
return 'malformed';
|
|
20
|
+
}
|
|
21
|
+
const tokens: unknown = Reflect.get(value, 'tokens');
|
|
22
|
+
const want: unknown = Reflect.get(value, 'want');
|
|
23
|
+
const waitedMs: unknown = Reflect.get(value, 'waitedMs');
|
|
24
|
+
if (
|
|
25
|
+
!(typeof tokens === 'number' && Number.isSafeInteger(tokens) && tokens >= 1) ||
|
|
26
|
+
!(typeof want === 'number' && Number.isSafeInteger(want) && want >= 1) ||
|
|
27
|
+
!(typeof waitedMs === 'number' && Number.isFinite(waitedMs) && waitedMs >= 0)
|
|
28
|
+
) {
|
|
29
|
+
return 'malformed';
|
|
30
|
+
}
|
|
31
|
+
return { tokens, want, waitedMs };
|
|
32
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { parentPort, workerData } from 'node:worker_threads';
|
|
2
2
|
import { type CloseOutcome, createCpuPool } from './cpu-pool.js';
|
|
3
|
+
import type { RunMemory } from './host-memory.js';
|
|
3
4
|
|
|
4
5
|
/**
|
|
5
6
|
* The thread an Nx run's CPU pool runs on (`run-cpu-pool.ts`; specs/cowshed/05_gateway.md, "Every Nx run budgets
|
|
@@ -9,14 +10,14 @@ import { type CloseOutcome, createCpuPool } from './cpu-pool.js';
|
|
|
9
10
|
* jobserver fifo and heartbeat share an event loop that runs nothing else, so the heartbeat proves that the actor which
|
|
10
11
|
* admits and grants is turning.
|
|
11
12
|
*
|
|
12
|
-
* The thread is started with the pool's count and the run's memory. It says once that the pool serves, with the environment a
|
|
13
|
-
* or why it cannot; the host's one message is its close, after which the thread says how the pool closed.
|
|
13
|
+
* The thread is started with the pool's count and the run's memory. It says once that the pool serves, with the environment a
|
|
14
|
+
* task inherits, or why it cannot; the host's one message is its close, after which the thread says how the pool closed.
|
|
14
15
|
*/
|
|
15
16
|
|
|
16
|
-
/** What the thread is started with: the run's count, and the run's memory
|
|
17
|
+
/** What the thread is started with: the run's count, and the run's memory its grants are charged against. */
|
|
17
18
|
export interface CpuPoolThreadData {
|
|
18
19
|
readonly tokens: number;
|
|
19
|
-
readonly memory:
|
|
20
|
+
readonly memory: RunMemory;
|
|
20
21
|
}
|
|
21
22
|
|
|
22
23
|
/** What the thread says, in order: the pool serves or failed; after the host's close, how the pool closed. */
|
|
@@ -7,9 +7,10 @@ import { promisify } from 'node:util';
|
|
|
7
7
|
import typia from 'typia';
|
|
8
8
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
9
9
|
import { holdLease, type Unleased } from './gateway-lease.js';
|
|
10
|
+
import { type LiveUsageReading, type RunMemory, readLiveUsage } from './host-memory.js';
|
|
10
11
|
import { type CpuLeaseProgress, type CpuLeaseQueued, CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
|
|
11
12
|
import { admit } from './pool-admission.js';
|
|
12
|
-
import {
|
|
13
|
+
import { type InheritedCpuPool, poolEnvironment } from './pool-environment.js';
|
|
13
14
|
|
|
14
15
|
/**
|
|
15
16
|
* A CPU pool: the tokens a process lends the runners it starts, held live rather than inherited as a
|
|
@@ -37,16 +38,6 @@ import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
|
37
38
|
* frees the slot. A Cargo reading the fifo and a nested runner holding a socket grant so share one count.
|
|
38
39
|
*/
|
|
39
40
|
|
|
40
|
-
/** The environment name a child reads its parent's pool endpoint from; its count is `CPU_TOKENS_ENV`. */
|
|
41
|
-
export const CPU_POOL_ENV = 'BOUNDED_EXEC_CPU_POOL';
|
|
42
|
-
|
|
43
|
-
/** An inherited pool descriptor; only a grant from its endpoint proves a live allocation. */
|
|
44
|
-
export interface InheritedCpuPool {
|
|
45
|
-
readonly socket: string;
|
|
46
|
-
/** The count the parent names beside its endpoint. */
|
|
47
|
-
readonly tokens: number;
|
|
48
|
-
}
|
|
49
|
-
|
|
50
41
|
/** A running pool: the environment a child inherits to reach it, and how to close it. */
|
|
51
42
|
export interface CpuPool {
|
|
52
43
|
/** The endpoint, the count the pool actually holds, and its make jobserver fifo. */
|
|
@@ -63,12 +54,16 @@ export type CloseOutcome = { ok: true } | { ok: false; error: string };
|
|
|
63
54
|
|
|
64
55
|
/**
|
|
65
56
|
* The one line a child sends: a count of whole tokens, at least one, and the bytes it is charged, its task's last
|
|
66
|
-
* recorded memory peak. A child whose task has no recorded peak
|
|
57
|
+
* recorded memory peak, with the tokens it was lent when that peak was measured. A child whose task has no recorded peak
|
|
58
|
+
* names none: it is charged nothing. A peak with no tokens is charged flat.
|
|
67
59
|
*/
|
|
68
60
|
interface CpuPoolRequest {
|
|
69
61
|
op: 'cpu-pool';
|
|
70
62
|
want: number & typia.tags.Type<'uint32'> & typia.tags.Minimum<1>;
|
|
71
63
|
memory?: number & typia.tags.Type<'uint64'>;
|
|
64
|
+
parallelism?: number & typia.tags.Type<'uint32'> & typia.tags.Minimum<1>;
|
|
65
|
+
task?: string & typia.tags.MaxLength<256>;
|
|
66
|
+
alone?: boolean;
|
|
72
67
|
}
|
|
73
68
|
|
|
74
69
|
/** A grant whose charge alone exceeds the run's memory, made while nothing else held memory: the run's memory. */
|
|
@@ -80,7 +75,7 @@ export interface Oversized {
|
|
|
80
75
|
type CpuPoolReply =
|
|
81
76
|
| CpuLeaseProgress
|
|
82
77
|
| CpuLeaseQueued
|
|
83
|
-
| { ok: true; lease: 'granted'; tokens: number; oversized?: Oversized }
|
|
78
|
+
| { ok: true; lease: 'granted'; tokens: number; oversized?: Oversized; charged?: number }
|
|
84
79
|
| { ok: false; code: 'invalid-request'; error: string };
|
|
85
80
|
|
|
86
81
|
/** Exactly a request's fields: any other field is an invalid request. */
|
|
@@ -109,7 +104,13 @@ const run = promisify(execFile);
|
|
|
109
104
|
type Phase =
|
|
110
105
|
| { readonly kind: 'reading' }
|
|
111
106
|
| { readonly kind: 'queued' }
|
|
112
|
-
| {
|
|
107
|
+
| {
|
|
108
|
+
readonly kind: 'holding';
|
|
109
|
+
readonly bytes: number;
|
|
110
|
+
readonly slot: boolean;
|
|
111
|
+
readonly memory: number;
|
|
112
|
+
readonly alone: boolean;
|
|
113
|
+
}
|
|
113
114
|
| { readonly kind: 'done' };
|
|
114
115
|
|
|
115
116
|
/** A child connection and what the pool owes it. Its grant, or its place in line, ends with the connection. */
|
|
@@ -118,59 +119,73 @@ interface Child {
|
|
|
118
119
|
phase: Phase;
|
|
119
120
|
}
|
|
120
121
|
|
|
121
|
-
/** A queued child, the count it asked for
|
|
122
|
+
/** A queued child, the count it asked for, the bytes it is charged, the tokens that charge was measured under, its task. */
|
|
122
123
|
interface Waiter {
|
|
123
124
|
readonly child: Child;
|
|
124
125
|
readonly want: number;
|
|
125
126
|
readonly memory: number;
|
|
127
|
+
readonly parallelism?: number | undefined;
|
|
128
|
+
readonly task?: string | undefined;
|
|
129
|
+
readonly alone?: boolean | undefined;
|
|
126
130
|
}
|
|
127
131
|
|
|
128
132
|
/** One framed reply line. The newline is the framing; every reply goes through here. */
|
|
129
133
|
const frame = (reply: CpuPoolReply): string => `${stringifyReply(reply)}\n`;
|
|
130
134
|
|
|
131
135
|
/**
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
|
|
136
|
-
export function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>): InheritedCpuPool | null {
|
|
137
|
-
const socket = environment[CPU_POOL_ENV];
|
|
138
|
-
const tokens = parseInheritedTokens(environment[CPU_TOKENS_ENV]);
|
|
139
|
-
if (socket === undefined || socket.length === 0 || socket.includes('\0') || tokens === null) {
|
|
140
|
-
return null;
|
|
141
|
-
}
|
|
142
|
-
return { socket, tokens };
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
/**
|
|
146
|
-
* Tokens a parent pool lent, held until released, or why there are none. `oversized` names the run's memory when the
|
|
147
|
-
* child's charge alone exceeds it and it was granted because nothing else held memory. `closed` resolves when the
|
|
148
|
-
* grant's connection closes, however it closes, so a sub-pool built on the grant ends with it.
|
|
136
|
+
* Tokens a parent pool lent, held until released, or why there are none. `charged` is the bytes the pool charged the
|
|
137
|
+
* grant, null from a pool that charges none. `oversized` names the run's memory when the child's charge alone exceeds
|
|
138
|
+
* it and it was granted because nothing else held memory. `closed` resolves when the grant's connection closes, however
|
|
139
|
+
* it closes, so a sub-pool built on the grant ends with it.
|
|
149
140
|
*/
|
|
150
141
|
export type TokenGrant =
|
|
151
|
-
| {
|
|
142
|
+
| {
|
|
143
|
+
granted: true;
|
|
144
|
+
tokens: number;
|
|
145
|
+
charged: number | null;
|
|
146
|
+
oversized: Oversized | null;
|
|
147
|
+
release: () => void;
|
|
148
|
+
closed: Promise<void>;
|
|
149
|
+
}
|
|
152
150
|
| Unleased;
|
|
153
151
|
|
|
154
152
|
const isOversized = typia.createEquals<Oversized>();
|
|
155
153
|
|
|
154
|
+
/** What a leaf tells the pool beyond its count and its charge. */
|
|
155
|
+
export interface AskContext {
|
|
156
|
+
/** The tokens the task was lent when its recorded peak was measured. */
|
|
157
|
+
readonly parallelism?: number | undefined;
|
|
158
|
+
/** The Nx task the leaf runs, named in the pool's line for the grant. */
|
|
159
|
+
readonly task?: string | undefined;
|
|
160
|
+
/** The kernel killed the command for memory: it is run again, granted only while nothing else holds a grant. */
|
|
161
|
+
readonly alone?: boolean | undefined;
|
|
162
|
+
}
|
|
163
|
+
|
|
156
164
|
/**
|
|
157
165
|
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
158
|
-
* it has none.
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
166
|
+
* it has none. `context.parallelism` is the tokens the task was lent when that peak was measured; the pool charges the
|
|
167
|
+
* peak in proportion to the tokens it lends now when that is more, and lends fewer when the run's memory holds no more.
|
|
168
|
+
* The inherited count does not bound the ask: a pool grants at most what it holds. A granted lease is the child's local
|
|
169
|
+
* allocation: its connection is the grant, and closing it returns only that allocation. An absent endpoint grants
|
|
170
|
+
* nothing and refuses nothing; a connected endpoint's refusal or protocol failure must fail admission rather than run
|
|
171
|
+
* the child beside its parent's count.
|
|
162
172
|
*/
|
|
163
173
|
export async function takeParentCpuTokens(
|
|
164
174
|
parent: InheritedCpuPool,
|
|
165
175
|
want: number,
|
|
166
176
|
memory: number | null,
|
|
177
|
+
context: AskContext = {},
|
|
167
178
|
): Promise<TokenGrant> {
|
|
179
|
+
const { parallelism, task, alone } = context;
|
|
168
180
|
if (!Number.isSafeInteger(want) || want < 1) {
|
|
169
181
|
throw new RangeError(`a child asks its parent pool for a positive count, not ${want} (pool ${parent.tokens})`);
|
|
170
182
|
}
|
|
171
183
|
if (memory !== null && (!Number.isSafeInteger(memory) || memory < 0)) {
|
|
172
184
|
throw new RangeError(`a child is charged a whole number of bytes, not ${memory}`);
|
|
173
185
|
}
|
|
186
|
+
if (parallelism !== undefined && (!Number.isSafeInteger(parallelism) || parallelism < 1)) {
|
|
187
|
+
throw new RangeError(`a child's charge is measured under a positive count of tokens, not ${parallelism}`);
|
|
188
|
+
}
|
|
174
189
|
if (parent.socket.length === 0 || parent.socket.includes('\0')) {
|
|
175
190
|
return {
|
|
176
191
|
granted: false,
|
|
@@ -178,12 +193,18 @@ export async function takeParentCpuTokens(
|
|
|
178
193
|
reason: `the parent CPU pool's endpoint ${JSON.stringify(parent.socket)} is not a socket path`,
|
|
179
194
|
};
|
|
180
195
|
}
|
|
181
|
-
const request =
|
|
196
|
+
const request = {
|
|
197
|
+
op: 'cpu-pool',
|
|
198
|
+
want,
|
|
199
|
+
...(memory === null ? {} : { memory, ...(parallelism === undefined ? {} : { parallelism }) }),
|
|
200
|
+
...(task === undefined ? {} : { task }),
|
|
201
|
+
...(alone === true ? { alone: true } : {}),
|
|
202
|
+
};
|
|
182
203
|
const lease = await holdLease(parent.socket, request, 'the parent CPU pool', new CpuLeaseWait());
|
|
183
204
|
if (!lease.granted) {
|
|
184
205
|
return lease;
|
|
185
206
|
}
|
|
186
|
-
const { tokens, oversized } = lease.answer;
|
|
207
|
+
const { tokens, oversized, charged } = lease.answer;
|
|
187
208
|
if (typeof tokens !== 'number' || !Number.isInteger(tokens) || tokens < 1 || tokens > want) {
|
|
188
209
|
lease.release();
|
|
189
210
|
return {
|
|
@@ -200,7 +221,22 @@ export async function takeParentCpuTokens(
|
|
|
200
221
|
reason: `the parent CPU pool named its memory ${JSON.stringify(oversized)} in a grant, which is no byte count`,
|
|
201
222
|
};
|
|
202
223
|
}
|
|
203
|
-
|
|
224
|
+
if (charged !== undefined && !(typeof charged === 'number' && Number.isSafeInteger(charged) && charged >= 0)) {
|
|
225
|
+
lease.release();
|
|
226
|
+
return {
|
|
227
|
+
granted: false,
|
|
228
|
+
cause: 'other',
|
|
229
|
+
reason: `the parent CPU pool charged ${JSON.stringify(charged)} in a grant, which is no byte count`,
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
return {
|
|
233
|
+
granted: true,
|
|
234
|
+
tokens,
|
|
235
|
+
charged: charged ?? null,
|
|
236
|
+
oversized: oversized ?? null,
|
|
237
|
+
release: lease.release,
|
|
238
|
+
closed: lease.closed,
|
|
239
|
+
};
|
|
204
240
|
}
|
|
205
241
|
|
|
206
242
|
/** A thrown value's message. */
|
|
@@ -354,13 +390,15 @@ function watchFifo(
|
|
|
354
390
|
/**
|
|
355
391
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
356
392
|
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
357
|
-
* pool charges its grants against `memory
|
|
393
|
+
* pool charges its grants against `memory` and says each grant with what the run's bound holds live
|
|
394
|
+
* (`liveUsage`); a sub-pool charges none (null).
|
|
358
395
|
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
359
396
|
*/
|
|
360
397
|
export async function createCpuPool(
|
|
361
398
|
tokens: number,
|
|
362
399
|
closed: Promise<void>,
|
|
363
|
-
memory:
|
|
400
|
+
memory: RunMemory | null,
|
|
401
|
+
liveUsage: (bound: string) => LiveUsageReading = readLiveUsage,
|
|
364
402
|
): Promise<{ ok: true; pool: CpuPool } | { ok: false; error: string }> {
|
|
365
403
|
if (!Number.isSafeInteger(tokens) || tokens < 1) {
|
|
366
404
|
throw new RangeError(`a CPU pool holds a positive safe integer of tokens, not ${tokens}`);
|
|
@@ -389,6 +427,9 @@ export async function createCpuPool(
|
|
|
389
427
|
let slotLent = false;
|
|
390
428
|
// The bytes the children holding grants are charged, against `memory`.
|
|
391
429
|
let heldMemory = 0;
|
|
430
|
+
// The children holding a grant, and whether one of them runs alone.
|
|
431
|
+
let holders = 0;
|
|
432
|
+
let aloneHolder = false;
|
|
392
433
|
// The readability wait while the queue's head finds no token, else undefined.
|
|
393
434
|
let watch: FifoWatch | undefined;
|
|
394
435
|
let closing = false;
|
|
@@ -477,24 +518,39 @@ export async function createCpuPool(
|
|
|
477
518
|
const admission = admit(
|
|
478
519
|
queue,
|
|
479
520
|
inHand + (slotLent ? 0 : 1),
|
|
480
|
-
memory === null ? null : { capacity: memory, held: heldMemory },
|
|
521
|
+
memory === null ? null : { capacity: memory.bytes, held: heldMemory },
|
|
522
|
+
{ holders, alone: aloneHolder },
|
|
481
523
|
);
|
|
482
524
|
queue.splice(0, admission.grants.length);
|
|
483
|
-
for (const { request, tokens: granted, oversized } of admission.grants) {
|
|
525
|
+
for (const { request, tokens: granted, charge, oversized } of admission.grants) {
|
|
484
526
|
// Fifo bytes first, the implicit slot last.
|
|
485
527
|
const bytes = Math.min(granted, inHand);
|
|
486
528
|
inHand -= bytes;
|
|
487
529
|
const slot = granted > bytes;
|
|
488
530
|
if (slot) slotLent = true;
|
|
489
|
-
heldMemory +=
|
|
490
|
-
|
|
531
|
+
heldMemory += charge;
|
|
532
|
+
holders += 1;
|
|
533
|
+
if (request.alone === true) aloneHolder = true;
|
|
534
|
+
request.child.phase = { kind: 'holding', bytes, slot, memory: charge, alone: request.alone === true };
|
|
491
535
|
request.child.socket.write(
|
|
492
536
|
frame(
|
|
493
|
-
|
|
494
|
-
? { ok: true, lease: 'granted', tokens: granted
|
|
495
|
-
:
|
|
537
|
+
memory === null
|
|
538
|
+
? { ok: true, lease: 'granted', tokens: granted }
|
|
539
|
+
: oversized
|
|
540
|
+
? { ok: true, lease: 'granted', tokens: granted, charged: charge, oversized: { memory: memory.bytes } }
|
|
541
|
+
: { ok: true, lease: 'granted', tokens: granted, charged: charge },
|
|
496
542
|
),
|
|
497
543
|
);
|
|
544
|
+
|
|
545
|
+
if (memory !== null) {
|
|
546
|
+
const live = liveUsage(memory.source);
|
|
547
|
+
const held = live.ok
|
|
548
|
+
? `${live.usage.anon + live.usage.shmem} bytes (anon ${live.usage.anon}, shmem ${live.usage.shmem}, ${live.usage.source})`
|
|
549
|
+
: `unread (${live.reason})`;
|
|
550
|
+
process.stderr.write(
|
|
551
|
+
`bounded-exec: the CPU pool at ${socketPath} granted ${request.task ?? 'a leaf'} ${granted} of ${request.want} tokens charged ${charge} bytes: holders charged ${heldMemory} of ${memory.bytes} bytes, live anon+shmem ${held}\n`,
|
|
552
|
+
);
|
|
553
|
+
}
|
|
498
554
|
}
|
|
499
555
|
if (admission.waiting === 'cpu') {
|
|
500
556
|
watchForTokens();
|
|
@@ -530,6 +586,8 @@ export async function createCpuPool(
|
|
|
530
586
|
// A closing pool drains its fifo whole; until then the bytes go back for the next holder.
|
|
531
587
|
if (child.phase.slot) slotLent = false;
|
|
532
588
|
heldMemory -= child.phase.memory;
|
|
589
|
+
holders -= 1;
|
|
590
|
+
if (child.phase.alone) aloneHolder = false;
|
|
533
591
|
if (!closing) returnTokens(child.phase.bytes);
|
|
534
592
|
}
|
|
535
593
|
child.phase = { kind: 'done' };
|
|
@@ -590,7 +648,21 @@ export async function createCpuPool(
|
|
|
590
648
|
}
|
|
591
649
|
const request = parsed;
|
|
592
650
|
child.phase = { kind: 'queued' };
|
|
593
|
-
|
|
651
|
+
const waiter: Waiter = {
|
|
652
|
+
child,
|
|
653
|
+
want: request.want,
|
|
654
|
+
memory: request.memory ?? 0,
|
|
655
|
+
parallelism: request.parallelism,
|
|
656
|
+
task: request.task,
|
|
657
|
+
alone: request.alone,
|
|
658
|
+
};
|
|
659
|
+
if (request.alone === true) {
|
|
660
|
+
// A leaf the kernel killed has waited once already: it goes ahead of the leaves that have not run.
|
|
661
|
+
const at = queue.findIndex((waiting) => waiting.alone !== true);
|
|
662
|
+
queue.splice(at === -1 ? queue.length : at, 0, waiter);
|
|
663
|
+
} else {
|
|
664
|
+
queue.push(waiter);
|
|
665
|
+
}
|
|
594
666
|
socket.write(frame({ ok: true, lease: 'queued', server: process.pid }));
|
|
595
667
|
drain(0);
|
|
596
668
|
});
|
|
@@ -695,17 +767,10 @@ export async function createCpuPool(
|
|
|
695
767
|
if (closing) {
|
|
696
768
|
return abort(CLOSED_WHILE_MADE);
|
|
697
769
|
}
|
|
698
|
-
// Cargo and make read the fifo from here; `-j` with a jobserver names no count of its own.
|
|
699
|
-
const jobserverFlags = `-j --jobserver-auth=fifo:${fifoPath}`;
|
|
700
770
|
return {
|
|
701
771
|
ok: true,
|
|
702
772
|
pool: {
|
|
703
|
-
environment:
|
|
704
|
-
[CPU_POOL_ENV]: socketPath,
|
|
705
|
-
[CPU_TOKENS_ENV]: String(tokens),
|
|
706
|
-
CARGO_MAKEFLAGS: jobserverFlags,
|
|
707
|
-
MAKEFLAGS: jobserverFlags,
|
|
708
|
-
},
|
|
773
|
+
environment: poolEnvironment(socketPath, tokens, fifoPath),
|
|
709
774
|
close: shutdown,
|
|
710
775
|
},
|
|
711
776
|
};
|