@smoothbricks/nx-plugin 0.7.18 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -99
- package/dist/.build-inputs.json +35 -29
- package/dist/built-output-packages.js +1 -1
- package/dist/cargo-closure-input.js +1 -1
- package/dist/cargo-source-hash.d.ts +3 -21
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +4 -91
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +2 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +7 -5
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +14 -11
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +207 -69
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +3 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +66 -48
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +28 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +30 -9
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +8 -5
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +187 -283
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress.d.ts +3 -0
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +2 -2
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +26 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +26 -5
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +105 -73
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/task-results.js +2 -1
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1179 -133
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +105 -11
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/package.json +67 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/cargo-closure-input.ts +1 -1
- package/src/cargo-source-hash.ts +16 -132
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +1 -2
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +28 -15
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +280 -106
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +70 -52
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +36 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +213 -182
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress.ts +37 -2
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +2 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +39 -6
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +112 -78
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/workspace-config-policy.ts +161 -43
- package/test-timings.json +157 -147
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -26,6 +26,8 @@ export interface ObservationHost {
|
|
|
26
26
|
step(event: BoundEvent): void;
|
|
27
27
|
/** Whether the command is running a test. */
|
|
28
28
|
testing(): boolean;
|
|
29
|
+
/** Same sleep-excluding clock as the executor's idle bound. */
|
|
30
|
+
awakeNow(): number;
|
|
29
31
|
readonly queueHost: QueueHost | undefined;
|
|
30
32
|
}
|
|
31
33
|
|
|
@@ -41,7 +43,7 @@ export class AdmissionObservation {
|
|
|
41
43
|
* and ends nothing: the bounds decide, and an idle expiry names it.
|
|
42
44
|
*/
|
|
43
45
|
private failure: string | null = null;
|
|
44
|
-
/** When the last successful snapshot completed
|
|
46
|
+
/** When the last successful snapshot completed on the awake clock: credit runs to it and no further. */
|
|
45
47
|
private confirmedAt: number | null = null;
|
|
46
48
|
private readonly abort = new AbortController();
|
|
47
49
|
private timer: NodeJS.Timeout | undefined;
|
|
@@ -64,12 +66,12 @@ export class AdmissionObservation {
|
|
|
64
66
|
queueWait(reserved: boolean): AdmissionQueueWait | null {
|
|
65
67
|
return this.admitted || (this.requests.length === 0 && !reserved)
|
|
66
68
|
? null
|
|
67
|
-
: { waitedMs: admissionWaitMs(this.wait,
|
|
69
|
+
: { waitedMs: admissionWaitMs(this.wait, this.host.awakeNow()), requests: this.requests };
|
|
68
70
|
}
|
|
69
71
|
|
|
70
72
|
/** A reserved leaf's request began to wait on its run's endpoint. */
|
|
71
73
|
waiting(): void {
|
|
72
|
-
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at:
|
|
74
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: this.host.awakeNow(), waiting: true });
|
|
73
75
|
}
|
|
74
76
|
|
|
75
77
|
/** Only mbx's activity report says whether a Cargo leaf's compiler or test waits in the pool: poll it. */
|
|
@@ -82,7 +84,7 @@ export class AdmissionObservation {
|
|
|
82
84
|
|
|
83
85
|
/** The run ended: the wait stops counting and no snapshot is taken or credited again. */
|
|
84
86
|
stop(): void {
|
|
85
|
-
this.wait = stepAdmissionWait(this.wait, { kind: 'stop', at:
|
|
87
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'stop', at: this.host.awakeNow() });
|
|
86
88
|
clearInterval(this.timer);
|
|
87
89
|
this.abort.abort();
|
|
88
90
|
}
|
|
@@ -101,11 +103,11 @@ export class AdmissionObservation {
|
|
|
101
103
|
}
|
|
102
104
|
this.failure = detail;
|
|
103
105
|
this.requests = [];
|
|
104
|
-
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at:
|
|
106
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: this.host.awakeNow(), waiting: false });
|
|
105
107
|
// The wait since the last confirmed snapshot is unconfirmed: the idle bound resumes from that snapshot.
|
|
106
108
|
this.host.step({
|
|
107
109
|
kind: 'admission-queue',
|
|
108
|
-
at: this.confirmedAt ??
|
|
110
|
+
at: this.confirmedAt ?? this.host.awakeNow(),
|
|
109
111
|
waiting: false,
|
|
110
112
|
testing: this.host.testing(),
|
|
111
113
|
});
|
|
@@ -129,9 +131,8 @@ export class AdmissionObservation {
|
|
|
129
131
|
}
|
|
130
132
|
this.lastSnapshotDiagnostics = diagnostics;
|
|
131
133
|
const waiting = snapshot.requests.length > 0;
|
|
132
|
-
const observedAt =
|
|
133
|
-
|
|
134
|
-
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: observedWaitAt, waiting });
|
|
134
|
+
const observedAt = this.host.awakeNow();
|
|
135
|
+
this.wait = stepAdmissionWait(this.wait, { kind: 'snapshot', at: observedAt, waiting });
|
|
135
136
|
this.failure = null;
|
|
136
137
|
this.confirmedAt = observedAt;
|
|
137
138
|
if (waiting) {
|
|
@@ -142,7 +143,7 @@ export class AdmissionObservation {
|
|
|
142
143
|
this.lastDiagnosticAt === null ||
|
|
143
144
|
Date.now() - this.lastDiagnosticAt >= HANG_CAPTURE_MS
|
|
144
145
|
) {
|
|
145
|
-
const waitedMs = admissionWaitMs(this.wait,
|
|
146
|
+
const waitedMs = admissionWaitMs(this.wait, observedAt);
|
|
146
147
|
this.host.tell(`cowshed: mbx ${describeAdmissionQueue({ waitedMs, requests: snapshot.requests })}\n`);
|
|
147
148
|
this.lastDiagnostic = positions;
|
|
148
149
|
this.lastDiagnosticAt = Date.now();
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/** The idle clock tracks awake time, not wall time or CPU consumed by this process. */
|
|
2
|
+
export async function openAwakeClock(): Promise<() => number> {
|
|
3
|
+
// Bun 1.4.4 uses Rust Instant, and Linux Node uses CLOCK_MONOTONIC. Darwin Node's libuv uses
|
|
4
|
+
// mach_continuous_time, which includes sleep; its awake reading comes from the cowshed addon.
|
|
5
|
+
if (process.platform !== 'darwin' || process.versions.bun !== undefined) return () => performance.now();
|
|
6
|
+
const { awakeTimeMs } = await import('@smoothbricks/cowshed/awake-clock');
|
|
7
|
+
awakeTimeMs(); // Refuse a missing or stale addon before the executor spawns anything.
|
|
8
|
+
return awakeTimeMs;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
/** A timer is only a wake-up. An early callback arms the remaining time on the same clock. */
|
|
12
|
+
export function atClockDeadline(at: number, now: () => number, reached: () => void): () => void {
|
|
13
|
+
const wake = (): void => {
|
|
14
|
+
const remaining = at - now();
|
|
15
|
+
if (remaining > 0) timer = setTimeout(wake, Math.ceil(remaining));
|
|
16
|
+
else reached();
|
|
17
|
+
};
|
|
18
|
+
let timer = setTimeout(wake, Math.ceil(Math.max(0, at - now())));
|
|
19
|
+
return () => clearTimeout(timer);
|
|
20
|
+
}
|
|
@@ -20,8 +20,8 @@ export interface BoundLimits {
|
|
|
20
20
|
}
|
|
21
21
|
|
|
22
22
|
/**
|
|
23
|
-
* When each armed bound expires
|
|
24
|
-
* its pool's grant, so
|
|
23
|
+
* When each armed bound expires: idle on the awake clock, fixture and total on the wall clock. A null bound is not
|
|
24
|
+
* armed. A queued run waits for its pool's grant, so only a fixture started at spawn can have a ceiling then.
|
|
25
25
|
*/
|
|
26
26
|
export interface Bounds {
|
|
27
27
|
readonly idle: number | null;
|
|
@@ -39,18 +39,25 @@ export type BoundEvent =
|
|
|
39
39
|
/**
|
|
40
40
|
* The command started; `testing` is the lifecycle's answer then, `fixture` whether a supervised fixture leads it,
|
|
41
41
|
* `queued` whether mbx schedules this leaf, as known before spawn, and `fixtureFirst` whether that fixture runs
|
|
42
|
-
* before the leaf's reservation (
|
|
42
|
+
* before the leaf's reservation (every reserved leaf's does), so its ceiling starts now, not at the grant.
|
|
43
43
|
*/
|
|
44
44
|
| {
|
|
45
45
|
readonly kind: 'spawn';
|
|
46
46
|
readonly at: number;
|
|
47
|
+
readonly wallAt: number;
|
|
47
48
|
readonly testing: boolean;
|
|
48
49
|
readonly fixture: boolean;
|
|
49
50
|
readonly queued: boolean;
|
|
50
51
|
readonly fixtureFirst: boolean;
|
|
51
52
|
}
|
|
52
53
|
/** Each admission begins a hold; the first one starts the queued leaf's ceilings. */
|
|
53
|
-
| {
|
|
54
|
+
| {
|
|
55
|
+
readonly kind: 'granted';
|
|
56
|
+
readonly at: number;
|
|
57
|
+
readonly wallAt: number;
|
|
58
|
+
readonly testing: boolean;
|
|
59
|
+
readonly fixture: boolean;
|
|
60
|
+
}
|
|
54
61
|
| { readonly kind: 'released'; readonly at: number }
|
|
55
62
|
/** A successful public snapshot proved whether a live task process is queued; `at` is completion, not poll start. */
|
|
56
63
|
| { readonly kind: 'admission-queue'; readonly at: number; readonly waiting: boolean; readonly testing: boolean }
|
|
@@ -83,12 +90,12 @@ export const UNARMED: Bounds = {
|
|
|
83
90
|
idleRemaining: null,
|
|
84
91
|
};
|
|
85
92
|
|
|
86
|
-
/** The bounds a command
|
|
87
|
-
function armed(limits: BoundLimits,
|
|
93
|
+
/** The bounds a command arms when it starts running. */
|
|
94
|
+
function armed(limits: BoundLimits, event: Extract<BoundEvent, { kind: 'spawn' | 'granted' }>): Bounds {
|
|
88
95
|
return {
|
|
89
|
-
idle: testing && limits.idleMs !== undefined ? at + limits.idleMs : null,
|
|
90
|
-
fixture: fixture && limits.fixtureMs !== undefined ?
|
|
91
|
-
total: limits.totalMs !== undefined ?
|
|
96
|
+
idle: event.testing && limits.idleMs !== undefined ? event.at + limits.idleMs : null,
|
|
97
|
+
fixture: event.fixture && limits.fixtureMs !== undefined ? event.wallAt + limits.fixtureMs : null,
|
|
98
|
+
total: limits.totalMs !== undefined ? event.wallAt + limits.totalMs : null,
|
|
92
99
|
queued: false,
|
|
93
100
|
admissionWaiting: false,
|
|
94
101
|
holds: null,
|
|
@@ -106,13 +113,13 @@ export function stepBounds(bounds: Bounds, limits: BoundLimits, event: BoundEven
|
|
|
106
113
|
queued: true,
|
|
107
114
|
holds: 0,
|
|
108
115
|
idleRemaining: limits.idleMs ?? null,
|
|
109
|
-
fixture: event.fixtureFirst && limits.fixtureMs !== undefined ? event.
|
|
116
|
+
fixture: event.fixtureFirst && limits.fixtureMs !== undefined ? event.wallAt + limits.fixtureMs : null,
|
|
110
117
|
}
|
|
111
|
-
: armed(limits, event
|
|
118
|
+
: armed(limits, event);
|
|
112
119
|
case 'granted': {
|
|
113
120
|
if (bounds.holds === null) return bounds;
|
|
114
121
|
if (bounds.queued) {
|
|
115
|
-
const admitted = armed(limits, event
|
|
122
|
+
const admitted = armed(limits, event);
|
|
116
123
|
return {
|
|
117
124
|
...admitted,
|
|
118
125
|
holds: 1,
|
|
@@ -177,11 +184,17 @@ export function stepBounds(bounds: Bounds, limits: BoundLimits, event: BoundEven
|
|
|
177
184
|
}
|
|
178
185
|
|
|
179
186
|
/** The bound that expires first, or `null` when none is armed. A tie names the total, then the fixture, then idle. */
|
|
180
|
-
export function earliestBound(
|
|
181
|
-
|
|
187
|
+
export function earliestBound(
|
|
188
|
+
bounds: Bounds,
|
|
189
|
+
awakeNow: number,
|
|
190
|
+
wallNow: number,
|
|
191
|
+
): { readonly kind: BoundKind; readonly remainingMs: number } | null {
|
|
192
|
+
let earliest: { kind: BoundKind; remainingMs: number } | null = null;
|
|
182
193
|
for (const kind of ['total', 'fixture', 'idle'] as const) {
|
|
183
194
|
const at = bounds[kind];
|
|
184
|
-
if (at
|
|
195
|
+
if (at === null) continue;
|
|
196
|
+
const remainingMs = at - (kind === 'idle' ? awakeNow : wallNow);
|
|
197
|
+
if (earliest === null || remainingMs < earliest.remainingMs) earliest = { kind, remainingMs };
|
|
185
198
|
}
|
|
186
199
|
return earliest;
|
|
187
200
|
}
|
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
import { rm } from 'node:fs/promises';
|
|
2
|
+
import { createServer, type Server, type Socket } from 'node:net';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import typia from 'typia';
|
|
5
|
+
import { ownedSocketDirectory } from '../../testing.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* A reserved nextest leaf's case gate (specs/cowshed/05_gateway.md, "A case starts on a core"). The leaf's runner
|
|
9
|
+
* starts every thread of its ask, and each case takes a core before it runs: one of the leaf's grant when one is free,
|
|
10
|
+
* else one reserved for that case alone, which the leaf's executor asks its run's endpoint for and gives back when the
|
|
11
|
+
* case ends. A grant never grows, so this is how the cores other work frees after the leaf's grant reach it.
|
|
12
|
+
*
|
|
13
|
+
* The gate is a Unix stream socket the executor serves for one run and names in the command's environment as
|
|
14
|
+
* {@link TEST_GATE_ENV}. mbx's test shim, through which nextest starts every case, connects before the case starts,
|
|
15
|
+
* writes one line ({@link CaseLine}), starts the case when the line {@link GO} answers, and keeps the connection until
|
|
16
|
+
* the case has exited: the connection's end is the case's end.
|
|
17
|
+
*/
|
|
18
|
+
export const TEST_GATE_ENV = 'MBX_TEST_GATE';
|
|
19
|
+
/**
|
|
20
|
+
* Set to `1` in a run's environment once its endpoint answered that the installed mbx's shim gates its cases
|
|
21
|
+
* (endpoint-life.ts, `gatesTests`): without it a leaf's runner is sized to its grant.
|
|
22
|
+
*/
|
|
23
|
+
export const TEST_GATE_SERVED_ENV = 'BOUNDED_EXEC_MBX_TEST_GATE';
|
|
24
|
+
/** The line that starts a case. */
|
|
25
|
+
export const GO = 'go';
|
|
26
|
+
|
|
27
|
+
/** The line a case's shim writes: nextest's binary id and test name, which together are the case's result id. */
|
|
28
|
+
interface CaseLine {
|
|
29
|
+
binary: string;
|
|
30
|
+
test: string;
|
|
31
|
+
}
|
|
32
|
+
const parseCaseLine = typia.json.createValidateParse<CaseLine>();
|
|
33
|
+
|
|
34
|
+
/** A case's id as its result names it (progress.ts): the binary id, `::`, the test. */
|
|
35
|
+
export function caseId(binary: string, test: string): string {
|
|
36
|
+
return `${binary}::${test}`;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** What the gate does outside itself. */
|
|
40
|
+
export interface CaseGateEffects {
|
|
41
|
+
/** Start the case: answer its connection. */
|
|
42
|
+
readonly start: (id: string) => void;
|
|
43
|
+
/** Ask the pool for one core in the case's name. */
|
|
44
|
+
readonly reserve: (id: string) => void;
|
|
45
|
+
/** Give the case's core back to the pool, or withdraw its request while it waits. */
|
|
46
|
+
readonly release: (id: string) => void;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** What a leaf's cases did at its gate. */
|
|
50
|
+
export interface CaseGateSummary {
|
|
51
|
+
/** Cases that ran on the leaf's grant, and on a core of their own. */
|
|
52
|
+
readonly own: number;
|
|
53
|
+
readonly lent: number;
|
|
54
|
+
/** How long the cases on a core of their own held it, summed. */
|
|
55
|
+
readonly lentCoreMs: number;
|
|
56
|
+
/** How long the cases waited at the gate, summed. */
|
|
57
|
+
readonly waitedCaseMs: number;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
type CaseState =
|
|
61
|
+
| { readonly kind: 'waiting'; readonly since: number }
|
|
62
|
+
| { readonly kind: 'own' }
|
|
63
|
+
| { readonly kind: 'lent'; readonly since: number };
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The gate's decisions, as a state machine over three events: a case arrived, the pool granted a case's request, a
|
|
67
|
+
* case ended. `own` is the leaf's granted cores; `wall` a case's last recorded wall, by which a freed core of the
|
|
68
|
+
* grant picks the longest waiting case, so the cores the pool gives go to the shorter ones and come back soonest.
|
|
69
|
+
*/
|
|
70
|
+
export class CaseGate {
|
|
71
|
+
readonly #cases = new Map<string, CaseState>();
|
|
72
|
+
readonly #waited = new Map<string, number>();
|
|
73
|
+
#ownRunning = 0;
|
|
74
|
+
#own = 0;
|
|
75
|
+
#lent = 0;
|
|
76
|
+
#lentCoreMs = 0;
|
|
77
|
+
#waitedCaseMs = 0;
|
|
78
|
+
#ranWallMs = 0;
|
|
79
|
+
|
|
80
|
+
constructor(
|
|
81
|
+
private readonly own: number,
|
|
82
|
+
private readonly wall: (id: string) => number,
|
|
83
|
+
private readonly effects: CaseGateEffects,
|
|
84
|
+
private readonly now: () => number = Date.now,
|
|
85
|
+
) {}
|
|
86
|
+
|
|
87
|
+
/** A case's shim connected. False for a case already at the gate, which its second connection may not join. */
|
|
88
|
+
arrived(id: string): boolean {
|
|
89
|
+
if (this.#cases.has(id)) return false;
|
|
90
|
+
if (this.#ownRunning < this.own) {
|
|
91
|
+
this.#startOwn(id, 0);
|
|
92
|
+
return true;
|
|
93
|
+
}
|
|
94
|
+
this.#cases.set(id, { kind: 'waiting', since: this.now() });
|
|
95
|
+
this.effects.reserve(id);
|
|
96
|
+
return true;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The pool granted the case's request. One whose case no longer waits gives the core straight back. */
|
|
100
|
+
granted(id: string): void {
|
|
101
|
+
const state = this.#cases.get(id);
|
|
102
|
+
if (state?.kind !== 'waiting') {
|
|
103
|
+
this.effects.release(id);
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
const at = this.now();
|
|
107
|
+
this.#startedAfter(id, at - state.since);
|
|
108
|
+
this.#cases.set(id, { kind: 'lent', since: at });
|
|
109
|
+
this.#lent += 1;
|
|
110
|
+
this.effects.start(id);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** A case's connection ended: its core goes back where it came from, a waiting case's request is withdrawn. */
|
|
114
|
+
ended(id: string): void {
|
|
115
|
+
const state = this.#cases.get(id);
|
|
116
|
+
if (state === undefined) return;
|
|
117
|
+
this.#cases.delete(id);
|
|
118
|
+
if (state.kind !== 'waiting') this.#ranWallMs += this.wall(id);
|
|
119
|
+
switch (state.kind) {
|
|
120
|
+
case 'waiting':
|
|
121
|
+
this.effects.release(id);
|
|
122
|
+
return;
|
|
123
|
+
case 'lent':
|
|
124
|
+
this.#lentCoreMs += this.now() - state.since;
|
|
125
|
+
this.effects.release(id);
|
|
126
|
+
return;
|
|
127
|
+
case 'own': {
|
|
128
|
+
this.#ownRunning -= 1;
|
|
129
|
+
let longest: { id: string; since: number } | null = null;
|
|
130
|
+
for (const [waiting, other] of this.#cases) {
|
|
131
|
+
if (other.kind !== 'waiting') continue;
|
|
132
|
+
if (longest === null || this.wall(waiting) > this.wall(longest.id)) {
|
|
133
|
+
longest = { id: waiting, since: other.since };
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
if (longest === null) return;
|
|
137
|
+
this.effects.release(longest.id);
|
|
138
|
+
this.#startOwn(longest.id, this.now() - longest.since);
|
|
139
|
+
return;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** How long the case waited at the gate before it started, in ms; zero for one that never did. */
|
|
145
|
+
waitedMs(id: string): number {
|
|
146
|
+
return this.#waited.get(id) ?? 0;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** The last recorded walls of the cases that ran and ended, summed. */
|
|
150
|
+
ranWallMs(): number {
|
|
151
|
+
return this.#ranWallMs;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
summary(): CaseGateSummary {
|
|
155
|
+
return { own: this.#own, lent: this.#lent, lentCoreMs: this.#lentCoreMs, waitedCaseMs: this.#waitedCaseMs };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
#startOwn(id: string, waitedMs: number): void {
|
|
159
|
+
this.#startedAfter(id, waitedMs);
|
|
160
|
+
this.#cases.set(id, { kind: 'own' });
|
|
161
|
+
this.#ownRunning += 1;
|
|
162
|
+
this.#own += 1;
|
|
163
|
+
this.effects.start(id);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
#startedAfter(id: string, waitedMs: number): void {
|
|
167
|
+
this.#waited.set(id, waitedMs);
|
|
168
|
+
this.#waitedCaseMs += waitedMs;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** One core asked of a pool for one case: whether the pool granted it, and how it is given back or withdrawn. */
|
|
173
|
+
export interface CaseCore {
|
|
174
|
+
/** Settles once the pool grants the core (true), or refuses, loses or is withdrawn the request (false). */
|
|
175
|
+
readonly granted: Promise<boolean>;
|
|
176
|
+
/** Gives the core back, or withdraws the request while it waits. Later calls do nothing. */
|
|
177
|
+
release(): void;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** A leaf's gate decisions and the cores its waiting cases asked their pool for. */
|
|
181
|
+
export interface GatedCases {
|
|
182
|
+
readonly cases: CaseGate;
|
|
183
|
+
/** Gives back every core a case still holds and withdraws every request that waits. */
|
|
184
|
+
releaseAll(): void;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* The gate of a leaf that holds `own` cores, over whichever pool `ask` asks: mbx's, through the run's reservation
|
|
189
|
+
* endpoint, or the run's own token pool (cpu-pool.ts, `lendParentToken`). `start` answers a case's connection.
|
|
190
|
+
* `walls` are the cases' last recorded walls; a request is told the work the leaf has left, their sum less the walls
|
|
191
|
+
* of the cases that ran, by which a token pool lends to the leaf that would otherwise end last.
|
|
192
|
+
*/
|
|
193
|
+
export function gateCases(
|
|
194
|
+
own: number,
|
|
195
|
+
walls: ReadonlyMap<string, number>,
|
|
196
|
+
start: (id: string) => void,
|
|
197
|
+
ask: (id: string, leftMs: number) => CaseCore,
|
|
198
|
+
): GatedCases {
|
|
199
|
+
const requests = new Map<string, CaseCore>();
|
|
200
|
+
let total = 0;
|
|
201
|
+
for (const wall of walls.values()) total += wall;
|
|
202
|
+
const cases: CaseGate = new CaseGate(own, (id) => walls.get(id) ?? 0, {
|
|
203
|
+
start,
|
|
204
|
+
reserve: (id) => {
|
|
205
|
+
const held = ask(id, Math.max(0, total - cases.ranWallMs()));
|
|
206
|
+
requests.set(id, held);
|
|
207
|
+
void held.granted.then((granted) => {
|
|
208
|
+
// Released since: the case started on a core of the grant, or ended while it waited.
|
|
209
|
+
if (requests.get(id) !== held) return;
|
|
210
|
+
if (granted) cases.granted(id);
|
|
211
|
+
else requests.delete(id);
|
|
212
|
+
});
|
|
213
|
+
},
|
|
214
|
+
release: (id) => {
|
|
215
|
+
requests.get(id)?.release();
|
|
216
|
+
requests.delete(id);
|
|
217
|
+
},
|
|
218
|
+
});
|
|
219
|
+
return {
|
|
220
|
+
cases,
|
|
221
|
+
releaseAll: () => {
|
|
222
|
+
for (const held of requests.values()) held.release();
|
|
223
|
+
requests.clear();
|
|
224
|
+
},
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* A case's wall as its result records it: nextest times a case from the shim's start, so its own duration includes the
|
|
230
|
+
* wait at the gate, which is taken off. Never below zero.
|
|
231
|
+
*/
|
|
232
|
+
export function runWallMs(nextestMs: number, waitedMs: number): number {
|
|
233
|
+
return Math.max(0, nextestMs - waitedMs);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const GATE_DIRECTORY = 'bounded-gate-';
|
|
237
|
+
const GATE_SOCKET = 'g.sock';
|
|
238
|
+
/** No case line is longer: a binary id and a test name. */
|
|
239
|
+
const LINE_BYTES = 16 * 1024;
|
|
240
|
+
|
|
241
|
+
export interface CaseGateServer {
|
|
242
|
+
/** The socket's path, as the command's environment names it. */
|
|
243
|
+
readonly socket: string;
|
|
244
|
+
/** Answer the case's connection: it starts. */
|
|
245
|
+
start(id: string): void;
|
|
246
|
+
/** Ends every connection and removes the socket's directory; the error that left something behind, or null. */
|
|
247
|
+
close(): Promise<string | null>;
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
export interface CaseGateHandlers {
|
|
251
|
+
/** A case's shim connected and named it. False refuses the connection. */
|
|
252
|
+
readonly arrived: (id: string) => boolean;
|
|
253
|
+
/** A named case's connection ended. */
|
|
254
|
+
readonly ended: (id: string) => void;
|
|
255
|
+
/** A client wrote something that is no case line, or named a case already at the gate; its connection is ended. */
|
|
256
|
+
readonly invalid: (detail: string) => void;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
function describe(error: unknown): string {
|
|
260
|
+
return error instanceof Error ? error.message : String(error);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/** Serve one run's case gate; a gate that cannot be served is the run's refusal. */
|
|
264
|
+
export async function serveCaseGate(
|
|
265
|
+
handlers: CaseGateHandlers,
|
|
266
|
+
): Promise<{ ok: true; gate: CaseGateServer } | { ok: false; error: string }> {
|
|
267
|
+
let directory: string;
|
|
268
|
+
try {
|
|
269
|
+
directory = await ownedSocketDirectory(GATE_DIRECTORY, GATE_SOCKET);
|
|
270
|
+
} catch (error) {
|
|
271
|
+
return { ok: false, error: `its directory could not be made: ${describe(error)}` };
|
|
272
|
+
}
|
|
273
|
+
const socketPath = join(directory, GATE_SOCKET);
|
|
274
|
+
const clients = new Set<Socket>();
|
|
275
|
+
const named = new Map<string, Socket>();
|
|
276
|
+
const serve = (client: Socket): void => {
|
|
277
|
+
clients.add(client);
|
|
278
|
+
// A case that is killed loses its connection with its process: that end is the case's end, read below.
|
|
279
|
+
client.on('error', () => {});
|
|
280
|
+
client.setEncoding('utf8');
|
|
281
|
+
let partial = '';
|
|
282
|
+
let id: string | null = null;
|
|
283
|
+
const refuse = (detail: string): void => {
|
|
284
|
+
client.destroy();
|
|
285
|
+
handlers.invalid(detail);
|
|
286
|
+
};
|
|
287
|
+
client.on('data', (text: string) => {
|
|
288
|
+
if (id !== null) {
|
|
289
|
+
refuse(`case ${id} wrote a second line at its gate`);
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
partial += text;
|
|
293
|
+
const end = partial.indexOf('\n');
|
|
294
|
+
if (end === -1) {
|
|
295
|
+
if (partial.length > LINE_BYTES) refuse(`a case line is at most ${LINE_BYTES} bytes`);
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
let parsed: ReturnType<typeof parseCaseLine> | null;
|
|
299
|
+
try {
|
|
300
|
+
parsed = parseCaseLine(partial.slice(0, end));
|
|
301
|
+
} catch {
|
|
302
|
+
parsed = null;
|
|
303
|
+
}
|
|
304
|
+
if (parsed === null || !parsed.success || partial.length > end + 1) {
|
|
305
|
+
refuse(`a case line names a binary and a test, one line; got ${JSON.stringify(partial.slice(0, 200))}`);
|
|
306
|
+
return;
|
|
307
|
+
}
|
|
308
|
+
const arriving = caseId(parsed.data.binary, parsed.data.test);
|
|
309
|
+
named.set(arriving, client);
|
|
310
|
+
if (!handlers.arrived(arriving)) {
|
|
311
|
+
named.delete(arriving);
|
|
312
|
+
refuse(`case ${arriving} is already at its gate`);
|
|
313
|
+
return;
|
|
314
|
+
}
|
|
315
|
+
id = arriving;
|
|
316
|
+
});
|
|
317
|
+
client.on('close', () => {
|
|
318
|
+
clients.delete(client);
|
|
319
|
+
if (id === null) return;
|
|
320
|
+
if (named.get(id) === client) named.delete(id);
|
|
321
|
+
handlers.ended(id);
|
|
322
|
+
});
|
|
323
|
+
};
|
|
324
|
+
const server: Server = createServer(serve);
|
|
325
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: listen is this process's own bind of a path in its fresh directory; the kernel answers it with the listening callback or an error, and no peer takes part
|
|
326
|
+
const listened = await new Promise<Error | null>((settle) => {
|
|
327
|
+
server.once('error', settle);
|
|
328
|
+
server.listen(socketPath, () => {
|
|
329
|
+
server.off('error', settle);
|
|
330
|
+
settle(null);
|
|
331
|
+
});
|
|
332
|
+
});
|
|
333
|
+
const close = async (): Promise<string | null> => {
|
|
334
|
+
const failures: string[] = [];
|
|
335
|
+
for (const client of clients) client.destroy();
|
|
336
|
+
if (listened === null) {
|
|
337
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: every accepted connection was destroyed just above, so the server closes without waiting on a peer
|
|
338
|
+
const closed = await new Promise<Error | undefined>((settle) => server.close(settle));
|
|
339
|
+
if (closed !== undefined) failures.push(`its socket did not close: ${closed.message}`);
|
|
340
|
+
}
|
|
341
|
+
try {
|
|
342
|
+
await rm(directory, { recursive: true, force: true });
|
|
343
|
+
} catch (error) {
|
|
344
|
+
failures.push(`its directory was not removed: ${describe(error)}`);
|
|
345
|
+
}
|
|
346
|
+
return failures.length === 0 ? null : `the case gate at ${socketPath}: ${failures.join('; ')}`;
|
|
347
|
+
};
|
|
348
|
+
if (listened !== null) {
|
|
349
|
+
const left = await close();
|
|
350
|
+
return {
|
|
351
|
+
ok: false,
|
|
352
|
+
error: `its socket ${socketPath} could not be bound: ${listened.message}${left === null ? '' : `; ${left}`}`,
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
return {
|
|
356
|
+
ok: true,
|
|
357
|
+
gate: {
|
|
358
|
+
socket: socketPath,
|
|
359
|
+
start: (id) => {
|
|
360
|
+
named.get(id)?.write(`${GO}\n`);
|
|
361
|
+
},
|
|
362
|
+
close,
|
|
363
|
+
},
|
|
364
|
+
};
|
|
365
|
+
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { parentPort, workerData } from 'node:worker_threads';
|
|
2
|
-
import { type CloseOutcome, createCpuPool } from './cpu-pool.js';
|
|
2
|
+
import { type CloseOutcome, createCpuPool, describeIdle } from './cpu-pool.js';
|
|
3
3
|
import type { RunMemory } from './host-memory.js';
|
|
4
4
|
|
|
5
5
|
/**
|
|
@@ -24,7 +24,7 @@ export interface CpuPoolThreadData {
|
|
|
24
24
|
export type CpuPoolThreadReport =
|
|
25
25
|
| { readonly kind: 'serving'; readonly environment: Record<string, string> }
|
|
26
26
|
| { readonly kind: 'failed'; readonly error: string }
|
|
27
|
-
| { readonly kind: 'closed'; readonly outcome: CloseOutcome };
|
|
27
|
+
| { readonly kind: 'closed'; readonly outcome: CloseOutcome; readonly idle: string | null };
|
|
28
28
|
|
|
29
29
|
const host = parentPort;
|
|
30
30
|
if (host === null) {
|
|
@@ -37,7 +37,7 @@ const made = await createCpuPool(data.tokens, new Promise<void>(() => {}), data.
|
|
|
37
37
|
if (made.ok) {
|
|
38
38
|
const { pool } = made;
|
|
39
39
|
host.once('message', () => {
|
|
40
|
-
void pool.close().then((outcome) => report({ kind: 'closed', outcome }));
|
|
40
|
+
void pool.close().then((outcome) => report({ kind: 'closed', outcome, idle: describeIdle(made.summary()) }));
|
|
41
41
|
});
|
|
42
42
|
report({ kind: 'serving', environment: pool.environment });
|
|
43
43
|
} else {
|