@smoothbricks/nx-plugin 0.7.18 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +160 -99
- package/dist/.build-inputs.json +35 -29
- package/dist/built-output-packages.js +1 -1
- package/dist/cargo-closure-input.js +1 -1
- package/dist/cargo-source-hash.d.ts +3 -21
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +4 -91
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +2 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +7 -5
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +14 -11
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +207 -69
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +3 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +66 -48
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +28 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +30 -9
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +8 -5
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +187 -283
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress.d.ts +3 -0
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +2 -2
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +26 -4
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +26 -5
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +105 -73
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/task-results.js +2 -1
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1179 -133
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +105 -11
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/package.json +67 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/cargo-closure-input.ts +1 -1
- package/src/cargo-source-hash.ts +16 -132
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +1 -2
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +28 -15
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +280 -106
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +70 -52
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +36 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +213 -182
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress.ts +37 -2
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +2 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +39 -6
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +112 -78
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/workspace-config-policy.ts +161 -43
- package/test-timings.json +157 -147
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -1,14 +1,21 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
2
|
import { rm } from 'node:fs/promises';
|
|
3
3
|
import { createConnection, type Socket } from 'node:net';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
-
import type { Readable, Writable } from 'node:stream';
|
|
6
5
|
import { logger } from 'nx/src/devkit-exports.js';
|
|
7
6
|
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
8
|
-
import typia, { type tags } from 'typia';
|
|
9
7
|
import { ownedSocketDirectory } from '../../testing.js';
|
|
10
|
-
import {
|
|
11
|
-
import { type
|
|
8
|
+
import { TEST_GATE_SERVED_ENV } from './case-gate.js';
|
|
9
|
+
import { type Ended, type EndpointEffect, type EndpointEvent, endpointStep, spawned } from './endpoint-life.js';
|
|
10
|
+
import { type MbxPriority, mbxHold, mbxPool } from './mbx-pool.js';
|
|
11
|
+
import {
|
|
12
|
+
type Connection,
|
|
13
|
+
type ConnectionEffect,
|
|
14
|
+
type ConnectionEvent,
|
|
15
|
+
connectionStep,
|
|
16
|
+
opened,
|
|
17
|
+
type ReservationRefusal,
|
|
18
|
+
} from './reservation-connection.js';
|
|
12
19
|
|
|
13
20
|
/**
|
|
14
21
|
* mbx's reservation endpoint of one Nx run (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A reserved leaf
|
|
@@ -53,17 +60,16 @@ export function describeUnserved(endpoint: Exclude<RunEndpoint, { kind: 'served'
|
|
|
53
60
|
|
|
54
61
|
const ENDPOINT_DIRECTORY = 'mbx-r-';
|
|
55
62
|
const ENDPOINT_SOCKET = 'r.sock';
|
|
56
|
-
/** What is kept of the endpoint's stderr to say why it ended: its last bytes. */
|
|
57
|
-
const SAID_BYTES = 4096;
|
|
58
63
|
|
|
59
|
-
/** A run's endpoint
|
|
64
|
+
/** A run's endpoint once it serves. */
|
|
60
65
|
interface Endpoint {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
readonly ended: Promise<string>;
|
|
66
|
+
/** The run's end, which ends the endpoint: how it ended, once its socket's directory is removed. */
|
|
67
|
+
end(): Promise<Ended>;
|
|
64
68
|
}
|
|
65
69
|
|
|
66
|
-
type Started =
|
|
70
|
+
type Started =
|
|
71
|
+
| { readonly ok: true; readonly endpoint: Endpoint; readonly socket: string; readonly gates: boolean }
|
|
72
|
+
| { readonly ok: false; readonly error: string };
|
|
67
73
|
|
|
68
74
|
/** The endpoints this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
69
75
|
const endpoints = new Map<string, Endpoint>();
|
|
@@ -71,13 +77,15 @@ const endpoints = new Map<string, Endpoint>();
|
|
|
71
77
|
/**
|
|
72
78
|
* Serve `context`'s run an mbx reservation endpoint and publish it in `env`, which every task inherits, when the
|
|
73
79
|
* environment names mbx's pool (`mbxPool`). An endpoint that does not start is said, and each reserved leaf of the run
|
|
74
|
-
* then refuses to run, naming `RESERVATIONS_ENV`; a leaf in no pool, or one mbx admits itself, runs as before.
|
|
80
|
+
* then refuses to run, naming `RESERVATIONS_ENV`; a leaf in no pool, or one mbx admits itself, runs as before. A run
|
|
81
|
+
* under a hold on mbx's pool is served no endpoint, and learns whether its mbx gates test cases (`learnHeldGate`).
|
|
75
82
|
*/
|
|
76
83
|
export async function serveRunReservations(
|
|
77
84
|
context: Pick<PreTasksExecutionContext, 'id'>,
|
|
78
85
|
env: NodeJS.ProcessEnv = process.env,
|
|
79
86
|
): Promise<void> {
|
|
80
87
|
if (mbxPool(env) === null) {
|
|
88
|
+
if (mbxHold(env)) await learnHeldGate(env);
|
|
81
89
|
return;
|
|
82
90
|
}
|
|
83
91
|
const started = await startEndpoint(env);
|
|
@@ -91,6 +99,33 @@ export async function serveRunReservations(
|
|
|
91
99
|
endpoints.set(context.id, started.endpoint);
|
|
92
100
|
env[RESERVATIONS_ENV] = started.socket;
|
|
93
101
|
delete env[RESERVATIONS_UNSERVED_ENV];
|
|
102
|
+
// A leaf's runner starts every thread of its ask only where the installed mbx's shim takes each case through its gate.
|
|
103
|
+
if (started.gates) env[TEST_GATE_SERVED_ENV] = '1';
|
|
104
|
+
else delete env[TEST_GATE_SERVED_ENV];
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* A run under a hold on mbx's pool reserves nothing and serves its own token pool (run-cpu-pool.ts), and its nextest
|
|
109
|
+
* leaves still start every case through mbx's shim. Whether that shim gates is the endpoint's own answer, so the run
|
|
110
|
+
* starts the endpoint once, outside the hold's `MBX_SCHED_DISABLE` and `MBX_SCHED_DIR`, which it refuses to serve
|
|
111
|
+
* under, asks, and ends it; no request is made of it (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a
|
|
112
|
+
* hold). An mbx that does not start it, or does not answer, gates nothing: the run's leaves are sized to their grants.
|
|
113
|
+
*/
|
|
114
|
+
async function learnHeldGate(env: NodeJS.ProcessEnv): Promise<void> {
|
|
115
|
+
const unheld = { ...env };
|
|
116
|
+
delete unheld.MBX_SCHED_DISABLE;
|
|
117
|
+
delete unheld.MBX_SCHED_DIR;
|
|
118
|
+
const started = await startEndpoint(unheld);
|
|
119
|
+
if (!started.ok) {
|
|
120
|
+
logger.warn(
|
|
121
|
+
`bounded-exec: the installed mbx did not say whether its test shim gates cases (${started.error}); this run's nextest leaves are sized to their grants`,
|
|
122
|
+
);
|
|
123
|
+
delete env[TEST_GATE_SERVED_ENV];
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
await started.endpoint.end();
|
|
127
|
+
if (started.gates) env[TEST_GATE_SERVED_ENV] = '1';
|
|
128
|
+
else delete env[TEST_GATE_SERVED_ENV];
|
|
94
129
|
}
|
|
95
130
|
|
|
96
131
|
/** Close the endpoint `context`'s run was served, if it was served one: its standard input's end ends it. */
|
|
@@ -100,86 +135,111 @@ export async function closeRunReservations(context: Pick<PostTasksExecutionConte
|
|
|
100
135
|
return;
|
|
101
136
|
}
|
|
102
137
|
endpoints.delete(context.id);
|
|
103
|
-
endpoint.
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
if (ended !== 'mbx reservations ended with status 0') {
|
|
107
|
-
logger.warn(`bounded-exec: this run's mbx reservation endpoint did not close cleanly: ${ended}`);
|
|
138
|
+
const ended = await endpoint.end();
|
|
139
|
+
if (!ended.clean) {
|
|
140
|
+
logger.warn(`bounded-exec: this run's mbx reservation endpoint did not close cleanly: ${ended.how}`);
|
|
108
141
|
}
|
|
109
142
|
}
|
|
110
143
|
|
|
111
|
-
/**
|
|
144
|
+
/**
|
|
145
|
+
* Start `mbx reservations` by its PATH name in an owned socket directory, and answer once its life answers its start
|
|
146
|
+
* (endpoint-life.ts). This is that machine's shell: it feeds the child's and the probe's events to `endpointStep` and
|
|
147
|
+
* executes the effects each step answers, and decides nothing itself.
|
|
148
|
+
*/
|
|
112
149
|
async function startEndpoint(env: NodeJS.ProcessEnv): Promise<Started> {
|
|
113
150
|
const directory = await ownedSocketDirectory(ENDPOINT_DIRECTORY, ENDPOINT_SOCKET);
|
|
114
151
|
const socket = join(directory, ENDPOINT_SOCKET);
|
|
115
152
|
const server = spawn('mbx', ['reservations', socket], { env, stdio: ['pipe', 'pipe', 'pipe'] });
|
|
116
|
-
|
|
117
|
-
server.
|
|
118
|
-
|
|
119
|
-
|
|
153
|
+
const remove = (): Promise<void> => rm(directory, { recursive: true, force: true });
|
|
154
|
+
let life = spawned(server.pid);
|
|
155
|
+
let probe: Socket | undefined;
|
|
156
|
+
let timer: NodeJS.Timeout | undefined;
|
|
157
|
+
// `Promise.withResolvers` would read better but needs lib es2024; this package inherits lib es2022.
|
|
158
|
+
let answer!: (started: Started) => void;
|
|
159
|
+
let conclude!: (ended: Ended) => void;
|
|
160
|
+
let fail!: (error: unknown) => void;
|
|
161
|
+
let failEnd!: (error: unknown) => void;
|
|
162
|
+
const started = new Promise<Started>((resolve, reject) => {
|
|
163
|
+
answer = resolve;
|
|
164
|
+
fail = reject;
|
|
120
165
|
});
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
server.once('close', (code, signal) => {
|
|
125
|
-
const last = said.trim().split('\n').at(-1) ?? '';
|
|
126
|
-
resolve(
|
|
127
|
-
`mbx reservations ended ${signal === null ? `with status ${code}` : `on ${signal}`}${last === '' ? '' : `: ${last}`}`,
|
|
128
|
-
);
|
|
129
|
-
});
|
|
166
|
+
const ended = new Promise<Ended>((resolve, reject) => {
|
|
167
|
+
conclude = resolve;
|
|
168
|
+
failEnd = reject;
|
|
130
169
|
});
|
|
131
|
-
const endpoint: Endpoint = {
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
});
|
|
163
|
-
});
|
|
164
|
-
}
|
|
170
|
+
const endpoint: Endpoint = {
|
|
171
|
+
end() {
|
|
172
|
+
feed({ type: 'run-end' });
|
|
173
|
+
return ended;
|
|
174
|
+
},
|
|
175
|
+
};
|
|
176
|
+
const execute = (effect: EndpointEffect): void => {
|
|
177
|
+
switch (effect.do) {
|
|
178
|
+
case 'probe': {
|
|
179
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: a Unix-socket connect completes or fails at once, and the probe ends at its answer, its connection's end or the child's end (endpoint-life.ts), which the endpoint's death brings: it is this run's child
|
|
180
|
+
const connection = createConnection(socket);
|
|
181
|
+
probe = connection;
|
|
182
|
+
connection.setEncoding('utf8');
|
|
183
|
+
connection.on('connect', () => connection.write('{"type":"capabilities"}\n'));
|
|
184
|
+
connection.on('data', (text: string) => feed({ type: 'probe', text }));
|
|
185
|
+
connection.on('error', () => feed({ type: 'probe-end' }));
|
|
186
|
+
connection.on('close', () => feed({ type: 'probe-end' }));
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
case 'hang-up':
|
|
190
|
+
probe?.destroy();
|
|
191
|
+
return;
|
|
192
|
+
case 'arm':
|
|
193
|
+
timer = setTimeout(() => feed({ type: 'expired' }), effect.ms);
|
|
194
|
+
return;
|
|
195
|
+
case 'disarm':
|
|
196
|
+
clearTimeout(timer);
|
|
197
|
+
return;
|
|
198
|
+
case 'kill':
|
|
199
|
+
server.kill('SIGKILL');
|
|
200
|
+
return;
|
|
165
201
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
202
|
+
case 'unref-child': {
|
|
203
|
+
server.unref();
|
|
204
|
+
// Node exposes its pipes as sockets; Bun's readable pipes also unref but are not sockets.
|
|
205
|
+
type Pipe = (typeof server.stdin | typeof server.stdout) & { unref?: () => void };
|
|
206
|
+
const pipes: readonly Pipe[] = [server.stdin, server.stdout, server.stderr];
|
|
207
|
+
for (const pipe of pipes) pipe.unref?.();
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
case 'close-stdin':
|
|
211
|
+
server.stdin.end();
|
|
212
|
+
return;
|
|
213
|
+
case 'serve':
|
|
214
|
+
answer({ ok: true, endpoint, socket, gates: effect.gates });
|
|
215
|
+
return;
|
|
216
|
+
case 'refuse':
|
|
217
|
+
void remove().then(() => answer({ ok: false, error: effect.why }), fail);
|
|
218
|
+
return;
|
|
219
|
+
case 'conclude':
|
|
220
|
+
void remove().then(() => conclude(effect.ended), failEnd);
|
|
221
|
+
return;
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
const feed = (event: EndpointEvent): void => {
|
|
225
|
+
const [next, effects] = endpointStep(life, event);
|
|
226
|
+
life = next;
|
|
227
|
+
for (const effect of effects) execute(effect);
|
|
228
|
+
};
|
|
229
|
+
server.stdout.setEncoding('utf8');
|
|
230
|
+
server.stdout.on('data', (text: string) => feed({ type: 'stdout', text }));
|
|
231
|
+
server.stdout.on('end', () => feed({ type: 'stdout-end' }));
|
|
232
|
+
server.stderr.setEncoding('utf8');
|
|
233
|
+
server.stderr.on('data', (text: string) => feed({ type: 'stderr', text }));
|
|
234
|
+
server.once('error', (error) => feed({ type: 'error', message: error.message }));
|
|
235
|
+
server.once('close', (code, signal) => feed({ type: 'close', code, signal }));
|
|
236
|
+
return started;
|
|
177
237
|
}
|
|
178
238
|
|
|
179
239
|
/** The pool's answer to one reservation: the cores it holds and a promise of why it was lost, or why there are none. */
|
|
180
240
|
export type Granted =
|
|
181
241
|
| { readonly ok: true; readonly cpus: number; readonly lost: Promise<string> }
|
|
182
|
-
| { readonly ok: false; readonly
|
|
242
|
+
| { readonly ok: false; readonly refusal: ReservationRefusal };
|
|
183
243
|
|
|
184
244
|
/** One request for cores on a run's endpoint. */
|
|
185
245
|
export interface Reservation {
|
|
@@ -189,144 +249,115 @@ export interface Reservation {
|
|
|
189
249
|
release(): void;
|
|
190
250
|
}
|
|
191
251
|
|
|
192
|
-
type Uint = number & tags.Type<'uint32'>;
|
|
193
|
-
|
|
194
|
-
/** The lines a client writes (mbx's `docs/scheduling.md`, "Serve reservations to many commands"). */
|
|
195
|
-
type ReservationRequest =
|
|
196
|
-
| { type: 'reserve'; id: Uint; cpus: Uint & tags.Minimum<1>; priority: MbxPriority }
|
|
197
|
-
| { type: 'release'; id: Uint };
|
|
198
|
-
|
|
199
|
-
/** The lines the endpoint answers; an added field is mbx's to add. */
|
|
200
|
-
type ReservationReply =
|
|
201
|
-
| { type: 'granted'; id: Uint; cpus: Uint & tags.Minimum<1> }
|
|
202
|
-
| { type: 'refused'; id: Uint; error: string }
|
|
203
|
-
| { type: 'error'; error: string };
|
|
204
|
-
|
|
205
|
-
const stringifyRequest = typia.json.createStringify<ReservationRequest>();
|
|
206
|
-
const parseReply = typia.json.createValidateParse<ReservationReply>();
|
|
207
|
-
|
|
208
|
-
/** A request's place on its connection: waiting for its answer, or held until released or lost. */
|
|
209
|
-
type Entry =
|
|
210
|
-
| { readonly kind: 'waiting'; readonly answer: (granted: Granted) => void }
|
|
211
|
-
| { readonly kind: 'held'; readonly lose: (why: string) => void };
|
|
212
|
-
|
|
213
252
|
/**
|
|
214
|
-
*
|
|
215
|
-
*
|
|
253
|
+
* The socket of one connection to an endpoint, shared by every reservation this process makes there. It is the shell
|
|
254
|
+
* of that connection's machine (reservation-connection.ts): it feeds its callers' requests and its socket's events to
|
|
255
|
+
* `connectionStep` and executes the effects each step answers, keeping each request's answer and each grant's loss to
|
|
256
|
+
* settle; it decides nothing itself.
|
|
216
257
|
*/
|
|
217
|
-
class
|
|
258
|
+
class EndpointSocket {
|
|
218
259
|
readonly #socket: Socket;
|
|
219
|
-
readonly #
|
|
260
|
+
readonly #answers = new Map<number, (granted: Granted) => void>();
|
|
261
|
+
readonly #losses = new Map<number, (why: string) => void>();
|
|
262
|
+
#connection: Connection;
|
|
220
263
|
#nextId = 1;
|
|
221
|
-
#unread = '';
|
|
222
|
-
#closed: string | null = null;
|
|
223
264
|
|
|
224
265
|
constructor(
|
|
225
|
-
|
|
266
|
+
endpoint: string,
|
|
226
267
|
private readonly forget: () => void,
|
|
227
268
|
) {
|
|
269
|
+
this.#connection = opened(endpoint);
|
|
228
270
|
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: a Unix-socket connect completes or fails at once, and a reservation's answer waits as long as the pool's admission holds it, as `mbx reserve`'s did; the connection's end answers every request still waiting
|
|
229
271
|
this.#socket = createConnection(endpoint);
|
|
230
272
|
this.#socket.setEncoding('utf8');
|
|
231
273
|
this.#socket.unref();
|
|
232
|
-
this.#socket.on('data', (text: string) => this.#
|
|
233
|
-
this.#socket.on('error', (error) =>
|
|
234
|
-
|
|
274
|
+
this.#socket.on('data', (text: string) => this.#feed({ type: 'data', text }));
|
|
275
|
+
this.#socket.on('error', (error) =>
|
|
276
|
+
this.#feed({
|
|
277
|
+
type: 'error',
|
|
278
|
+
message: error.message,
|
|
279
|
+
code: 'code' in error && typeof error.code === 'string' ? error.code : null,
|
|
280
|
+
}),
|
|
281
|
+
);
|
|
282
|
+
this.#socket.on('close', () => this.#feed({ type: 'close' }));
|
|
235
283
|
}
|
|
236
284
|
|
|
237
|
-
reserve(cpus: number, priority: MbxPriority): Reservation {
|
|
238
|
-
if (this.#closed !== null) {
|
|
239
|
-
return { granted: Promise.resolve({ ok: false, error: this.#closed }), release() {} };
|
|
240
|
-
}
|
|
285
|
+
reserve(cpus: number, priority: MbxPriority, min: number, name?: string): Reservation {
|
|
241
286
|
const id = this.#nextId++;
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
this.#entries.set(id, { kind: 'waiting', answer });
|
|
245
|
-
this.#socket.ref();
|
|
246
|
-
this.#socket.write(`${stringifyRequest({ type: 'reserve', id, cpus, priority })}\n`);
|
|
247
|
-
let released = false;
|
|
287
|
+
const granted = new Promise<Granted>((resolve) => this.#answers.set(id, resolve));
|
|
288
|
+
this.#feed({ type: 'reserve', id, cpus, priority, min, name });
|
|
248
289
|
return {
|
|
249
290
|
granted,
|
|
250
291
|
release: () => {
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
if (entry === undefined) return;
|
|
255
|
-
this.#entries.delete(id);
|
|
256
|
-
if (entry.kind === 'waiting') entry.answer({ ok: false, error: 'released while it waited' });
|
|
257
|
-
this.#socket.write(`${stringifyRequest({ type: 'release', id })}\n`);
|
|
258
|
-
if (this.#entries.size === 0) this.#socket.unref();
|
|
292
|
+
// A grant given back is no longer one to lose.
|
|
293
|
+
this.#losses.delete(id);
|
|
294
|
+
this.#feed({ type: 'release', id });
|
|
259
295
|
},
|
|
260
296
|
};
|
|
261
297
|
}
|
|
262
298
|
|
|
263
|
-
#
|
|
264
|
-
this.#
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
this.#unread = this.#unread.slice(end + 1);
|
|
268
|
-
const parsed = parseReply(line);
|
|
269
|
-
if (!parsed.success) {
|
|
270
|
-
this.#close(`mbx reservation endpoint ${this.endpoint} answered ${JSON.stringify(line)}, which is no reply`);
|
|
271
|
-
return;
|
|
272
|
-
}
|
|
273
|
-
this.#answer(parsed.data);
|
|
274
|
-
if (this.#closed !== null) return;
|
|
275
|
-
}
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
#answer(reply: ReservationReply): void {
|
|
279
|
-
if (reply.type === 'error') {
|
|
280
|
-
this.#close(`mbx reservation endpoint ${this.endpoint} closed this connection: ${reply.error}`);
|
|
281
|
-
return;
|
|
282
|
-
}
|
|
283
|
-
const entry = this.#entries.get(reply.id);
|
|
284
|
-
// Released before its answer arrived: the endpoint drops what it won.
|
|
285
|
-
if (entry === undefined) return;
|
|
286
|
-
if (entry.kind === 'held') {
|
|
287
|
-
this.#close(`mbx reservation endpoint ${this.endpoint} answered reservation ${reply.id} twice`);
|
|
288
|
-
return;
|
|
289
|
-
}
|
|
290
|
-
if (reply.type === 'granted') {
|
|
291
|
-
let lose!: (why: string) => void;
|
|
292
|
-
const lost = new Promise<string>((resolve) => (lose = resolve));
|
|
293
|
-
this.#entries.set(reply.id, { kind: 'held', lose });
|
|
294
|
-
entry.answer({ ok: true, cpus: reply.cpus, lost });
|
|
295
|
-
return;
|
|
296
|
-
}
|
|
297
|
-
this.#entries.delete(reply.id);
|
|
298
|
-
entry.answer({ ok: false, error: reply.error });
|
|
299
|
-
if (this.#entries.size === 0) this.#socket.unref();
|
|
299
|
+
#feed(event: ConnectionEvent): void {
|
|
300
|
+
const [next, effects] = connectionStep(this.#connection, event);
|
|
301
|
+
this.#connection = next;
|
|
302
|
+
for (const effect of effects) this.#execute(effect);
|
|
300
303
|
}
|
|
301
304
|
|
|
302
|
-
#
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
305
|
+
#execute(effect: ConnectionEffect): void {
|
|
306
|
+
switch (effect.do) {
|
|
307
|
+
case 'ref':
|
|
308
|
+
this.#socket.ref();
|
|
309
|
+
return;
|
|
310
|
+
case 'unref':
|
|
311
|
+
this.#socket.unref();
|
|
312
|
+
return;
|
|
313
|
+
case 'write':
|
|
314
|
+
this.#socket.write(effect.line);
|
|
315
|
+
return;
|
|
316
|
+
case 'answer': {
|
|
317
|
+
const settle = this.#answers.get(effect.id);
|
|
318
|
+
this.#answers.delete(effect.id);
|
|
319
|
+
if (!effect.answer.ok) {
|
|
320
|
+
settle?.(effect.answer);
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
const lost = new Promise<string>((resolve) => this.#losses.set(effect.id, resolve));
|
|
324
|
+
settle?.({ ok: true, cpus: effect.answer.cpus, lost });
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
case 'lose':
|
|
328
|
+
this.#losses.get(effect.id)?.(effect.why);
|
|
329
|
+
this.#losses.delete(effect.id);
|
|
330
|
+
return;
|
|
331
|
+
case 'hang-up':
|
|
332
|
+
this.forget();
|
|
333
|
+
this.#socket.destroy();
|
|
334
|
+
return;
|
|
309
335
|
}
|
|
310
|
-
this.#entries.clear();
|
|
311
|
-
this.#socket.destroy();
|
|
312
336
|
}
|
|
313
337
|
}
|
|
314
338
|
|
|
315
339
|
/** This process's connection to each endpoint it has asked. */
|
|
316
|
-
const connections = new Map<string,
|
|
340
|
+
const connections = new Map<string, EndpointSocket>();
|
|
317
341
|
|
|
318
342
|
/**
|
|
319
|
-
* Ask the endpoint at `socket` for at most `cpus` cores at `priority`,
|
|
320
|
-
* opened by its first request and kept while another one
|
|
343
|
+
* Ask the endpoint at `socket` for at most `cpus` cores at `priority`, starting on no fewer than `min` of them until
|
|
344
|
+
* mbx's aging bound, over this process's one connection to it, opened by its first request and kept while another one
|
|
345
|
+
* may come. `name` is the class mbx predicts the reservation by; absent, mbx's own for an unnamed one.
|
|
321
346
|
*/
|
|
322
|
-
export function reserveCores(
|
|
347
|
+
export function reserveCores(
|
|
348
|
+
socket: string,
|
|
349
|
+
cpus: number,
|
|
350
|
+
priority: MbxPriority,
|
|
351
|
+
min: number,
|
|
352
|
+
name?: string,
|
|
353
|
+
): Reservation {
|
|
323
354
|
let connection = connections.get(socket);
|
|
324
355
|
if (connection === undefined) {
|
|
325
|
-
const
|
|
326
|
-
if (connections.get(socket) ===
|
|
356
|
+
const fresh: EndpointSocket = new EndpointSocket(socket, () => {
|
|
357
|
+
if (connections.get(socket) === fresh) connections.delete(socket);
|
|
327
358
|
});
|
|
328
|
-
connections.set(socket,
|
|
329
|
-
connection =
|
|
359
|
+
connections.set(socket, fresh);
|
|
360
|
+
connection = fresh;
|
|
330
361
|
}
|
|
331
|
-
return connection.reserve(cpus, priority);
|
|
362
|
+
return connection.reserve(cpus, priority, min, name);
|
|
332
363
|
}
|
|
@@ -29,6 +29,8 @@ export interface Ask {
|
|
|
29
29
|
readonly parallelism: number | null;
|
|
30
30
|
readonly alone: boolean;
|
|
31
31
|
readonly task: string | undefined;
|
|
32
|
+
/** Its cases each ask the pool for a token of their own (case-gate.ts). */
|
|
33
|
+
readonly gated: boolean;
|
|
32
34
|
}
|
|
33
35
|
|
|
34
36
|
/**
|
|
@@ -43,6 +45,7 @@ export async function askParentPool(inherited: InheritedCpuPool | null, ask: Ask
|
|
|
43
45
|
parallelism: ask.parallelism ?? undefined,
|
|
44
46
|
alone: ask.alone ? true : undefined,
|
|
45
47
|
task: ask.task,
|
|
48
|
+
gated: ask.gated ? true : undefined,
|
|
46
49
|
});
|
|
47
50
|
if (answer.granted) {
|
|
48
51
|
return { grant: answer, refusal: null, wait: undefined, said: [] };
|
|
@@ -28,6 +28,10 @@ export interface PoolAsk {
|
|
|
28
28
|
readonly parallelism?: number | undefined;
|
|
29
29
|
/** A leaf the kernel killed for memory, run again: granted while nothing else holds a grant, and alone. */
|
|
30
30
|
readonly alone?: boolean | undefined;
|
|
31
|
+
/** One token for one test case of a leaf that holds a grant: granted only while no leaf's request waits. */
|
|
32
|
+
readonly lend?: boolean | undefined;
|
|
33
|
+
/** A lent request's leaf's predicted work left, in ms: its cases' last recorded walls, less those that ran. */
|
|
34
|
+
readonly left?: number | undefined;
|
|
31
35
|
}
|
|
32
36
|
|
|
33
37
|
/** Who holds a grant when the pool decides: how many leaves, and whether one of them runs alone. */
|
|
@@ -56,11 +60,12 @@ export interface PoolGrant<Ask extends PoolAsk> {
|
|
|
56
60
|
|
|
57
61
|
/**
|
|
58
62
|
* The grants of the queue's first requests, in its order, and what the first request left waiting waits for: free
|
|
59
|
-
* tokens (`cpu`), the run's memory (`memory`)
|
|
63
|
+
* tokens (`cpu`), the run's memory (`memory`), the grants others hold (`alone`) or, a lent request, the settle time
|
|
64
|
+
* after a leaf's grant returned (`settle`). `waiting` is null when every request was granted.
|
|
60
65
|
*/
|
|
61
66
|
export interface Admission<Ask extends PoolAsk> {
|
|
62
67
|
readonly grants: readonly PoolGrant<Ask>[];
|
|
63
|
-
readonly waiting: 'cpu' | 'memory' | 'alone' | null;
|
|
68
|
+
readonly waiting: 'cpu' | 'memory' | 'alone' | 'settle' | null;
|
|
64
69
|
}
|
|
65
70
|
|
|
66
71
|
/** The bytes `ask` is charged when it is lent `tokens`: its peak, scaled up when that is more than it was measured under. */
|
|
@@ -86,6 +91,7 @@ export function admit<Ask extends PoolAsk>(
|
|
|
86
91
|
available: number,
|
|
87
92
|
memory: MemoryLedger | null,
|
|
88
93
|
holding: Holding = NOTHING_HELD,
|
|
94
|
+
lendable = true,
|
|
89
95
|
): Admission<Ask> {
|
|
90
96
|
const grants: PoolGrant<Ask>[] = [];
|
|
91
97
|
if (holding.alone) {
|
|
@@ -97,6 +103,9 @@ export function admit<Ask extends PoolAsk>(
|
|
|
97
103
|
if (request.alone === true && holding.holders + grants.length > 0) {
|
|
98
104
|
return { grants, waiting: 'alone' };
|
|
99
105
|
}
|
|
106
|
+
if (request.lend === true && !lendable) {
|
|
107
|
+
return { grants, waiting: 'settle' };
|
|
108
|
+
}
|
|
100
109
|
const oversized = memory !== null && request.memory > memory.capacity;
|
|
101
110
|
const room = memory === null ? Number.POSITIVE_INFINITY : memory.capacity - held;
|
|
102
111
|
if (request.memory > room && !(oversized && held === 0)) {
|
|
@@ -117,3 +126,51 @@ export function admit<Ask extends PoolAsk>(
|
|
|
117
126
|
}
|
|
118
127
|
return { grants, waiting: null };
|
|
119
128
|
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Where a request joins the queue (specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets itself"). A leaf the
|
|
132
|
+
* kernel killed, run again alone, has waited once already and goes ahead of the leaves that have not run. A leaf goes
|
|
133
|
+
* behind the leaves and ahead of every lent request: a case is lent only a token no leaf waits for. A lent request goes
|
|
134
|
+
* among the lent ones behind those whose leaves have as much work left or more, so a free token goes to the leaf that
|
|
135
|
+
* would otherwise end last, in arrival order among equals.
|
|
136
|
+
*/
|
|
137
|
+
export function place(queue: readonly PoolAsk[], ask: PoolAsk): number {
|
|
138
|
+
const before = (found: number): number => (found === -1 ? queue.length : found);
|
|
139
|
+
if (ask.alone === true) {
|
|
140
|
+
return before(queue.findIndex((waiting) => waiting.alone !== true));
|
|
141
|
+
}
|
|
142
|
+
if (ask.lend !== true) {
|
|
143
|
+
return before(queue.findIndex((waiting) => waiting.lend === true));
|
|
144
|
+
}
|
|
145
|
+
const left = ask.left ?? 0;
|
|
146
|
+
return before(queue.findIndex((waiting) => waiting.lend === true && (waiting.left ?? 0) < left));
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* The token-time a pool's tokens stood idle beside a starved hold: a leaf holding fewer tokens than it asked that
|
|
151
|
+
* cannot ask for more, or a case waiting for a lent token. The pool tells it the idle count and whether a hold is
|
|
152
|
+
* starved at each change, on its own clock; what is counted is the idle count over the time a starved hold stood.
|
|
153
|
+
*/
|
|
154
|
+
export class IdleBesideStarved {
|
|
155
|
+
#at: number;
|
|
156
|
+
#idle = 0;
|
|
157
|
+
#starved = false;
|
|
158
|
+
#tokenMs = 0;
|
|
159
|
+
|
|
160
|
+
constructor(now: number) {
|
|
161
|
+
this.#at = now;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** From `now` on, `idle` tokens are free and a hold is `starved` or none is. */
|
|
165
|
+
observe(now: number, idle: number, starved: boolean): void {
|
|
166
|
+
this.#tokenMs = this.tokenMs(now);
|
|
167
|
+
this.#at = now;
|
|
168
|
+
this.#idle = idle;
|
|
169
|
+
this.#starved = starved;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** The token-milliseconds counted up to `now`. */
|
|
173
|
+
tokenMs(now: number): number {
|
|
174
|
+
return this.#tokenMs + (this.#starved ? this.#idle * (now - this.#at) : 0);
|
|
175
|
+
}
|
|
176
|
+
}
|