@smoothbricks/nx-plugin 0.7.17 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +315 -114
- package/dist/.build-inputs.json +47 -38
- package/dist/built-output-packages.js +1 -1
- package/dist/bun-lock-scope.d.ts +128 -20
- package/dist/bun-lock-scope.d.ts.map +1 -1
- package/dist/bun-lock-scope.js +267 -44
- package/dist/cargo-closure-input.d.ts +8 -6
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +15 -12
- package/dist/cargo-source-hash.d.ts +26 -25
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +84 -105
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts +20 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +43 -22
- package/dist/ensure-built-receipt.d.ts +3 -2
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +3 -2
- package/dist/eslint-config-closure.d.ts +8 -3
- package/dist/eslint-config-closure.d.ts.map +1 -1
- package/dist/eslint-config-closure.js +24 -5
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +19 -6
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +22 -15
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/cpu-rise.d.ts +73 -0
- package/dist/executors/bounded-exec/cpu-rise.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-rise.js +111 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts +1 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +256 -86
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +11 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +125 -47
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +42 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +45 -8
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +12 -6
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +190 -284
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress-channel.d.ts +38 -0
- package/dist/executors/bounded-exec/progress-channel.d.ts.map +1 -0
- package/dist/executors/bounded-exec/progress-channel.js +113 -0
- package/dist/executors/bounded-exec/progress.d.ts +5 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts +24 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +123 -16
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +7 -2
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/schema.js +17 -1
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +39 -10
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +60 -14
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +294 -152
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/managed-files/generator.d.ts +5 -1
- package/dist/managed-files/generator.d.ts.map +1 -1
- package/dist/managed-files/generator.js +22 -7
- package/dist/managed-files/render-memo.d.ts +13 -0
- package/dist/managed-files/render-memo.d.ts.map +1 -0
- package/dist/managed-files/render-memo.js +94 -0
- package/dist/task-results.js +2 -1
- package/dist/test-program.d.ts +11 -6
- package/dist/test-program.d.ts.map +1 -1
- package/dist/test-program.js +39 -35
- package/dist/tool-bins.d.ts +12 -0
- package/dist/tool-bins.d.ts.map +1 -1
- package/dist/tool-bins.js +31 -6
- package/dist/typecheck-test-policy.d.ts +10 -1
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +40 -24
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/dist/workspace-index.d.ts +14 -1
- package/dist/workspace-index.d.ts.map +1 -1
- package/dist/workspace-index.js +36 -4
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1851 -221
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +122 -14
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/managed/templates/github/actions/save-nix-devenv/action.yml +19 -1
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +68 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/bun-lock-scope.ts +293 -40
- package/src/cargo-closure-input.ts +20 -12
- package/src/cargo-source-hash.ts +119 -145
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +59 -21
- package/src/ensure-built-receipt.ts +3 -2
- package/src/eslint-config-closure.ts +27 -6
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +41 -19
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/cpu-rise.ts +156 -0
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +341 -123
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +150 -50
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +57 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +216 -183
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress-channel.ts +129 -0
- package/src/executors/bounded-exec/progress.ts +39 -3
- package/src/executors/bounded-exec/ram-temp.ts +137 -17
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +21 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +83 -22
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +357 -166
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/managed-files/generator.ts +21 -6
- package/src/managed-files/render-memo.ts +135 -0
- package/src/test-program.ts +36 -41
- package/src/tool-bins.ts +35 -6
- package/src/typecheck-test-policy.ts +49 -24
- package/src/workspace-config-policy.ts +161 -43
- package/src/workspace-index.ts +40 -5
- package/test-timings.json +157 -141
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
import * as _validateReport_1 from "typia/lib/internal/_validateReport";
|
|
2
|
+
import { rm } from 'node:fs/promises';
|
|
3
|
+
import { createServer } from 'node:net';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import typia from 'typia';
|
|
6
|
+
import { ownedSocketDirectory } from '../../testing.js';
|
|
7
|
+
/**
|
|
8
|
+
* A reserved nextest leaf's case gate (specs/cowshed/05_gateway.md, "A case starts on a core"). The leaf's runner
|
|
9
|
+
* starts every thread of its ask, and each case takes a core before it runs: one of the leaf's grant when one is free,
|
|
10
|
+
* else one reserved for that case alone, which the leaf's executor asks its run's endpoint for and gives back when the
|
|
11
|
+
* case ends. A grant never grows, so this is how the cores other work frees after the leaf's grant reach it.
|
|
12
|
+
*
|
|
13
|
+
* The gate is a Unix stream socket the executor serves for one run and names in the command's environment as
|
|
14
|
+
* {@link TEST_GATE_ENV}. mbx's test shim, through which nextest starts every case, connects before the case starts,
|
|
15
|
+
* writes one line ({@link CaseLine}), starts the case when the line {@link GO} answers, and keeps the connection until
|
|
16
|
+
* the case has exited: the connection's end is the case's end.
|
|
17
|
+
*/
|
|
18
|
+
export const TEST_GATE_ENV = 'MBX_TEST_GATE';
|
|
19
|
+
/**
|
|
20
|
+
* Set to `1` in a run's environment once its endpoint answered that the installed mbx's shim gates its cases
|
|
21
|
+
* (endpoint-life.ts, `gatesTests`): without it a leaf's runner is sized to its grant.
|
|
22
|
+
*/
|
|
23
|
+
export const TEST_GATE_SERVED_ENV = 'BOUNDED_EXEC_MBX_TEST_GATE';
|
|
24
|
+
/** The line that starts a case. */
|
|
25
|
+
export const GO = 'go';
|
|
26
|
+
const parseCaseLine = (() => {
|
|
27
|
+
let errors;
|
|
28
|
+
let _report;
|
|
29
|
+
const _io0 = input => "string" === typeof input.binary && "string" === typeof input.test;
|
|
30
|
+
const _vo0 = (input, _path, _exceptionable = true) => ["string" === typeof input.binary || _report(_exceptionable, {
|
|
31
|
+
path: _path + ".binary",
|
|
32
|
+
expected: "string",
|
|
33
|
+
value: input.binary
|
|
34
|
+
}), "string" === typeof input.test || _report(_exceptionable, {
|
|
35
|
+
path: _path + ".test",
|
|
36
|
+
expected: "string",
|
|
37
|
+
value: input.test
|
|
38
|
+
})].every(flag => flag);
|
|
39
|
+
const __is = input => "object" === typeof input && null !== input && _io0(input);
|
|
40
|
+
const __validate = input => {
|
|
41
|
+
if (false === __is(input)) {
|
|
42
|
+
errors = [];
|
|
43
|
+
_report = _validateReport_1._validateReport(errors);
|
|
44
|
+
((input, _path, _exceptionable = true) => ("object" === typeof input && null !== input || _report(true, {
|
|
45
|
+
path: _path + "",
|
|
46
|
+
expected: "CaseLine",
|
|
47
|
+
value: input
|
|
48
|
+
})) && _vo0(input, _path + "", true) || _report(true, {
|
|
49
|
+
path: _path + "",
|
|
50
|
+
expected: "CaseLine",
|
|
51
|
+
value: input
|
|
52
|
+
}))(input, "$input", true);
|
|
53
|
+
const success = 0 === errors.length;
|
|
54
|
+
return success ? {
|
|
55
|
+
success,
|
|
56
|
+
data: input
|
|
57
|
+
} : {
|
|
58
|
+
success,
|
|
59
|
+
errors,
|
|
60
|
+
data: input
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
return {
|
|
64
|
+
success: true,
|
|
65
|
+
data: input
|
|
66
|
+
};
|
|
67
|
+
};
|
|
68
|
+
return input => __validate(JSON.parse(input));
|
|
69
|
+
})();
|
|
70
|
+
/** A case's id as its result names it (progress.ts): the binary id, `::`, the test. */
|
|
71
|
+
export function caseId(binary, test) {
|
|
72
|
+
return `${binary}::${test}`;
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The gate's decisions, as a state machine over three events: a case arrived, the pool granted a case's request, a
|
|
76
|
+
* case ended. `own` is the leaf's granted cores; `wall` a case's last recorded wall, by which a freed core of the
|
|
77
|
+
* grant picks the longest waiting case, so the cores the pool gives go to the shorter ones and come back soonest.
|
|
78
|
+
*/
|
|
79
|
+
export class CaseGate {
|
|
80
|
+
own;
|
|
81
|
+
wall;
|
|
82
|
+
effects;
|
|
83
|
+
now;
|
|
84
|
+
#cases = new Map();
|
|
85
|
+
#waited = new Map();
|
|
86
|
+
#ownRunning = 0;
|
|
87
|
+
#own = 0;
|
|
88
|
+
#lent = 0;
|
|
89
|
+
#lentCoreMs = 0;
|
|
90
|
+
#waitedCaseMs = 0;
|
|
91
|
+
#ranWallMs = 0;
|
|
92
|
+
constructor(own, wall, effects, now = Date.now) {
|
|
93
|
+
this.own = own;
|
|
94
|
+
this.wall = wall;
|
|
95
|
+
this.effects = effects;
|
|
96
|
+
this.now = now;
|
|
97
|
+
}
|
|
98
|
+
/** A case's shim connected. False for a case already at the gate, which its second connection may not join. */
|
|
99
|
+
arrived(id) {
|
|
100
|
+
if (this.#cases.has(id))
|
|
101
|
+
return false;
|
|
102
|
+
if (this.#ownRunning < this.own) {
|
|
103
|
+
this.#startOwn(id, 0);
|
|
104
|
+
return true;
|
|
105
|
+
}
|
|
106
|
+
this.#cases.set(id, { kind: 'waiting', since: this.now() });
|
|
107
|
+
this.effects.reserve(id);
|
|
108
|
+
return true;
|
|
109
|
+
}
|
|
110
|
+
/** The pool granted the case's request. One whose case no longer waits gives the core straight back. */
|
|
111
|
+
granted(id) {
|
|
112
|
+
const state = this.#cases.get(id);
|
|
113
|
+
if (state?.kind !== 'waiting') {
|
|
114
|
+
this.effects.release(id);
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
const at = this.now();
|
|
118
|
+
this.#startedAfter(id, at - state.since);
|
|
119
|
+
this.#cases.set(id, { kind: 'lent', since: at });
|
|
120
|
+
this.#lent += 1;
|
|
121
|
+
this.effects.start(id);
|
|
122
|
+
}
|
|
123
|
+
/** A case's connection ended: its core goes back where it came from, a waiting case's request is withdrawn. */
|
|
124
|
+
ended(id) {
|
|
125
|
+
const state = this.#cases.get(id);
|
|
126
|
+
if (state === undefined)
|
|
127
|
+
return;
|
|
128
|
+
this.#cases.delete(id);
|
|
129
|
+
if (state.kind !== 'waiting')
|
|
130
|
+
this.#ranWallMs += this.wall(id);
|
|
131
|
+
switch (state.kind) {
|
|
132
|
+
case 'waiting':
|
|
133
|
+
this.effects.release(id);
|
|
134
|
+
return;
|
|
135
|
+
case 'lent':
|
|
136
|
+
this.#lentCoreMs += this.now() - state.since;
|
|
137
|
+
this.effects.release(id);
|
|
138
|
+
return;
|
|
139
|
+
case 'own': {
|
|
140
|
+
this.#ownRunning -= 1;
|
|
141
|
+
let longest = null;
|
|
142
|
+
for (const [waiting, other] of this.#cases) {
|
|
143
|
+
if (other.kind !== 'waiting')
|
|
144
|
+
continue;
|
|
145
|
+
if (longest === null || this.wall(waiting) > this.wall(longest.id)) {
|
|
146
|
+
longest = { id: waiting, since: other.since };
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (longest === null)
|
|
150
|
+
return;
|
|
151
|
+
this.effects.release(longest.id);
|
|
152
|
+
this.#startOwn(longest.id, this.now() - longest.since);
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
/** How long the case waited at the gate before it started, in ms; zero for one that never did. */
|
|
158
|
+
waitedMs(id) {
|
|
159
|
+
return this.#waited.get(id) ?? 0;
|
|
160
|
+
}
|
|
161
|
+
/** The last recorded walls of the cases that ran and ended, summed. */
|
|
162
|
+
ranWallMs() {
|
|
163
|
+
return this.#ranWallMs;
|
|
164
|
+
}
|
|
165
|
+
summary() {
|
|
166
|
+
return { own: this.#own, lent: this.#lent, lentCoreMs: this.#lentCoreMs, waitedCaseMs: this.#waitedCaseMs };
|
|
167
|
+
}
|
|
168
|
+
#startOwn(id, waitedMs) {
|
|
169
|
+
this.#startedAfter(id, waitedMs);
|
|
170
|
+
this.#cases.set(id, { kind: 'own' });
|
|
171
|
+
this.#ownRunning += 1;
|
|
172
|
+
this.#own += 1;
|
|
173
|
+
this.effects.start(id);
|
|
174
|
+
}
|
|
175
|
+
#startedAfter(id, waitedMs) {
|
|
176
|
+
this.#waited.set(id, waitedMs);
|
|
177
|
+
this.#waitedCaseMs += waitedMs;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* The gate of a leaf that holds `own` cores, over whichever pool `ask` asks: mbx's, through the run's reservation
|
|
182
|
+
* endpoint, or the run's own token pool (cpu-pool.ts, `lendParentToken`). `start` answers a case's connection.
|
|
183
|
+
* `walls` are the cases' last recorded walls; a request is told the work the leaf has left, their sum less the walls
|
|
184
|
+
* of the cases that ran, by which a token pool lends to the leaf that would otherwise end last.
|
|
185
|
+
*/
|
|
186
|
+
export function gateCases(own, walls, start, ask) {
|
|
187
|
+
const requests = new Map();
|
|
188
|
+
let total = 0;
|
|
189
|
+
for (const wall of walls.values())
|
|
190
|
+
total += wall;
|
|
191
|
+
const cases = new CaseGate(own, (id) => walls.get(id) ?? 0, {
|
|
192
|
+
start,
|
|
193
|
+
reserve: (id) => {
|
|
194
|
+
const held = ask(id, Math.max(0, total - cases.ranWallMs()));
|
|
195
|
+
requests.set(id, held);
|
|
196
|
+
void held.granted.then((granted) => {
|
|
197
|
+
// Released since: the case started on a core of the grant, or ended while it waited.
|
|
198
|
+
if (requests.get(id) !== held)
|
|
199
|
+
return;
|
|
200
|
+
if (granted)
|
|
201
|
+
cases.granted(id);
|
|
202
|
+
else
|
|
203
|
+
requests.delete(id);
|
|
204
|
+
});
|
|
205
|
+
},
|
|
206
|
+
release: (id) => {
|
|
207
|
+
requests.get(id)?.release();
|
|
208
|
+
requests.delete(id);
|
|
209
|
+
},
|
|
210
|
+
});
|
|
211
|
+
return {
|
|
212
|
+
cases,
|
|
213
|
+
releaseAll: () => {
|
|
214
|
+
for (const held of requests.values())
|
|
215
|
+
held.release();
|
|
216
|
+
requests.clear();
|
|
217
|
+
},
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* A case's wall as its result records it: nextest times a case from the shim's start, so its own duration includes the
|
|
222
|
+
* wait at the gate, which is taken off. Never below zero.
|
|
223
|
+
*/
|
|
224
|
+
export function runWallMs(nextestMs, waitedMs) {
|
|
225
|
+
return Math.max(0, nextestMs - waitedMs);
|
|
226
|
+
}
|
|
227
|
+
const GATE_DIRECTORY = 'bounded-gate-';
|
|
228
|
+
const GATE_SOCKET = 'g.sock';
|
|
229
|
+
/** No case line is longer: a binary id and a test name. */
|
|
230
|
+
const LINE_BYTES = 16 * 1024;
|
|
231
|
+
function describe(error) {
|
|
232
|
+
return error instanceof Error ? error.message : String(error);
|
|
233
|
+
}
|
|
234
|
+
/** Serve one run's case gate; a gate that cannot be served is the run's refusal. */
|
|
235
|
+
export async function serveCaseGate(handlers) {
|
|
236
|
+
let directory;
|
|
237
|
+
try {
|
|
238
|
+
directory = await ownedSocketDirectory(GATE_DIRECTORY, GATE_SOCKET);
|
|
239
|
+
}
|
|
240
|
+
catch (error) {
|
|
241
|
+
return { ok: false, error: `its directory could not be made: ${describe(error)}` };
|
|
242
|
+
}
|
|
243
|
+
const socketPath = join(directory, GATE_SOCKET);
|
|
244
|
+
const clients = new Set();
|
|
245
|
+
const named = new Map();
|
|
246
|
+
const serve = (client) => {
|
|
247
|
+
clients.add(client);
|
|
248
|
+
// A case that is killed loses its connection with its process: that end is the case's end, read below.
|
|
249
|
+
client.on('error', () => { });
|
|
250
|
+
client.setEncoding('utf8');
|
|
251
|
+
let partial = '';
|
|
252
|
+
let id = null;
|
|
253
|
+
const refuse = (detail) => {
|
|
254
|
+
client.destroy();
|
|
255
|
+
handlers.invalid(detail);
|
|
256
|
+
};
|
|
257
|
+
client.on('data', (text) => {
|
|
258
|
+
if (id !== null) {
|
|
259
|
+
refuse(`case ${id} wrote a second line at its gate`);
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
partial += text;
|
|
263
|
+
const end = partial.indexOf('\n');
|
|
264
|
+
if (end === -1) {
|
|
265
|
+
if (partial.length > LINE_BYTES)
|
|
266
|
+
refuse(`a case line is at most ${LINE_BYTES} bytes`);
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
let parsed;
|
|
270
|
+
try {
|
|
271
|
+
parsed = parseCaseLine(partial.slice(0, end));
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
parsed = null;
|
|
275
|
+
}
|
|
276
|
+
if (parsed === null || !parsed.success || partial.length > end + 1) {
|
|
277
|
+
refuse(`a case line names a binary and a test, one line; got ${JSON.stringify(partial.slice(0, 200))}`);
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
const arriving = caseId(parsed.data.binary, parsed.data.test);
|
|
281
|
+
named.set(arriving, client);
|
|
282
|
+
if (!handlers.arrived(arriving)) {
|
|
283
|
+
named.delete(arriving);
|
|
284
|
+
refuse(`case ${arriving} is already at its gate`);
|
|
285
|
+
return;
|
|
286
|
+
}
|
|
287
|
+
id = arriving;
|
|
288
|
+
});
|
|
289
|
+
client.on('close', () => {
|
|
290
|
+
clients.delete(client);
|
|
291
|
+
if (id === null)
|
|
292
|
+
return;
|
|
293
|
+
if (named.get(id) === client)
|
|
294
|
+
named.delete(id);
|
|
295
|
+
handlers.ended(id);
|
|
296
|
+
});
|
|
297
|
+
};
|
|
298
|
+
const server = createServer(serve);
|
|
299
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: listen is this process's own bind of a path in its fresh directory; the kernel answers it with the listening callback or an error, and no peer takes part
|
|
300
|
+
const listened = await new Promise((settle) => {
|
|
301
|
+
server.once('error', settle);
|
|
302
|
+
server.listen(socketPath, () => {
|
|
303
|
+
server.off('error', settle);
|
|
304
|
+
settle(null);
|
|
305
|
+
});
|
|
306
|
+
});
|
|
307
|
+
const close = async () => {
|
|
308
|
+
const failures = [];
|
|
309
|
+
for (const client of clients)
|
|
310
|
+
client.destroy();
|
|
311
|
+
if (listened === null) {
|
|
312
|
+
// eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: every accepted connection was destroyed just above, so the server closes without waiting on a peer
|
|
313
|
+
const closed = await new Promise((settle) => server.close(settle));
|
|
314
|
+
if (closed !== undefined)
|
|
315
|
+
failures.push(`its socket did not close: ${closed.message}`);
|
|
316
|
+
}
|
|
317
|
+
try {
|
|
318
|
+
await rm(directory, { recursive: true, force: true });
|
|
319
|
+
}
|
|
320
|
+
catch (error) {
|
|
321
|
+
failures.push(`its directory was not removed: ${describe(error)}`);
|
|
322
|
+
}
|
|
323
|
+
return failures.length === 0 ? null : `the case gate at ${socketPath}: ${failures.join('; ')}`;
|
|
324
|
+
};
|
|
325
|
+
if (listened !== null) {
|
|
326
|
+
const left = await close();
|
|
327
|
+
return {
|
|
328
|
+
ok: false,
|
|
329
|
+
error: `its socket ${socketPath} could not be bound: ${listened.message}${left === null ? '' : `; ${left}`}`,
|
|
330
|
+
};
|
|
331
|
+
}
|
|
332
|
+
return {
|
|
333
|
+
ok: true,
|
|
334
|
+
gate: {
|
|
335
|
+
socket: socketPath,
|
|
336
|
+
start: (id) => {
|
|
337
|
+
named.get(id)?.write(`${GO}\n`);
|
|
338
|
+
},
|
|
339
|
+
close,
|
|
340
|
+
},
|
|
341
|
+
};
|
|
342
|
+
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-pool-thread.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool-thread.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,
|
|
1
|
+
{"version":3,"file":"cpu-pool-thread.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool-thread.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,YAAY,EAA+B,MAAM,eAAe,CAAC;AAC/E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAElD;;;;;;;;;;GAUG;AAEH,6GAA6G;AAC7G,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;CAC5B;AAED,8GAA8G;AAC9G,MAAM,MAAM,mBAAmB,GAC3B;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,GAC1E;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { parentPort, workerData } from 'node:worker_threads';
|
|
2
|
-
import { createCpuPool } from './cpu-pool.js';
|
|
2
|
+
import { createCpuPool, describeIdle } from './cpu-pool.js';
|
|
3
3
|
const host = parentPort;
|
|
4
4
|
if (host === null) {
|
|
5
5
|
throw new Error('cpu-pool-thread runs as the worker thread run-cpu-pool starts, never as a module of its own');
|
|
@@ -11,7 +11,7 @@ const made = await createCpuPool(data.tokens, new Promise(() => { }), data.memor
|
|
|
11
11
|
if (made.ok) {
|
|
12
12
|
const { pool } = made;
|
|
13
13
|
host.once('message', () => {
|
|
14
|
-
void pool.close().then((outcome) => report({ kind: 'closed', outcome }));
|
|
14
|
+
void pool.close().then((outcome) => report({ kind: 'closed', outcome, idle: describeIdle(made.summary()) }));
|
|
15
15
|
});
|
|
16
16
|
report({ kind: 'serving', environment: pool.environment });
|
|
17
17
|
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import typia from 'typia';
|
|
2
|
+
import type { CaseCore } from './case-gate.js';
|
|
2
3
|
import { type Unleased } from './gateway-lease.js';
|
|
3
4
|
import { type LiveUsageReading, type RunMemory } from './host-memory.js';
|
|
4
5
|
import { type InheritedCpuPool } from './pool-environment.js';
|
|
@@ -38,6 +39,22 @@ export interface CpuPool {
|
|
|
38
39
|
*/
|
|
39
40
|
close(): Promise<CloseOutcome>;
|
|
40
41
|
}
|
|
42
|
+
/** What a pool says when it closes: the token-time idle beside starved holds, and the grants that made them. */
|
|
43
|
+
export interface PoolSummary {
|
|
44
|
+
/** Token-milliseconds no socket grant held while a hold was starved (pool-admission.ts, `IdleBesideStarved`). */
|
|
45
|
+
readonly idleBesideStarvedMs: number;
|
|
46
|
+
/** Leaves granted fewer tokens than they asked, whether gated or not. */
|
|
47
|
+
readonly starved: number;
|
|
48
|
+
/** Lent tokens granted, one per test case. */
|
|
49
|
+
readonly lent: number;
|
|
50
|
+
}
|
|
51
|
+
/** When a pool lends: its clock, and how long after a leaf's grant returns it grants no lent request. */
|
|
52
|
+
export interface Lending {
|
|
53
|
+
readonly settleMs: number;
|
|
54
|
+
readonly now: () => number;
|
|
55
|
+
}
|
|
56
|
+
/** The line a run's pool says of `summary` when it closes; null for a pool that starved no leaf and lent nothing. */
|
|
57
|
+
export declare function describeIdle(summary: PoolSummary): string | null;
|
|
41
58
|
export type CloseOutcome = {
|
|
42
59
|
ok: true;
|
|
43
60
|
} | {
|
|
@@ -70,6 +87,8 @@ export interface AskContext {
|
|
|
70
87
|
readonly task?: string | undefined;
|
|
71
88
|
/** The kernel killed the command for memory: it is run again, granted only while nothing else holds a grant. */
|
|
72
89
|
readonly alone?: boolean | undefined;
|
|
90
|
+
/** Its cases each ask the pool for a token of their own. */
|
|
91
|
+
readonly gated?: boolean | undefined;
|
|
73
92
|
}
|
|
74
93
|
/**
|
|
75
94
|
* Ask the immediate parent pool for `want` tokens, charged `memory` bytes: the task's last recorded peak, or null when
|
|
@@ -81,6 +100,13 @@ export interface AskContext {
|
|
|
81
100
|
* the child beside its parent's count.
|
|
82
101
|
*/
|
|
83
102
|
export declare function takeParentCpuTokens(parent: InheritedCpuPool, want: number, memory: number | null, context?: AskContext): Promise<TokenGrant>;
|
|
103
|
+
/**
|
|
104
|
+
* One token of `parent` asked for one test case of a leaf that holds a grant of it, named `task` in the request
|
|
105
|
+
* (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold). The pool grants it only while no other
|
|
106
|
+
* request waits. `release` gives the token back, or withdraws the request while it waits. A pool that refuses, falls
|
|
107
|
+
* silent or is gone grants nothing: the case waits for a token of its leaf's own grant.
|
|
108
|
+
*/
|
|
109
|
+
export declare function lendParentToken(parent: InheritedCpuPool, task: string, left: number): CaseCore;
|
|
84
110
|
/**
|
|
85
111
|
* Make a pool of `tokens` that lives while `closed` does: when `closed` resolves the pool refuses
|
|
86
112
|
* grants, ends its child connections, drains its jobserver fifo and removes its directory. A run's own
|
|
@@ -88,9 +114,10 @@ export declare function takeParentCpuTokens(parent: InheritedCpuPool, want: numb
|
|
|
88
114
|
* (`liveUsage`); a sub-pool charges none (null).
|
|
89
115
|
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
90
116
|
*/
|
|
91
|
-
export declare function createCpuPool(tokens: number, closed: Promise<void>, memory: RunMemory | null, liveUsage?: (bound: string) => LiveUsageReading): Promise<{
|
|
117
|
+
export declare function createCpuPool(tokens: number, closed: Promise<void>, memory: RunMemory | null, liveUsage?: (bound: string) => LiveUsageReading, lending?: Lending): Promise<{
|
|
92
118
|
ok: true;
|
|
93
119
|
pool: CpuPool;
|
|
120
|
+
summary: () => PoolSummary;
|
|
94
121
|
} | {
|
|
95
122
|
ok: false;
|
|
96
123
|
error: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAiB,MAAM,kBAAkB,CAAC;AAGxF,OAAO,EAAE,KAAK,gBAAgB,EAAmB,MAAM,uBAAuB,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,yFAAyF;AACzF,MAAM,WAAW,OAAO;IACtB,oFAAoF;IACpF,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C;;;;OAIG;IACH,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;CAChC;AAED,MAAM,MAAM,YAAY,GAAG;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/cpu-pool.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAC;AAC/C,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9D,OAAO,EAAE,KAAK,gBAAgB,EAAE,KAAK,SAAS,EAAiB,MAAM,kBAAkB,CAAC;AAGxF,OAAO,EAAE,KAAK,gBAAgB,EAAmB,MAAM,uBAAuB,CAAC;AAE/E;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,yFAAyF;AACzF,MAAM,WAAW,OAAO;IACtB,oFAAoF;IACpF,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7C;;;;OAIG;IACH,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,CAAC;CAChC;AAED,gHAAgH;AAChH,MAAM,WAAW,WAAW;IAC1B,iHAAiH;IACjH,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,yEAAyE;IACzE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,8CAA8C;IAC9C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,yGAAyG;AACzG,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,MAAM,CAAC;CAC5B;AASD,qHAAqH;AACrH,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,MAAM,GAAG,IAAI,CAIhE;AAED,MAAM,MAAM,YAAY,GAAG;IAAE,EAAE,EAAE,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAsBvE,kHAAkH;AAClH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;CACrD;AAqED;;;;;GAKG;AACH,MAAM,MAAM,UAAU,GAClB;IACE,OAAO,EAAE,IAAI,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IAC5B,OAAO,EAAE,MAAM,IAAI,CAAC;IACpB,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB,GACD,QAAQ,CAAC;AAIb,kEAAkE;AAClE,MAAM,WAAW,UAAU;IACzB,wEAAwE;IACxE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,yEAAyE;IACzE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,gHAAgH;IAChH,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACrC,4DAA4D;IAC5D,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACtC;AAED;;;;;;;;GAQG;AACH,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,gBAAgB,EACxB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,GAAE,UAAe,GACvB,OAAO,CAAC,UAAU,CAAC,CA+DrB;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,QAAQ,CAsB9F;AAwJD;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,EACrB,MAAM,EAAE,SAAS,GAAG,IAAI,EACxB,SAAS,GAAE,CAAC,KAAK,EAAE,MAAM,KAAK,gBAAgC,EAC9D,OAAO,GAAE,OAAiB,GACzB,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,WAAW,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAuajG"}
|
|
@@ -16,12 +16,24 @@ import { ownedSocketDirectory } from '../../testing.js';
|
|
|
16
16
|
import { holdLease } from './gateway-lease.js';
|
|
17
17
|
import { readLiveUsage } from './host-memory.js';
|
|
18
18
|
import { CpuLeaseWait, HEARTBEAT_MS } from './lease-wait.js';
|
|
19
|
-
import { admit } from './pool-admission.js';
|
|
19
|
+
import { admit, IdleBesideStarved, place } from './pool-admission.js';
|
|
20
20
|
import { poolEnvironment } from './pool-environment.js';
|
|
21
|
+
/**
|
|
22
|
+
* The settle time (specs/cowshed/05_gateway.md, "Without mbx, every Nx run budgets itself"): Nx starts a run's next
|
|
23
|
+
* task when one ends, and that task's leaf takes what its predecessor freed before any case is lent it.
|
|
24
|
+
*/
|
|
25
|
+
const LEND_SETTLE_MS = 3_000;
|
|
26
|
+
const LENDING = { settleMs: LEND_SETTLE_MS, now: () => performance.now() };
|
|
27
|
+
/** The line a run's pool says of `summary` when it closes; null for a pool that starved no leaf and lent nothing. */
|
|
28
|
+
export function describeIdle(summary) {
|
|
29
|
+
return summary.starved === 0 && summary.lent === 0
|
|
30
|
+
? null
|
|
31
|
+
: `cowshed: cpu-pool idle beside_starved_core_s=${(summary.idleBesideStarvedMs / 1000).toFixed(3)} starved=${summary.starved} lent=${summary.lent}\n`;
|
|
32
|
+
}
|
|
21
33
|
/** Exactly a request's fields: any other field is an invalid request. */
|
|
22
34
|
const isRequest = (() => {
|
|
23
|
-
const _io0 = (input, _exceptionable = true) => "cpu-pool" === input.op && ("number" === typeof input.want && (_isTypeUint32_1._isTypeUint32(input.want) && 1 <= input.want)) && (undefined === input.memory || "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory)) && (undefined === input.parallelism || "number" === typeof input.parallelism && (_isTypeUint32_1._isTypeUint32(input.parallelism) && 1 <= input.parallelism)) && (undefined === input.task || "string" === typeof input.task && _stringLengthLte_1._stringLengthLte(input.task, 256)) && (undefined === input.alone || "boolean" === typeof input.alone) && (2 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
24
|
-
if (["op", "want", "memory", "parallelism", "task", "alone"].some(prop => key === prop))
|
|
35
|
+
const _io0 = (input, _exceptionable = true) => "cpu-pool" === input.op && ("number" === typeof input.want && (_isTypeUint32_1._isTypeUint32(input.want) && 1 <= input.want)) && (undefined === input.memory || "number" === typeof input.memory && _isTypeUint64_1._isTypeUint64(input.memory)) && (undefined === input.parallelism || "number" === typeof input.parallelism && (_isTypeUint32_1._isTypeUint32(input.parallelism) && 1 <= input.parallelism)) && (undefined === input.task || "string" === typeof input.task && _stringLengthLte_1._stringLengthLte(input.task, 256)) && (undefined === input.alone || "boolean" === typeof input.alone) && (undefined === input.lend || "boolean" === typeof input.lend) && (undefined === input.left || "number" === typeof input.left && _isTypeUint64_1._isTypeUint64(input.left)) && (undefined === input.gated || "boolean" === typeof input.gated) && (2 === Object.keys(input).length || Object.keys(input).every(key => {
|
|
36
|
+
if (["op", "want", "memory", "parallelism", "task", "alone", "lend", "left", "gated"].some(prop => key === prop))
|
|
25
37
|
return true;
|
|
26
38
|
const value = input[key];
|
|
27
39
|
if (undefined === value)
|
|
@@ -97,7 +109,7 @@ const isOversized = (() => {
|
|
|
97
109
|
* the child beside its parent's count.
|
|
98
110
|
*/
|
|
99
111
|
export async function takeParentCpuTokens(parent, want, memory, context = {}) {
|
|
100
|
-
const { parallelism, task, alone } = context;
|
|
112
|
+
const { parallelism, task, alone, gated } = context;
|
|
101
113
|
if (!Number.isSafeInteger(want) || want < 1) {
|
|
102
114
|
throw new RangeError(`a child asks its parent pool for a positive count, not ${want} (pool ${parent.tokens})`);
|
|
103
115
|
}
|
|
@@ -120,6 +132,7 @@ export async function takeParentCpuTokens(parent, want, memory, context = {}) {
|
|
|
120
132
|
...(memory === null ? {} : { memory, ...(parallelism === undefined ? {} : { parallelism }) }),
|
|
121
133
|
...(task === undefined ? {} : { task }),
|
|
122
134
|
...(alone === true ? { alone: true } : {}),
|
|
135
|
+
...(gated === true ? { gated: true } : {}),
|
|
123
136
|
};
|
|
124
137
|
const lease = await holdLease(parent.socket, request, 'the parent CPU pool', new CpuLeaseWait());
|
|
125
138
|
if (!lease.granted) {
|
|
@@ -159,6 +172,32 @@ export async function takeParentCpuTokens(parent, want, memory, context = {}) {
|
|
|
159
172
|
closed: lease.closed,
|
|
160
173
|
};
|
|
161
174
|
}
|
|
175
|
+
/**
|
|
176
|
+
* One token of `parent` asked for one test case of a leaf that holds a grant of it, named `task` in the request
|
|
177
|
+
* (specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold). The pool grants it only while no other
|
|
178
|
+
* request waits. `release` gives the token back, or withdraws the request while it waits. A pool that refuses, falls
|
|
179
|
+
* silent or is gone grants nothing: the case waits for a token of its leaf's own grant.
|
|
180
|
+
*/
|
|
181
|
+
export function lendParentToken(parent, task, left) {
|
|
182
|
+
const withdrawn = new AbortController();
|
|
183
|
+
const lease = holdLease(parent.socket, { op: 'cpu-pool', want: 1, lend: true, left: Math.max(0, Math.round(left)), task: task.slice(0, MAX_TASK_LENGTH) }, 'the parent CPU pool', new CpuLeaseWait(), undefined, withdrawn.signal);
|
|
184
|
+
let released = false;
|
|
185
|
+
return {
|
|
186
|
+
granted: lease.then((held) => held.granted && !released),
|
|
187
|
+
release: () => {
|
|
188
|
+
if (released)
|
|
189
|
+
return;
|
|
190
|
+
released = true;
|
|
191
|
+
withdrawn.abort();
|
|
192
|
+
void lease.then((held) => {
|
|
193
|
+
if (held.granted)
|
|
194
|
+
held.release();
|
|
195
|
+
});
|
|
196
|
+
},
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
/** A request's `task` is at most this long (`CpuPoolRequest`). */
|
|
200
|
+
const MAX_TASK_LENGTH = 256;
|
|
162
201
|
/** A thrown value's message. */
|
|
163
202
|
const describe = (error) => (error instanceof Error ? error.message : String(error));
|
|
164
203
|
/** Write `count` token bytes to the fifo's descriptor; a short write throws. The fifo holds a pipe buffer of them. */
|
|
@@ -297,7 +336,7 @@ function watchFifo(path, took, failed) {
|
|
|
297
336
|
* (`liveUsage`); a sub-pool charges none (null).
|
|
298
337
|
* Setup failures are values; a non-positive or unsafe count is a programmer bug and throws.
|
|
299
338
|
*/
|
|
300
|
-
export async function createCpuPool(tokens, closed, memory, liveUsage = readLiveUsage) {
|
|
339
|
+
export async function createCpuPool(tokens, closed, memory, liveUsage = readLiveUsage, lending = LENDING) {
|
|
301
340
|
if (!Number.isSafeInteger(tokens) || tokens < 1) {
|
|
302
341
|
throw new RangeError(`a CPU pool holds a positive safe integer of tokens, not ${tokens}`);
|
|
303
342
|
}
|
|
@@ -330,6 +369,15 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
330
369
|
// The children holding a grant, and whether one of them runs alone.
|
|
331
370
|
let holders = 0;
|
|
332
371
|
let aloneHolder = false;
|
|
372
|
+
// The tokens socket grants hold, the leaves that hold fewer than they asked and take no gate, and what the pool says
|
|
373
|
+
// of both when it closes.
|
|
374
|
+
let heldTokens = 0;
|
|
375
|
+
let starvedHolders = 0;
|
|
376
|
+
let partialGrants = 0;
|
|
377
|
+
let lentGrants = 0;
|
|
378
|
+
const idle = new IdleBesideStarved(lending.now());
|
|
379
|
+
// A leaf return arms this one-shot window. A later leaf return replaces it; a case return does not.
|
|
380
|
+
let settle;
|
|
333
381
|
// The readability wait while the queue's head finds no token, else undefined.
|
|
334
382
|
let watch;
|
|
335
383
|
let closing = false;
|
|
@@ -410,7 +458,7 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
410
458
|
return;
|
|
411
459
|
}
|
|
412
460
|
}
|
|
413
|
-
const admission = admit(queue, inHand + (slotLent ? 0 : 1), memory === null ? null : { capacity: memory.bytes, held: heldMemory }, { holders, alone: aloneHolder });
|
|
461
|
+
const admission = admit(queue, inHand + (slotLent ? 0 : 1), memory === null ? null : { capacity: memory.bytes, held: heldMemory }, { holders, alone: aloneHolder }, settle === undefined);
|
|
414
462
|
queue.splice(0, admission.grants.length);
|
|
415
463
|
for (const { request, tokens: granted, charge, oversized } of admission.grants) {
|
|
416
464
|
// Fifo bytes first, the implicit slot last.
|
|
@@ -421,15 +469,33 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
421
469
|
slotLent = true;
|
|
422
470
|
heldMemory += charge;
|
|
423
471
|
holders += 1;
|
|
472
|
+
heldTokens += granted;
|
|
473
|
+
const partial = granted < request.want;
|
|
474
|
+
const starved = partial && request.gated !== true;
|
|
475
|
+
if (partial)
|
|
476
|
+
partialGrants += 1;
|
|
477
|
+
if (starved)
|
|
478
|
+
starvedHolders += 1;
|
|
479
|
+
if (request.lend === true)
|
|
480
|
+
lentGrants += 1;
|
|
424
481
|
if (request.alone === true)
|
|
425
482
|
aloneHolder = true;
|
|
426
|
-
request.child.phase = {
|
|
483
|
+
request.child.phase = {
|
|
484
|
+
kind: 'holding',
|
|
485
|
+
bytes,
|
|
486
|
+
slot,
|
|
487
|
+
memory: charge,
|
|
488
|
+
alone: request.alone === true,
|
|
489
|
+
tokens: granted,
|
|
490
|
+
starved,
|
|
491
|
+
lend: request.lend === true,
|
|
492
|
+
};
|
|
427
493
|
request.child.socket.write(frame(memory === null
|
|
428
494
|
? { ok: true, lease: 'granted', tokens: granted }
|
|
429
495
|
: oversized
|
|
430
496
|
? { ok: true, lease: 'granted', tokens: granted, charged: charge, oversized: { memory: memory.bytes } }
|
|
431
497
|
: { ok: true, lease: 'granted', tokens: granted, charged: charge }));
|
|
432
|
-
if (memory !== null) {
|
|
498
|
+
if (memory !== null && request.lend !== true) {
|
|
433
499
|
const live = liveUsage(memory.source);
|
|
434
500
|
const held = live.ok
|
|
435
501
|
? `${live.usage.anon + live.usage.shmem} bytes (anon ${live.usage.anon}, shmem ${live.usage.shmem}, ${live.usage.source})`
|
|
@@ -444,6 +510,7 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
444
510
|
stopWatching();
|
|
445
511
|
}
|
|
446
512
|
returnTokens(inHand);
|
|
513
|
+
idle.observe(lending.now(), tokens - heldTokens, starvedHolders > 0 || queue.at(-1)?.lend === true);
|
|
447
514
|
}
|
|
448
515
|
const refuse = (child, error) => {
|
|
449
516
|
child.phase = { kind: 'done' };
|
|
@@ -473,6 +540,18 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
473
540
|
slotLent = false;
|
|
474
541
|
heldMemory -= child.phase.memory;
|
|
475
542
|
holders -= 1;
|
|
543
|
+
heldTokens -= child.phase.tokens;
|
|
544
|
+
if (child.phase.starved)
|
|
545
|
+
starvedHolders -= 1;
|
|
546
|
+
// What a leaf returns is its successor's first: Nx starts the run's next task as this one ends.
|
|
547
|
+
if (!closing && !child.phase.lend && lending.settleMs > 0) {
|
|
548
|
+
clearTimeout(settle);
|
|
549
|
+
settle = setTimeout(() => {
|
|
550
|
+
settle = undefined;
|
|
551
|
+
drain(0);
|
|
552
|
+
}, lending.settleMs);
|
|
553
|
+
settle.unref();
|
|
554
|
+
}
|
|
476
555
|
if (child.phase.alone)
|
|
477
556
|
aloneHolder = false;
|
|
478
557
|
if (!closing)
|
|
@@ -534,6 +613,10 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
534
613
|
refuse(child, 'a request is {"op":"cpu-pool","want":<a whole number of tokens, at least 1>,"memory"?:<bytes it is charged>}');
|
|
535
614
|
return;
|
|
536
615
|
}
|
|
616
|
+
if (parsed.lend === true && (parsed.want !== 1 || parsed.memory !== undefined || parsed.alone === true)) {
|
|
617
|
+
refuse(child, 'a lent request is one token for one test case: {"op":"cpu-pool","want":1,"lend":true,"task"?:<its case>}');
|
|
618
|
+
return;
|
|
619
|
+
}
|
|
537
620
|
const request = parsed;
|
|
538
621
|
child.phase = { kind: 'queued' };
|
|
539
622
|
const waiter = {
|
|
@@ -543,15 +626,11 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
543
626
|
parallelism: request.parallelism,
|
|
544
627
|
task: request.task,
|
|
545
628
|
alone: request.alone,
|
|
629
|
+
lend: request.lend,
|
|
630
|
+
left: request.left,
|
|
631
|
+
gated: request.gated,
|
|
546
632
|
};
|
|
547
|
-
|
|
548
|
-
// A leaf the kernel killed has waited once already: it goes ahead of the leaves that have not run.
|
|
549
|
-
const at = queue.findIndex((waiting) => waiting.alone !== true);
|
|
550
|
-
queue.splice(at === -1 ? queue.length : at, 0, waiter);
|
|
551
|
-
}
|
|
552
|
-
else {
|
|
553
|
-
queue.push(waiter);
|
|
554
|
-
}
|
|
633
|
+
queue.splice(place(queue, waiter), 0, waiter);
|
|
555
634
|
socket.write(frame({ ok: true, lease: 'queued', server: process.pid }));
|
|
556
635
|
drain(0);
|
|
557
636
|
});
|
|
@@ -563,6 +642,8 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
563
642
|
shut ??= (async () => {
|
|
564
643
|
closing = true;
|
|
565
644
|
clearInterval(heartbeat);
|
|
645
|
+
clearTimeout(settle);
|
|
646
|
+
idle.observe(lending.now(), 0, false);
|
|
566
647
|
stopWatching();
|
|
567
648
|
await started;
|
|
568
649
|
for (const socket of children) {
|
|
@@ -659,5 +740,10 @@ export async function createCpuPool(tokens, closed, memory, liveUsage = readLive
|
|
|
659
740
|
environment: poolEnvironment(socketPath, tokens, fifoPath),
|
|
660
741
|
close: shutdown,
|
|
661
742
|
},
|
|
743
|
+
summary: () => ({
|
|
744
|
+
idleBesideStarvedMs: idle.tokenMs(lending.now()),
|
|
745
|
+
starved: partialGrants,
|
|
746
|
+
lent: lentGrants,
|
|
747
|
+
}),
|
|
662
748
|
};
|
|
663
749
|
}
|