@smoothbricks/nx-plugin 0.7.17 → 0.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +315 -114
- package/dist/.build-inputs.json +47 -38
- package/dist/built-output-packages.js +1 -1
- package/dist/bun-lock-scope.d.ts +128 -20
- package/dist/bun-lock-scope.d.ts.map +1 -1
- package/dist/bun-lock-scope.js +267 -44
- package/dist/cargo-closure-input.d.ts +8 -6
- package/dist/cargo-closure-input.d.ts.map +1 -1
- package/dist/cargo-closure-input.js +15 -12
- package/dist/cargo-source-hash.d.ts +26 -25
- package/dist/cargo-source-hash.d.ts.map +1 -1
- package/dist/cargo-source-hash.js +84 -105
- package/dist/cargo-toolchain-paths.d.ts +2 -2
- package/dist/cargo-toolchain-paths.js +2 -2
- package/dist/cargo-workspace.d.ts +20 -2
- package/dist/cargo-workspace.d.ts.map +1 -1
- package/dist/cargo-workspace.js +43 -22
- package/dist/ensure-built-receipt.d.ts +3 -2
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +3 -2
- package/dist/eslint-config-closure.d.ts +8 -3
- package/dist/eslint-config-closure.d.ts.map +1 -1
- package/dist/eslint-config-closure.js +24 -5
- package/dist/executors/bounded-exec/admission-queue.d.ts +3 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -1
- package/dist/executors/bounded-exec/admission-queue.js +9 -10
- package/dist/executors/bounded-exec/awake-clock.d.ts +5 -0
- package/dist/executors/bounded-exec/awake-clock.d.ts.map +1 -0
- package/dist/executors/bounded-exec/awake-clock.js +22 -0
- package/dist/executors/bounded-exec/bounds.d.ts +19 -6
- package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
- package/dist/executors/bounded-exec/bounds.js +22 -15
- package/dist/executors/bounded-exec/case-gate.d.ts +114 -0
- package/dist/executors/bounded-exec/case-gate.d.ts.map +1 -0
- package/dist/executors/bounded-exec/case-gate.js +342 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.js +2 -2
- package/dist/executors/bounded-exec/cpu-pool.d.ts +28 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +102 -16
- package/dist/executors/bounded-exec/cpu-rise.d.ts +73 -0
- package/dist/executors/bounded-exec/cpu-rise.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-rise.js +111 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts +146 -0
- package/dist/executors/bounded-exec/endpoint-life.d.ts.map +1 -0
- package/dist/executors/bounded-exec/endpoint-life.js +228 -0
- package/dist/executors/bounded-exec/executor.d.ts +1 -1
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +256 -86
- package/dist/executors/bounded-exec/gateway-lease.d.ts +1 -1
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +8 -1
- package/dist/executors/bounded-exec/go-run-summary.d.ts +51 -0
- package/dist/executors/bounded-exec/go-run-summary.d.ts.map +1 -0
- package/dist/executors/bounded-exec/go-run-summary.js +98 -0
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts +11 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +125 -47
- package/dist/executors/bounded-exec/lease-wait.d.ts +1 -0
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +1 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts +42 -7
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-pool.js +45 -8
- package/dist/executors/bounded-exec/mbx-reservations.d.ts +12 -6
- package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
- package/dist/executors/bounded-exec/mbx-reservations.js +190 -284
- package/dist/executors/bounded-exec/parent-grant.d.ts +2 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -1
- package/dist/executors/bounded-exec/parent-grant.js +1 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +29 -3
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +47 -1
- package/dist/executors/bounded-exec/progress-channel.d.ts +38 -0
- package/dist/executors/bounded-exec/progress-channel.d.ts.map +1 -0
- package/dist/executors/bounded-exec/progress-channel.js +113 -0
- package/dist/executors/bounded-exec/progress.d.ts +5 -1
- package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
- package/dist/executors/bounded-exec/progress.js +31 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts +24 -1
- package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
- package/dist/executors/bounded-exec/ram-temp.js +123 -16
- package/dist/executors/bounded-exec/reservation-connection.d.ts +90 -0
- package/dist/executors/bounded-exec/reservation-connection.d.ts.map +1 -0
- package/dist/executors/bounded-exec/reservation-connection.js +248 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +3 -0
- package/dist/executors/bounded-exec/runner-sizing.d.ts +9 -5
- package/dist/executors/bounded-exec/runner-sizing.d.ts.map +1 -1
- package/dist/executors/bounded-exec/runner-sizing.js +13 -9
- package/dist/executors/bounded-exec/schema.d.ts +7 -2
- package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
- package/dist/executors/bounded-exec/schema.js +17 -1
- package/dist/executors/bounded-exec/suite-minimum.d.ts +21 -0
- package/dist/executors/bounded-exec/suite-minimum.d.ts.map +1 -0
- package/dist/executors/bounded-exec/suite-minimum.js +85 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts +2 -0
- package/dist/executors/bounded-exec/test-result-writer.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-result-writer.js +1 -0
- package/dist/executors/bounded-exec/test-results.d.ts +5 -0
- package/dist/executors/bounded-exec/test-results.d.ts.map +1 -1
- package/dist/executors/bounded-exec/test-results.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +39 -10
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +60 -14
- package/dist/executors/cargo-zigbuild/executor.d.ts +10 -8
- package/dist/executors/cargo-zigbuild/executor.d.ts.map +1 -1
- package/dist/executors/cargo-zigbuild/executor.js +32 -19
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +294 -152
- package/dist/managed-files/ci-workflow.d.ts +4 -1
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +9 -14
- package/dist/managed-files/files.d.ts +3 -3
- package/dist/managed-files/files.js +3 -3
- package/dist/managed-files/generator.d.ts +5 -1
- package/dist/managed-files/generator.d.ts.map +1 -1
- package/dist/managed-files/generator.js +22 -7
- package/dist/managed-files/render-memo.d.ts +13 -0
- package/dist/managed-files/render-memo.d.ts.map +1 -0
- package/dist/managed-files/render-memo.js +94 -0
- package/dist/task-results.js +2 -1
- package/dist/test-program.d.ts +11 -6
- package/dist/test-program.d.ts.map +1 -1
- package/dist/test-program.js +39 -35
- package/dist/tool-bins.d.ts +12 -0
- package/dist/tool-bins.d.ts.map +1 -1
- package/dist/tool-bins.js +31 -6
- package/dist/typecheck-test-policy.d.ts +10 -1
- package/dist/typecheck-test-policy.d.ts.map +1 -1
- package/dist/typecheck-test-policy.js +40 -24
- package/dist/workspace-config-policy.d.ts.map +1 -1
- package/dist/workspace-config-policy.js +135 -42
- package/dist/workspace-index.d.ts +14 -1
- package/dist/workspace-index.d.ts.map +1 -1
- package/dist/workspace-index.js +36 -4
- package/executors.json +2 -0
- package/managed/raw/envrc +24 -0
- package/managed/raw/patches/nx@24.0.3.patch +1851 -221
- package/managed/raw/patches/ttsc@0.37.1.patch +1055 -0
- package/managed/raw/tooling/direnv/developer-links.ts +122 -14
- package/managed/raw/tooling/direnv/devenv.smoo.nix +20 -19
- package/managed/templates/github/actions/save-nix-devenv/action.yml +19 -1
- package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
- package/package.json +68 -11
- package/runtime/nx-publish/.build-inputs.json +2 -2
- package/runtime/nx-publish/cache_adapter.js +293 -3
- package/runtime/nx-publish/nx_publish.js +36 -14
- package/src/built-output-packages.ts +1 -1
- package/src/bun-lock-scope.ts +293 -40
- package/src/cargo-closure-input.ts +20 -12
- package/src/cargo-source-hash.ts +119 -145
- package/src/cargo-toolchain-paths.ts +2 -2
- package/src/cargo-workspace.ts +59 -21
- package/src/ensure-built-receipt.ts +3 -2
- package/src/eslint-config-closure.ts +27 -6
- package/src/executors/bounded-exec/admission-queue.ts +11 -10
- package/src/executors/bounded-exec/awake-clock.ts +20 -0
- package/src/executors/bounded-exec/bounds.ts +41 -19
- package/src/executors/bounded-exec/case-gate.ts +365 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +3 -3
- package/src/executors/bounded-exec/cpu-pool.ts +139 -12
- package/src/executors/bounded-exec/cpu-rise.ts +156 -0
- package/src/executors/bounded-exec/endpoint-life.ts +279 -0
- package/src/executors/bounded-exec/executor.ts +341 -123
- package/src/executors/bounded-exec/gateway-lease.ts +6 -0
- package/src/executors/bounded-exec/go-run-summary.ts +137 -0
- package/src/executors/bounded-exec/hang-diagnostics.ts +150 -50
- package/src/executors/bounded-exec/lease-wait.ts +1 -0
- package/src/executors/bounded-exec/mbx-pool.ts +57 -9
- package/src/executors/bounded-exec/mbx-reservations.ts +216 -183
- package/src/executors/bounded-exec/parent-grant.ts +3 -0
- package/src/executors/bounded-exec/pool-admission.ts +59 -2
- package/src/executors/bounded-exec/progress-channel.ts +129 -0
- package/src/executors/bounded-exec/progress.ts +39 -3
- package/src/executors/bounded-exec/ram-temp.ts +137 -17
- package/src/executors/bounded-exec/reservation-connection.ts +221 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +2 -0
- package/src/executors/bounded-exec/runner-sizing.ts +13 -8
- package/src/executors/bounded-exec/schema.json +1 -1
- package/src/executors/bounded-exec/schema.ts +21 -2
- package/src/executors/bounded-exec/suite-minimum.ts +81 -0
- package/src/executors/bounded-exec/test-result-writer.ts +3 -0
- package/src/executors/bounded-exec/test-results.ts +5 -0
- package/src/executors/bounded-exec/verdict.ts +83 -22
- package/src/executors/cargo-zigbuild/executor.ts +30 -19
- package/src/index.ts +357 -166
- package/src/managed-files/ci-workflow.ts +13 -17
- package/src/managed-files/files.ts +3 -3
- package/src/managed-files/generator.ts +21 -6
- package/src/managed-files/render-memo.ts +135 -0
- package/src/test-program.ts +36 -41
- package/src/tool-bins.ts +35 -6
- package/src/typecheck-test-policy.ts +49 -24
- package/src/workspace-config-policy.ts +161 -43
- package/src/workspace-index.ts +40 -5
- package/test-timings.json +157 -141
- package/managed/raw/patches/ttsc@0.36.3.patch +0 -1660
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import typia, { type tags } from 'typia';
|
|
2
|
+
import type { MbxPriority } from './mbx-pool.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* One process's connection to its run's mbx reservation endpoint (mbx-reservations.ts; specs/cowshed/05_gateway.md,
|
|
6
|
+
* "Parallelism inside a job"), as a state machine over what the process asks of it and what its socket says. Pure: no
|
|
7
|
+
* I/O and no clock. The shell maps its callers' requests and the socket's events into `ConnectionEvent`s and executes
|
|
8
|
+
* the `ConnectionEffect`s each step answers, in their order; it decides nothing.
|
|
9
|
+
*
|
|
10
|
+
* Every request is answered once: granted, refused by the pool, withdrawn by its own release, or failed by the
|
|
11
|
+
* connection's end, which also loses every grant still held. A grant lasts until its release or that end, and the
|
|
12
|
+
* endpoint is told of every release it can still hear. Nothing is written once the connection has ended.
|
|
13
|
+
*/
|
|
14
|
+
export interface Connection {
|
|
15
|
+
readonly endpoint: string;
|
|
16
|
+
/** What the socket said after its last whole line. */
|
|
17
|
+
readonly unread: string;
|
|
18
|
+
/** Each request neither released nor refused: waiting for its answer, or holding its grant. */
|
|
19
|
+
readonly entries: ReadonlyMap<number, 'waiting' | 'held'>;
|
|
20
|
+
/** Why the connection ended, which answers every later request; null while it serves. */
|
|
21
|
+
readonly closed: ReservationRefusal | null;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** Why a reservation holds no cores: the endpoint asked, and the connect error's code when it could not be reached. */
|
|
25
|
+
export interface ReservationRefusal {
|
|
26
|
+
readonly endpoint: string;
|
|
27
|
+
/** The code of the error that ended the connection (`ENOENT`: the endpoint's socket is gone), else null. */
|
|
28
|
+
readonly errno: string | null;
|
|
29
|
+
readonly message: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** What the shell observes: its callers' requests, each under an id of its own, and its socket's events. */
|
|
33
|
+
export type ConnectionEvent =
|
|
34
|
+
| {
|
|
35
|
+
readonly type: 'reserve';
|
|
36
|
+
readonly id: number;
|
|
37
|
+
readonly cpus: number;
|
|
38
|
+
readonly priority: MbxPriority;
|
|
39
|
+
/** The fewest cores the reservation starts on before mbx's aging bound; one is no minimum. */
|
|
40
|
+
readonly min: number;
|
|
41
|
+
/** The class mbx predicts the reservation by. */
|
|
42
|
+
readonly name?: string | undefined;
|
|
43
|
+
}
|
|
44
|
+
| { readonly type: 'release'; readonly id: number }
|
|
45
|
+
| { readonly type: 'data'; readonly text: string }
|
|
46
|
+
| { readonly type: 'error'; readonly message: string; readonly code: string | null }
|
|
47
|
+
| { readonly type: 'close' };
|
|
48
|
+
|
|
49
|
+
/** A request's answer: its cores, or why it has none. */
|
|
50
|
+
export type Answer =
|
|
51
|
+
| { readonly ok: true; readonly cpus: number }
|
|
52
|
+
| { readonly ok: false; readonly refusal: ReservationRefusal };
|
|
53
|
+
|
|
54
|
+
export type ConnectionEffect =
|
|
55
|
+
/** Keep the process alive for the connection: a request waits or holds. */
|
|
56
|
+
| { readonly do: 'ref' }
|
|
57
|
+
/** Nothing waits or holds: the connection no longer keeps the process alive. */
|
|
58
|
+
| { readonly do: 'unref' }
|
|
59
|
+
| { readonly do: 'write'; readonly line: string }
|
|
60
|
+
| { readonly do: 'answer'; readonly id: number; readonly answer: Answer }
|
|
61
|
+
/** A grant its request was answered with is lost, and why. */
|
|
62
|
+
| { readonly do: 'lose'; readonly id: number; readonly why: string }
|
|
63
|
+
/** Forget the connection, so the next request opens another, and destroy its socket. */
|
|
64
|
+
| { readonly do: 'hang-up' };
|
|
65
|
+
|
|
66
|
+
export type ConnectionStep = readonly [Connection, readonly ConnectionEffect[]];
|
|
67
|
+
|
|
68
|
+
type Uint = number & tags.Type<'uint32'>;
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The lines a client writes (mbx's `docs/scheduling.md`, "Serve reservations to many commands"). `min` is the fewest
|
|
72
|
+
* of its cores a reservation starts on before it has waited mbx's aging bound; absent, it starts on one.
|
|
73
|
+
*/
|
|
74
|
+
type ReservationRequest =
|
|
75
|
+
| {
|
|
76
|
+
type: 'reserve';
|
|
77
|
+
id: Uint;
|
|
78
|
+
cpus: Uint & tags.Minimum<1>;
|
|
79
|
+
priority: MbxPriority;
|
|
80
|
+
min?: Uint & tags.Minimum<2>;
|
|
81
|
+
/** The class mbx keeps the reservation's run times under and predicts it by. */
|
|
82
|
+
name?: string;
|
|
83
|
+
}
|
|
84
|
+
| { type: 'release'; id: Uint };
|
|
85
|
+
|
|
86
|
+
/** The lines the endpoint answers; an added field is mbx's to add. */
|
|
87
|
+
type ReservationReply =
|
|
88
|
+
| { type: 'granted'; id: Uint; cpus: Uint & tags.Minimum<1> }
|
|
89
|
+
| { type: 'refused'; id: Uint; error: string }
|
|
90
|
+
| { type: 'error'; error: string };
|
|
91
|
+
|
|
92
|
+
const stringifyRequest = typia.json.createStringify<ReservationRequest>();
|
|
93
|
+
const parseReply = typia.json.createValidateParse<ReservationReply>();
|
|
94
|
+
|
|
95
|
+
/** A connection just opened to `endpoint`. */
|
|
96
|
+
export function opened(endpoint: string): Connection {
|
|
97
|
+
return { endpoint, unread: '', entries: new Map(), closed: null };
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** The next state of `connection` at `event`, and what the shell does about it. */
|
|
101
|
+
export function connectionStep(connection: Connection, event: ConnectionEvent): ConnectionStep {
|
|
102
|
+
if (connection.closed !== null) {
|
|
103
|
+
// An ended connection answers a request with its end; nothing else of it is left to decide.
|
|
104
|
+
return event.type === 'reserve'
|
|
105
|
+
? [connection, [{ do: 'answer', id: event.id, answer: { ok: false, refusal: connection.closed } }]]
|
|
106
|
+
: [connection, []];
|
|
107
|
+
}
|
|
108
|
+
switch (event.type) {
|
|
109
|
+
case 'reserve': {
|
|
110
|
+
// A minimum of one is no minimum, and mbx refuses one above the ask.
|
|
111
|
+
const gathers = Math.min(event.min, event.cpus);
|
|
112
|
+
const line = stringifyRequest({
|
|
113
|
+
type: 'reserve',
|
|
114
|
+
id: event.id,
|
|
115
|
+
cpus: event.cpus,
|
|
116
|
+
priority: event.priority,
|
|
117
|
+
...(gathers > 1 ? { min: gathers } : {}),
|
|
118
|
+
...(event.name === undefined ? {} : { name: event.name }),
|
|
119
|
+
});
|
|
120
|
+
return [
|
|
121
|
+
{ ...connection, entries: new Map(connection.entries).set(event.id, 'waiting') },
|
|
122
|
+
[{ do: 'ref' }, { do: 'write', line: `${line}\n` }],
|
|
123
|
+
];
|
|
124
|
+
}
|
|
125
|
+
case 'release': {
|
|
126
|
+
const entry = connection.entries.get(event.id);
|
|
127
|
+
// Released, refused or never asked: nothing of it is left to give back.
|
|
128
|
+
if (entry === undefined) return [connection, []];
|
|
129
|
+
const entries = without(connection.entries, event.id);
|
|
130
|
+
return [
|
|
131
|
+
{ ...connection, entries },
|
|
132
|
+
[
|
|
133
|
+
...(entry === 'waiting' ? [refused(connection, event.id, 'released while it waited')] : []),
|
|
134
|
+
{ do: 'write', line: `${stringifyRequest({ type: 'release', id: event.id })}\n` },
|
|
135
|
+
...unreferenced(entries),
|
|
136
|
+
],
|
|
137
|
+
];
|
|
138
|
+
}
|
|
139
|
+
case 'data':
|
|
140
|
+
return read({ ...connection, unread: connection.unread + event.text }, []);
|
|
141
|
+
case 'error':
|
|
142
|
+
return close(connection, `mbx reservation endpoint ${connection.endpoint}: ${event.message}`, event.code, []);
|
|
143
|
+
case 'close':
|
|
144
|
+
return close(connection, `mbx reservation endpoint ${connection.endpoint} closed the connection`, null, []);
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function without(entries: ReadonlyMap<number, 'waiting' | 'held'>, id: number): Map<number, 'waiting' | 'held'> {
|
|
149
|
+
const rest = new Map(entries);
|
|
150
|
+
rest.delete(id);
|
|
151
|
+
return rest;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function unreferenced(entries: ReadonlyMap<number, 'waiting' | 'held'>): ConnectionEffect[] {
|
|
155
|
+
return entries.size === 0 ? [{ do: 'unref' }] : [];
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function refused(connection: Connection, id: number, message: string): ConnectionEffect {
|
|
159
|
+
return { do: 'answer', id, answer: { ok: false, refusal: { endpoint: connection.endpoint, errno: null, message } } };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** `line` as a reply, or null when it is none: typia's parse validates what `JSON.parse` returns and throws what it throws. */
|
|
163
|
+
function replyOf(line: string): ReservationReply | null {
|
|
164
|
+
try {
|
|
165
|
+
const parsed = parseReply(line);
|
|
166
|
+
return parsed.success ? parsed.data : null;
|
|
167
|
+
} catch {
|
|
168
|
+
return null;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
/** Each whole line of `connection.unread` in turn, until none is left or one ends the connection. */
|
|
172
|
+
function read(connection: Connection, done: readonly ConnectionEffect[]): ConnectionStep {
|
|
173
|
+
const end = connection.unread.indexOf('\n');
|
|
174
|
+
if (end === -1) return [connection, done];
|
|
175
|
+
const line = connection.unread.slice(0, end);
|
|
176
|
+
const rest: Connection = { ...connection, unread: connection.unread.slice(end + 1) };
|
|
177
|
+
const reply = replyOf(line);
|
|
178
|
+
if (reply === null) {
|
|
179
|
+
const why = `mbx reservation endpoint ${connection.endpoint} answered ${JSON.stringify(line)}, which is no reply`;
|
|
180
|
+
return close(rest, why, null, done);
|
|
181
|
+
}
|
|
182
|
+
const [next, effects] = answer(rest, reply);
|
|
183
|
+
return next.closed === null ? read(next, [...done, ...effects]) : [next, [...done, ...effects]];
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function answer(connection: Connection, reply: ReservationReply): ConnectionStep {
|
|
187
|
+
const { endpoint } = connection;
|
|
188
|
+
if (reply.type === 'error') {
|
|
189
|
+
return close(connection, `mbx reservation endpoint ${endpoint} closed this connection: ${reply.error}`, null, []);
|
|
190
|
+
}
|
|
191
|
+
const entry = connection.entries.get(reply.id);
|
|
192
|
+
// Released before its answer arrived: the endpoint drops what it won.
|
|
193
|
+
if (entry === undefined) return [connection, []];
|
|
194
|
+
if (entry === 'held') {
|
|
195
|
+
return close(connection, `mbx reservation endpoint ${endpoint} answered reservation ${reply.id} twice`, null, []);
|
|
196
|
+
}
|
|
197
|
+
if (reply.type === 'granted') {
|
|
198
|
+
return [
|
|
199
|
+
{ ...connection, entries: new Map(connection.entries).set(reply.id, 'held') },
|
|
200
|
+
[{ do: 'answer', id: reply.id, answer: { ok: true, cpus: reply.cpus } }],
|
|
201
|
+
];
|
|
202
|
+
}
|
|
203
|
+
const entries = without(connection.entries, reply.id);
|
|
204
|
+
return [{ ...connection, entries }, [refused(connection, reply.id, reply.error), ...unreferenced(entries)]];
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/** End the connection: what waits is refused by its end, what is held is lost to it. */
|
|
208
|
+
function close(
|
|
209
|
+
connection: Connection,
|
|
210
|
+
message: string,
|
|
211
|
+
errno: string | null,
|
|
212
|
+
done: readonly ConnectionEffect[],
|
|
213
|
+
): ConnectionStep {
|
|
214
|
+
const refusal: ReservationRefusal = { endpoint: connection.endpoint, errno, message };
|
|
215
|
+
const settled = Array.from(
|
|
216
|
+
connection.entries,
|
|
217
|
+
([id, entry]): ConnectionEffect =>
|
|
218
|
+
entry === 'waiting' ? { do: 'answer', id, answer: { ok: false, refusal } } : { do: 'lose', id, why: message },
|
|
219
|
+
);
|
|
220
|
+
return [{ ...connection, unread: '', entries: new Map(), closed: refusal }, [...done, ...settled, { do: 'hang-up' }]];
|
|
221
|
+
}
|
|
@@ -160,6 +160,8 @@ function servePoolThread(tokens: number, memory: RunMemory): Promise<ServedPool>
|
|
|
160
160
|
finish({ ok: false, error: report.error });
|
|
161
161
|
now.served({ ok: false, error: report.error });
|
|
162
162
|
} else if (now.kind === 'closing' && report.kind === 'closed') {
|
|
163
|
+
// What the run's pool left idle beside starved holds, said with the run's end.
|
|
164
|
+
if (report.idle !== null) process.stderr.write(report.idle);
|
|
163
165
|
finish(report.outcome);
|
|
164
166
|
now.closed(report.outcome);
|
|
165
167
|
} else {
|
|
@@ -303,6 +303,8 @@ export function withInvocationFlag(
|
|
|
303
303
|
/**
|
|
304
304
|
* Size the runner to the lesser of its bounded demand and grant, keeping the test-thread ceilings. Rewrite named flags
|
|
305
305
|
* so they cannot outrank that size. `CPU_TOKENS_ENV` still reports the actual held grant.
|
|
306
|
+
* A `gated` nextest runner starts every thread of its ask instead: each case takes its token at the leaf's gate
|
|
307
|
+
* (case-gate.ts; specs/cowshed/05_gateway.md, "A case starts on a core", Inside a hold).
|
|
306
308
|
*
|
|
307
309
|
* Cargo's rustc count is the pool's: the executor's sub-pool exports its jobserver fifo, so no `CARGO_BUILD_JOBS`
|
|
308
310
|
* comes from the grant (specs/cowshed/05_gateway.md, "Cargo reads its holder's jobserver"). Test threads take no
|
|
@@ -313,6 +315,7 @@ export function sizeRunner(
|
|
|
313
315
|
demand: Demand,
|
|
314
316
|
command: string,
|
|
315
317
|
tokens: number,
|
|
318
|
+
gated = false,
|
|
316
319
|
): { command: string; env: Record<string, string> } {
|
|
317
320
|
const runnerCount = Math.min(tokens, demand.want);
|
|
318
321
|
const count = String(runnerCount);
|
|
@@ -320,8 +323,8 @@ export function sizeRunner(
|
|
|
320
323
|
switch (demand.kind) {
|
|
321
324
|
case 'nextest':
|
|
322
325
|
return {
|
|
323
|
-
command: command.replace(TEST_THREADS, `--test-threads=${count}`),
|
|
324
|
-
env: { [CPU_TOKENS_ENV]: held, NEXTEST_TEST_THREADS: count },
|
|
326
|
+
command: command.replace(TEST_THREADS, `--test-threads=${gated ? demand.want : count}`),
|
|
327
|
+
env: { [CPU_TOKENS_ENV]: held, NEXTEST_TEST_THREADS: gated ? String(demand.want) : count },
|
|
325
328
|
};
|
|
326
329
|
case 'bun-parallel':
|
|
327
330
|
return { command: command.replace(BUN_PARALLEL, `--parallel=${count}`), env: { [CPU_TOKENS_ENV]: held } };
|
|
@@ -371,9 +374,11 @@ export function sizeRunner(
|
|
|
371
374
|
* The shell statements that run first in a reserved leaf's reservation and size its runner to the grant, which only
|
|
372
375
|
* the command learns as it starts: `CPU_TOKENS_ENV` becomes the cores mbx granted (`GRANTED_CPUS_ENV`), never more
|
|
373
376
|
* than the leaf asked for (`WANT_ENV`, set before it reserved). The test-thread variables a sizing sets follow from it,
|
|
374
|
-
* under their inherited ceilings, outranking the target's own as a numeric grant's do (`sizeRunner`).
|
|
377
|
+
* under their inherited ceilings, outranking the target's own as a numeric grant's do (`sizeRunner`). A `gated` nextest
|
|
378
|
+
* leaf's runner starts every thread of its ask instead: each case takes its core at the leaf's gate (case-gate.ts;
|
|
379
|
+
* specs/cowshed/05_gateway.md, "A case starts on a core").
|
|
375
380
|
*/
|
|
376
|
-
export function grantedSizing(demand: ReservedDemand): string {
|
|
381
|
+
export function grantedSizing(demand: ReservedDemand, gated: boolean): string {
|
|
377
382
|
const asked = `$${WANT_ENV}`;
|
|
378
383
|
const named = `$${GRANTED_CPUS_ENV}`;
|
|
379
384
|
const tokens = `$${CPU_TOKENS_ENV}`;
|
|
@@ -382,7 +387,7 @@ export function grantedSizing(demand: ReservedDemand): string {
|
|
|
382
387
|
const statements = [`export ${CPU_TOKENS_ENV}="$(( ${named} < ${asked} ? ${named} : ${asked} ))"`];
|
|
383
388
|
switch (demand.kind) {
|
|
384
389
|
case 'nextest':
|
|
385
|
-
statements.push(`export NEXTEST_TEST_THREADS="${tokens}"`);
|
|
390
|
+
statements.push(`export NEXTEST_TEST_THREADS="${gated ? asked : tokens}"`);
|
|
386
391
|
break;
|
|
387
392
|
case 'process':
|
|
388
393
|
// A process's script may start Cargo, whose test threads take no jobserver token: its whole grant.
|
|
@@ -400,13 +405,13 @@ export function grantedSizing(demand: ReservedDemand): string {
|
|
|
400
405
|
|
|
401
406
|
/**
|
|
402
407
|
* `command` with a reserved runner's named count read from its grant, `CPU_TOKENS_ENV` as `grantedSizing` sets it,
|
|
403
|
-
* where the runner's shell runs it; the runner then runs on all it holds.
|
|
408
|
+
* where the runner's shell runs it; the runner then runs on all it holds. A `gated` nextest runner's count is its ask.
|
|
404
409
|
*/
|
|
405
|
-
export function sizeRunnerToGrant(demand: ReservedDemand, command: string): string {
|
|
410
|
+
export function sizeRunnerToGrant(demand: ReservedDemand, command: string, gated: boolean): string {
|
|
406
411
|
const granted = `\${${CPU_TOKENS_ENV}}`;
|
|
407
412
|
switch (demand.kind) {
|
|
408
413
|
case 'nextest':
|
|
409
|
-
return command.replace(TEST_THREADS, `--test-threads=${granted}`);
|
|
414
|
+
return command.replace(TEST_THREADS, `--test-threads=${gated ? `\${${WANT_ENV}}` : granted}`);
|
|
410
415
|
case 'bun-parallel':
|
|
411
416
|
return command.replace(BUN_PARALLEL, `--parallel=${granted}`);
|
|
412
417
|
case 'stryker': {
|
|
@@ -42,7 +42,7 @@
|
|
|
42
42
|
},
|
|
43
43
|
"fixtureCommand": {
|
|
44
44
|
"type": "string",
|
|
45
|
-
"description": "A compiler-fixture producer for command: a shell command run first, in the same cwd and environment, as a lifecycle build interval the progress bound does not run during. Under mbx it runs
|
|
45
|
+
"description": "A compiler-fixture producer for command: a shell command run first, in the same cwd and environment, as a lifecycle build interval the progress bound does not run during. Under mbx's pool it runs before its leaf's CPU reservation, as pooled work: a reservation is never held across a build. Its failure fails the run and command starts only after it exits 0. For warmth outside Nx outputs (a compiler plugin cache, a Cargo target directory): inside the consumer's task a cache hit restores the verdict and warms nothing, an execution warms first. Requires fixtureTimeoutMs."
|
|
46
46
|
},
|
|
47
47
|
"fixtureTimeoutMs": {
|
|
48
48
|
"type": "number",
|
|
@@ -29,8 +29,8 @@ export interface BoundedExecCommand {
|
|
|
29
29
|
|
|
30
30
|
/**
|
|
31
31
|
* A compiler-fixture producer for `command`: a shell command run first, in the same cwd and environment, inside
|
|
32
|
-
* the run's CPU grant; under mbx
|
|
33
|
-
*
|
|
32
|
+
* the run's CPU grant; under mbx's pool before its leaf's reservation, as pooled work whose compilers mbx admits one
|
|
33
|
+
* by one, since a reservation is never held across a build.
|
|
34
34
|
* Its whole life is a lifecycle build interval — the progress bound does not run during it, as it does not run
|
|
35
35
|
* during a `cargo test` build. Its failure fails the run; `command` starts only after it exits 0.
|
|
36
36
|
*
|
|
@@ -64,3 +64,22 @@ export interface BoundedExecCommand {
|
|
|
64
64
|
args?: string | string[];
|
|
65
65
|
__unparsed__?: string[];
|
|
66
66
|
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The command line a run of `options` starts: `command`, then `args`, then the arguments Nx forwards from its own
|
|
70
|
+
* command line unless `forwardAllArgs` is false. The executor runs it and graph inference reads its runner from it.
|
|
71
|
+
*/
|
|
72
|
+
export function boundedCommand(
|
|
73
|
+
options: Pick<BoundedExecCommand, 'command' | 'args' | 'forwardAllArgs' | '__unparsed__'>,
|
|
74
|
+
): string {
|
|
75
|
+
const parts = [options.command];
|
|
76
|
+
if (Array.isArray(options.args)) {
|
|
77
|
+
parts.push(...options.args);
|
|
78
|
+
} else if (options.args) {
|
|
79
|
+
parts.push(options.args);
|
|
80
|
+
}
|
|
81
|
+
if (options.forwardAllArgs !== false && options.__unparsed__?.length) {
|
|
82
|
+
parts.push(...options.__unparsed__);
|
|
83
|
+
}
|
|
84
|
+
return parts.join(' ');
|
|
85
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The minimum a whole-suite test leaf names on its reservation (specs/cowshed/05_gateway.md, "A minimum for a whole
|
|
3
|
+
* suite"): "the fewest threads at which its last recorded test walls, longest first, never leave 180 s without a test
|
|
4
|
+
* finishing". The 180 s there is the leaf's own no-progress bound, `idleMs`.
|
|
5
|
+
*
|
|
6
|
+
* A grant never grows, so a suite that starts on one core runs every test in turn for its whole life: a suite of
|
|
7
|
+
* minutes-long cases granted one core beside seventeen short leaves ran 21 minutes where its longest case is 8.5.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { readFile } from 'node:fs/promises';
|
|
11
|
+
import { join } from 'node:path';
|
|
12
|
+
import { TEST_RESULTS_FILE } from './test-results.js';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Each test's wall as the task's last run closed it, by the test's id, from its results in `directory`; none for a
|
|
16
|
+
* task that has not run, whose results are gone or cannot be read. They size a request and order a leaf's cases at its
|
|
17
|
+
* gate (case-gate.ts), never a charge: a leaf with none names no minimum and starts as it did before it had any.
|
|
18
|
+
*/
|
|
19
|
+
export async function readLastWalls(directory: string): Promise<Map<string, number>> {
|
|
20
|
+
let text: string;
|
|
21
|
+
try {
|
|
22
|
+
text = await readFile(join(directory, TEST_RESULTS_FILE), 'utf8');
|
|
23
|
+
} catch {
|
|
24
|
+
return new Map();
|
|
25
|
+
}
|
|
26
|
+
const walls = new Map<string, number>();
|
|
27
|
+
for (const line of text.split('\n')) {
|
|
28
|
+
if (line === '') continue;
|
|
29
|
+
let row: unknown;
|
|
30
|
+
try {
|
|
31
|
+
row = JSON.parse(line);
|
|
32
|
+
} catch {
|
|
33
|
+
return new Map();
|
|
34
|
+
}
|
|
35
|
+
if (typeof row !== 'object' || row === null || !('durationMs' in row) || !('id' in row)) continue;
|
|
36
|
+
const { id, durationMs: wall } = row;
|
|
37
|
+
if (typeof id === 'string' && typeof wall === 'number' && Number.isFinite(wall) && wall >= 0) walls.set(id, wall);
|
|
38
|
+
}
|
|
39
|
+
return walls;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** When each test finishes on `threads` threads: longest first, each on the thread that frees first. */
|
|
43
|
+
function finishTimes(wallsLongestFirst: readonly number[], threads: number): number[] {
|
|
44
|
+
const free = new Array<number>(threads).fill(0);
|
|
45
|
+
const finished: number[] = [];
|
|
46
|
+
for (const wall of wallsLongestFirst) {
|
|
47
|
+
let soonest = 0;
|
|
48
|
+
for (let thread = 1; thread < threads; thread++) {
|
|
49
|
+
if ((free[thread] ?? 0) < (free[soonest] ?? 0)) soonest = thread;
|
|
50
|
+
}
|
|
51
|
+
const end = (free[soonest] ?? 0) + wall;
|
|
52
|
+
free[soonest] = end;
|
|
53
|
+
finished.push(end);
|
|
54
|
+
}
|
|
55
|
+
return finished.sort((a, b) => a - b);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Whether no span longer than `idleMs` passes without a test finishing, from the start to the last finish. */
|
|
59
|
+
function keepsFinishing(finished: readonly number[], idleMs: number): boolean {
|
|
60
|
+
let last = 0;
|
|
61
|
+
for (const end of finished) {
|
|
62
|
+
if (end - last > idleMs) return false;
|
|
63
|
+
last = end;
|
|
64
|
+
}
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The fewest threads, from one to `want`, at which the recorded walls keep a test finishing within every `idleMs`;
|
|
70
|
+
* `want` when none does (a single test longer than the bound finishes within it on no count). One, which is no
|
|
71
|
+
* minimum, for a leaf with no recorded wall.
|
|
72
|
+
*/
|
|
73
|
+
export function suiteMinimum(recordedWallsMs: readonly number[], want: number, idleMs: number): number {
|
|
74
|
+
if (recordedWallsMs.length === 0) return 1;
|
|
75
|
+
const longestFirst = [...recordedWallsMs].sort((a, b) => b - a);
|
|
76
|
+
const most = Math.min(want, longestFirst.length);
|
|
77
|
+
for (let threads = 1; threads <= most; threads++) {
|
|
78
|
+
if (keepsFinishing(finishTimes(longestFirst, threads), idleMs)) return threads;
|
|
79
|
+
}
|
|
80
|
+
return most;
|
|
81
|
+
}
|
|
@@ -13,6 +13,8 @@ export interface NativeTestResult {
|
|
|
13
13
|
readonly source: ClosedTestResult['source'];
|
|
14
14
|
readonly contradictsCursor?: number;
|
|
15
15
|
readonly durationMs: number | null;
|
|
16
|
+
/** How long the case waited at its leaf's gate before it ran, which `durationMs` leaves out; absent for none. */
|
|
17
|
+
readonly waitMs?: number;
|
|
16
18
|
readonly sourceRef: string | null;
|
|
17
19
|
readonly failureOutput: string | null;
|
|
18
20
|
}
|
|
@@ -115,6 +117,7 @@ export async function createTestResultWriter(
|
|
|
115
117
|
source: result.source,
|
|
116
118
|
...(result.contradictsCursor === undefined ? {} : { contradictsCursor: result.contradictsCursor }),
|
|
117
119
|
durationMs: result.durationMs,
|
|
120
|
+
...(result.waitMs === undefined ? {} : { waitMs: result.waitMs }),
|
|
118
121
|
sourceRef: normalizeSourceRef(result.sourceRef, sourceRoot),
|
|
119
122
|
causeRef,
|
|
120
123
|
},
|
|
@@ -15,6 +15,11 @@ export interface ClosedTestResult {
|
|
|
15
15
|
readonly contradictsCursor?: ByteOffset;
|
|
16
16
|
/** Null when the native runner reports no duration (for example a skipped test). */
|
|
17
17
|
readonly durationMs: (number & tags.Minimum<0>) | null;
|
|
18
|
+
/**
|
|
19
|
+
* How long the case waited for a core at its leaf's gate before it ran (specs/cowshed/05_gateway.md, "A case starts
|
|
20
|
+
* on a core"), which `durationMs` leaves out; absent for a case that did not wait.
|
|
21
|
+
*/
|
|
22
|
+
readonly waitMs?: number & tags.Minimum<0>;
|
|
18
23
|
/** Native file:line, never a location inferred from human output. */
|
|
19
24
|
readonly sourceRef: string | null;
|
|
20
25
|
/** UTF-8 byte range [startByte, endByte), relative to the task artifact directory. */
|
|
@@ -3,6 +3,7 @@ import { join } from 'node:path';
|
|
|
3
3
|
import typia from 'typia';
|
|
4
4
|
import type { CpuGrantRecord } from './cpu-grant.js';
|
|
5
5
|
import { parseCpuGrant } from './cpu-grant.js';
|
|
6
|
+
import { CPU_RISING_CLASS, type CpuRise, describeCpuRises } from './cpu-rise.js';
|
|
6
7
|
import type { CpuWaitRow } from './lease-wait.js';
|
|
7
8
|
import { describeHolders, type LockWait, type Opener } from './lock-wait.js';
|
|
8
9
|
import type { MbxQueuedRequest } from './mbx-queue.js';
|
|
@@ -18,18 +19,25 @@ import { TEST_FAILURES_FILE, TEST_RESULTS_FILE } from './test-results.js';
|
|
|
18
19
|
* outlived `timeoutMs` (`total`), its fixture segment outlived `fixtureTimeoutMs`
|
|
19
20
|
* (`fixture`), or every failing test in the runner's report failed on
|
|
20
21
|
* its runner's per-test timeout (`test`). A loaded host produces exactly these.
|
|
21
|
-
* - `wedged`: `idleTimeoutMs` fired: no test finished for that long during a
|
|
22
|
-
* the suite's completion bound was exceeded, not that every process stopped:
|
|
23
|
-
* exceed it too. The verdict names tests reported running but never finished and
|
|
24
|
-
* aborted; diagnostics distinguish a stalled wait from a progressing but oversized
|
|
25
|
-
* When the snapshot named a lock wait, `lockWaits` names the file the run holds on a numbered descriptor
|
|
26
|
-
* processes outside the run that also hold it open: open-handle evidence, not a kernel lock owner.
|
|
22
|
+
* - `wedged`: `idleTimeoutMs` fired: no test finished and no running test reported progress for that long during a
|
|
23
|
+
* chargeable test interval. This proves the suite's completion bound was exceeded, not that every process stopped:
|
|
24
|
+
* an expensive active fixture can exceed it too. The verdict names tests reported running but never finished and
|
|
25
|
+
* tests the report says were aborted; diagnostics distinguish a stalled wait from a progressing but oversized
|
|
26
|
+
* lifecycle. When the snapshot named a lock wait, `lockWaits` names the file the run holds on a numbered descriptor
|
|
27
|
+
* and the processes outside the run that also hold it open: open-handle evidence, not a kernel lock owner. When it
|
|
28
|
+
* named a process whose CPU time rose, `cpuRising` names it: a build inside the test, which is a defect and not a
|
|
29
|
+
* hang. CPU never restarts the bound, which counts finished tests and the progress running tests report.
|
|
27
30
|
* - `no-space`: the run failed and its output said a write found no space (ENOSPC). Whatever its tests then reported,
|
|
28
31
|
* they failed on the disk, not on their own: the verdict names the line that said so and, when the task held one,
|
|
29
32
|
* its RAM temp volume as it stood when that line arrived.
|
|
30
33
|
* - `queued`: the command failed before any admission after a live task PID was explicitly seen waiting in mbx, or
|
|
31
34
|
* while its reserved leaf's request waited on its run's mbx reservation endpoint. Its observed wait, and any public
|
|
32
35
|
* queue positions, describe admission pressure, never a wedged test.
|
|
36
|
+
* - `operational`: bounded-exec itself ended or refused the run for a cause its command did not make, and `cause` says
|
|
37
|
+
* which: its leaf's mbx reservation was refused (`reservation-refused`, naming the endpoint asked and the connect
|
|
38
|
+
* error's code, `ENOENT` when the endpoint's socket is gone), or the executor's own machinery failed (`executor`: a
|
|
39
|
+
* lost CPU grant or pool, a broken progress channel or runner lifecycle, leftovers it could not retire). The tests the
|
|
40
|
+
* report named are failures, never bounds, and the run never records exit 0.
|
|
33
41
|
* - `failed`: anything else that did not pass. That includes an assertion named in the report,
|
|
34
42
|
* or a non-zero exit with no report to say why.
|
|
35
43
|
*/
|
|
@@ -50,9 +58,29 @@ export type BoundedExecVerdict =
|
|
|
50
58
|
readonly tests: readonly string[];
|
|
51
59
|
/** Lock waits the snapshot named: holder evidence (open handles), not a kernel lock owner. */
|
|
52
60
|
readonly lockWaits: readonly LockWait[];
|
|
61
|
+
/** Processes of the run's tree whose CPU time rose while the bound counted down: a build inside the test. */
|
|
62
|
+
readonly cpuRising: readonly CpuRise[];
|
|
53
63
|
}
|
|
54
64
|
| ({ readonly outcome: 'no-space'; readonly exitCode: number; readonly tests: readonly string[] } & NoSpaceSighting)
|
|
55
|
-
| { readonly outcome: 'failed'; readonly exitCode: number; readonly tests: readonly string[] }
|
|
65
|
+
| { readonly outcome: 'failed'; readonly exitCode: number; readonly tests: readonly string[] }
|
|
66
|
+
| {
|
|
67
|
+
readonly outcome: 'operational';
|
|
68
|
+
readonly exitCode: number;
|
|
69
|
+
readonly tests: readonly string[];
|
|
70
|
+
readonly cause: OperationalCause;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
/** Why bounded-exec ended or refused a run whatever its command did (`operational`). */
|
|
74
|
+
export type OperationalCause =
|
|
75
|
+
| {
|
|
76
|
+
readonly kind: 'reservation-refused';
|
|
77
|
+
/** The run's mbx reservation endpoint the leaf asked. */
|
|
78
|
+
readonly endpoint: string;
|
|
79
|
+
/** The code of the error that ended the connection (`ENOENT`: the endpoint's socket is gone), else null. */
|
|
80
|
+
readonly errno: string | null;
|
|
81
|
+
readonly message: string;
|
|
82
|
+
}
|
|
83
|
+
| { readonly kind: 'executor'; readonly message: string };
|
|
56
84
|
|
|
57
85
|
/** Observed admission wait, supplied only while the command has never received a grant. */
|
|
58
86
|
export interface AdmissionQueueWait {
|
|
@@ -60,6 +88,8 @@ export interface AdmissionQueueWait {
|
|
|
60
88
|
readonly requests: readonly MbxQueuedRequest[];
|
|
61
89
|
}
|
|
62
90
|
const isQueuedVerdict = typia.createIs<Extract<BoundedExecVerdict, { outcome: 'queued' }>>();
|
|
91
|
+
const isOperationalVerdict = typia.createIs<Extract<BoundedExecVerdict, { outcome: 'operational' }>>();
|
|
92
|
+
const isCpuRising = typia.createIs<readonly CpuRise[]>();
|
|
63
93
|
|
|
64
94
|
/**
|
|
65
95
|
* The task's RAM temp volume, read when a line saying ENOSPC arrived: the writer's own lease and the space left on the
|
|
@@ -344,6 +374,8 @@ export interface RunEnd {
|
|
|
344
374
|
readonly noSpace: NoSpaceSighting | null;
|
|
345
375
|
/** The lock waits the expiry's snapshot named; empty when none, or when that capture did not complete. */
|
|
346
376
|
readonly lockWaits: readonly LockWait[];
|
|
377
|
+
/** The processes the expiry's snapshot saw gain CPU time; empty when none did, or when that capture did not finish. */
|
|
378
|
+
readonly cpuRising: readonly CpuRise[];
|
|
347
379
|
/** Explicit observed queue evidence before first admission; absent after any grant. */
|
|
348
380
|
readonly admissionQueue?: AdmissionQueueWait | null;
|
|
349
381
|
}
|
|
@@ -366,7 +398,14 @@ export function judgeRun(run: RunEnd): BoundedExecVerdict {
|
|
|
366
398
|
}
|
|
367
399
|
if (run.expiry?.kind === 'idle') {
|
|
368
400
|
const tests = [...new Set([...run.expiry.tests, ...(run.report?.cancelled ?? [])])];
|
|
369
|
-
return {
|
|
401
|
+
return {
|
|
402
|
+
outcome: 'wedged',
|
|
403
|
+
idleMs: run.expiry.limitMs,
|
|
404
|
+
elapsedMs: run.elapsedMs,
|
|
405
|
+
tests,
|
|
406
|
+
lockWaits: run.lockWaits,
|
|
407
|
+
cpuRising: run.cpuRising,
|
|
408
|
+
};
|
|
370
409
|
}
|
|
371
410
|
if (run.report !== null && run.report.failed.length > 0) {
|
|
372
411
|
return { outcome: 'failed', exitCode: run.exitCode, tests: run.report.failed };
|
|
@@ -391,13 +430,17 @@ export function judgeRun(run: RunEnd): BoundedExecVerdict {
|
|
|
391
430
|
}
|
|
392
431
|
|
|
393
432
|
/**
|
|
394
|
-
* A run that failed for a reason its command did not cause: its
|
|
395
|
-
* cleanly. It is
|
|
396
|
-
* reports are failures, never bounds.
|
|
433
|
+
* A run that failed for a reason its command did not cause (`OperationalCause`): its reservation was refused, its CPU
|
|
434
|
+
* grant was lost, or its sub-pool did not close cleanly. It is operational whatever the command exited with and
|
|
435
|
+
* whatever bound also fired, and the tests it reports are failures, never bounds. It never records exit 0.
|
|
397
436
|
*/
|
|
398
|
-
export function judgeOperationalFailure(
|
|
437
|
+
export function judgeOperationalFailure(
|
|
438
|
+
exitCode: number,
|
|
439
|
+
report: JunitFailures | null,
|
|
440
|
+
cause: OperationalCause,
|
|
441
|
+
): BoundedExecVerdict {
|
|
399
442
|
const tests = report === null ? [] : [...report.failed, ...report.timedOut, ...report.cancelled];
|
|
400
|
-
return { outcome: '
|
|
443
|
+
return { outcome: 'operational', exitCode: exitCode === 0 ? 1 : exitCode, tests, cause };
|
|
401
444
|
}
|
|
402
445
|
|
|
403
446
|
/** Where `task`'s verdict and report live: this checkout's own Nx workspace-data directory. */
|
|
@@ -556,15 +599,25 @@ function parseVerdict(value: unknown): BoundedExecVerdict {
|
|
|
556
599
|
}
|
|
557
600
|
break;
|
|
558
601
|
case 'wedged': {
|
|
559
|
-
// A record written before lock waits were named carries none.
|
|
602
|
+
// A record written before lock waits or CPU rises were named carries none.
|
|
560
603
|
const lockWaits = value.lockWaits === undefined ? [] : parseLockWaits(value.lockWaits);
|
|
604
|
+
const recorded: unknown = value.cpuRising;
|
|
605
|
+
const cpuRising = recorded === undefined ? [] : isCpuRising(recorded) ? recorded : null;
|
|
561
606
|
if (
|
|
562
607
|
typeof value.idleMs === 'number' &&
|
|
563
608
|
typeof value.elapsedMs === 'number' &&
|
|
564
609
|
isStrings(value.tests) &&
|
|
565
|
-
lockWaits !== null
|
|
610
|
+
lockWaits !== null &&
|
|
611
|
+
cpuRising !== null
|
|
566
612
|
) {
|
|
567
|
-
return {
|
|
613
|
+
return {
|
|
614
|
+
outcome: 'wedged',
|
|
615
|
+
idleMs: value.idleMs,
|
|
616
|
+
elapsedMs: value.elapsedMs,
|
|
617
|
+
tests: value.tests,
|
|
618
|
+
lockWaits,
|
|
619
|
+
cpuRising,
|
|
620
|
+
};
|
|
568
621
|
}
|
|
569
622
|
break;
|
|
570
623
|
}
|
|
@@ -581,6 +634,9 @@ function parseVerdict(value: unknown): BoundedExecVerdict {
|
|
|
581
634
|
}
|
|
582
635
|
break;
|
|
583
636
|
}
|
|
637
|
+
case 'operational':
|
|
638
|
+
if (isOperationalVerdict(value)) return value;
|
|
639
|
+
break;
|
|
584
640
|
case 'failed':
|
|
585
641
|
if (typeof value.exitCode === 'number' && isStrings(value.tests)) {
|
|
586
642
|
return { outcome: 'failed', exitCode: value.exitCode, tests: value.tests };
|
|
@@ -714,20 +770,25 @@ export function describeVerdict(verdict: BoundedExecVerdict): string {
|
|
|
714
770
|
: verdict.bound === 'fixture'
|
|
715
771
|
? `its fixture outlived its ${verdict.limitMs}ms bound (${verdict.elapsedMs}ms)`
|
|
716
772
|
: `outlived its ${verdict.limitMs}ms bound (${verdict.elapsedMs}ms)`;
|
|
717
|
-
case 'wedged':
|
|
718
|
-
|
|
773
|
+
case 'wedged': {
|
|
774
|
+
const locks = verdict.lockWaits.map(
|
|
775
|
+
(wait) => `; waiting on a lock: ${wait.path} held open by ${describeHolders(wait.holders)}`,
|
|
776
|
+
);
|
|
777
|
+
const cpu = verdict.cpuRising.length > 0 ? [`; ${CPU_RISING_CLASS}: ${describeCpuRises(verdict.cpuRising)}`] : [];
|
|
778
|
+
const running =
|
|
719
779
|
verdict.tests.length > 0
|
|
720
780
|
? `still running: ${verdict.tests.join(', ')}`
|
|
721
|
-
: 'the runner named no test still running'
|
|
722
|
-
}${verdict.
|
|
723
|
-
|
|
724
|
-
.join('')}`;
|
|
781
|
+
: 'the runner named no test still running';
|
|
782
|
+
return `wedged: no test finished or reported progress for ${verdict.idleMs}ms after ${verdict.elapsedMs}ms; ${running}${[...locks, ...cpu].join('')}`;
|
|
783
|
+
}
|
|
725
784
|
case 'no-space':
|
|
726
785
|
return `${describeNoSpace(verdict)}${verdict.tests.length > 0 ? `; the tests that then failed: ${verdict.tests.join(', ')}` : ''}`;
|
|
727
786
|
case 'failed':
|
|
728
787
|
return verdict.tests.length > 0
|
|
729
788
|
? `failed: ${verdict.tests.join(', ')}`
|
|
730
789
|
: `exited ${verdict.exitCode} with no report naming a timeout`;
|
|
790
|
+
case 'operational':
|
|
791
|
+
return `${verdict.cause.message}${verdict.tests.length > 0 ? `; the tests that failed: ${verdict.tests.join(', ')}` : ''}`;
|
|
731
792
|
}
|
|
732
793
|
}
|
|
733
794
|
|