@smoothbricks/nx-plugin 0.6.0 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -56
- package/dist/.build-inputs.json +16 -13
- package/dist/cargo-retention-hook.d.ts +8 -1
- package/dist/cargo-retention-hook.d.ts.map +1 -1
- package/dist/cargo-retention-hook.js +12 -1
- package/dist/ensure-built-receipt.d.ts +9 -0
- package/dist/ensure-built-receipt.d.ts.map +1 -1
- package/dist/ensure-built-receipt.js +42 -1
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +28 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.js +20 -0
- package/dist/executors/bounded-exec/cpu-pool.d.ts +4 -4
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +7 -6
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +20 -2
- package/dist/executors/bounded-exec/gateway-lease.d.ts +3 -2
- package/dist/executors/bounded-exec/gateway-lease.d.ts.map +1 -1
- package/dist/executors/bounded-exec/gateway-lease.js +23 -3
- package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
- package/dist/executors/bounded-exec/hang-diagnostics.js +25 -1
- package/dist/executors/bounded-exec/lease-wait.d.ts +23 -2
- package/dist/executors/bounded-exec/lease-wait.d.ts.map +1 -1
- package/dist/executors/bounded-exec/lease-wait.js +15 -11
- package/dist/executors/bounded-exec/mbx-pool.d.ts +31 -0
- package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -0
- package/dist/executors/bounded-exec/mbx-pool.js +24 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts +9 -3
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +109 -12
- package/dist/executors/bounded-exec/verdict.d.ts +8 -0
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +1 -1
- package/dist/executors/typescript-emit/emit-overrides.d.ts +14 -0
- package/dist/executors/typescript-emit/emit-overrides.d.ts.map +1 -0
- package/dist/executors/typescript-emit/emit-overrides.js +24 -0
- package/dist/executors/typescript-emit/executor.d.ts.map +1 -1
- package/dist/executors/typescript-emit/executor.js +1 -19
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +6 -0
- package/dist/managed-files/publish-workflow.js +14 -11
- package/dist/producer-receipts.d.ts.map +1 -1
- package/dist/producer-receipts.js +441 -1
- package/managed/raw/envrc +15 -0
- package/managed/raw/tooling/direnv/developer-links.ts +12 -1
- package/managed/raw/tooling/direnv/devenv.smoo.nix +21 -8
- package/managed/raw/tooling/direnv/linked-snapshots.ts +50 -13
- package/managed/raw/tooling/direnv/setup-environment.ts +23 -11
- package/managed/raw/tooling/git-hooks/pre-push.sh +4 -1
- package/managed/raw/tooling/git-reword-commit +151 -0
- package/package.json +22 -5
- package/src/cargo-retention-hook.ts +12 -1
- package/src/ensure-built-receipt.ts +41 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +43 -0
- package/src/executors/bounded-exec/cpu-pool.ts +7 -6
- package/src/executors/bounded-exec/executor.ts +20 -2
- package/src/executors/bounded-exec/gateway-lease.ts +17 -7
- package/src/executors/bounded-exec/hang-diagnostics.ts +28 -1
- package/src/executors/bounded-exec/lease-wait.ts +39 -12
- package/src/executors/bounded-exec/mbx-pool.ts +48 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +117 -15
- package/src/executors/bounded-exec/verdict.ts +8 -1
- package/src/executors/typescript-emit/emit-overrides.ts +24 -0
- package/src/executors/typescript-emit/executor.ts +1 -20
- package/src/managed-files/files.ts +6 -0
- package/src/managed-files/publish-workflow.ts +20 -15
- package/src/producer-receipts.ts +28 -0
- package/test-timings.json +120 -118
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { connect } from 'node:net';
|
|
2
2
|
import typia from 'typia';
|
|
3
|
-
import { type CpuLeaseProgress, CpuLeaseWait } from './lease-wait.js';
|
|
3
|
+
import { type CpuLeaseProgress, type CpuLeaseQueued, CpuLeaseWait, type CpuWaitRow } from './lease-wait.js';
|
|
4
4
|
|
|
5
5
|
const isCpuProgress = typia.createEquals<CpuLeaseProgress>();
|
|
6
|
+
const isCpuQueued = typia.createEquals<CpuLeaseQueued>();
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* The client half of a lease held on a unix socket: the gateway's host disk-lifecycle lease
|
|
@@ -35,8 +36,8 @@ export interface LeaseBounds {
|
|
|
35
36
|
*/
|
|
36
37
|
export type UnleasedCause = 'absent' | 'predates' | 'predates-field' | 'other';
|
|
37
38
|
|
|
38
|
-
/** A refusal: the lease is not held, and why. */
|
|
39
|
-
export type Unleased = { granted: false; cause: UnleasedCause; reason: string; code?: string };
|
|
39
|
+
/** A refusal: the lease is not held, and why. A CPU wait that expired carries its typed row. */
|
|
40
|
+
export type Unleased = { granted: false; cause: UnleasedCause; reason: string; code?: string; wait?: CpuWaitRow };
|
|
40
41
|
|
|
41
42
|
/**
|
|
42
43
|
* A granted lease with its `granted` answer, held until released, or why there is none. `closed`
|
|
@@ -89,8 +90,14 @@ export function holdLease(
|
|
|
89
90
|
}
|
|
90
91
|
resolve(outcome);
|
|
91
92
|
};
|
|
92
|
-
const unleased = (reason: string, cause: UnleasedCause = 'other', code?: string) =>
|
|
93
|
-
settle({
|
|
93
|
+
const unleased = (reason: string, cause: UnleasedCause = 'other', code?: string, wait?: CpuWaitRow) =>
|
|
94
|
+
settle({
|
|
95
|
+
granted: false,
|
|
96
|
+
cause,
|
|
97
|
+
reason,
|
|
98
|
+
...(code === undefined ? {} : { code }),
|
|
99
|
+
...(wait === undefined ? {} : { wait }),
|
|
100
|
+
});
|
|
94
101
|
const wait = (ms: number, onExpiry: () => void) => {
|
|
95
102
|
clearTimeout(timer);
|
|
96
103
|
timer = setTimeout(onExpiry, ms);
|
|
@@ -104,10 +111,12 @@ export function holdLease(
|
|
|
104
111
|
armCpu();
|
|
105
112
|
return;
|
|
106
113
|
}
|
|
114
|
+
const peer = failure.server === null ? socket : `${socket} (pid ${failure.server})`;
|
|
107
115
|
unleased(
|
|
108
|
-
`${failure.code}: cpu-pool: the lease endpoint ${
|
|
116
|
+
`${failure.code}: cpu-pool: the lease endpoint ${peer} exceeded ${failure.boundMs} ms (waited ${failure.elapsedMs} ms; last progress: ${failure.last})`,
|
|
109
117
|
'other',
|
|
110
118
|
failure.code,
|
|
119
|
+
{ ...failure, socket },
|
|
111
120
|
);
|
|
112
121
|
});
|
|
113
122
|
};
|
|
@@ -161,7 +170,8 @@ export function holdLease(
|
|
|
161
170
|
state = 'queued';
|
|
162
171
|
onQueued?.();
|
|
163
172
|
if (bounds instanceof CpuLeaseWait) {
|
|
164
|
-
|
|
173
|
+
// A pool from before its acknowledgement named its process names none.
|
|
174
|
+
bounds.queued(isCpuQueued(parsed) ? parsed.server : null);
|
|
165
175
|
armCpu();
|
|
166
176
|
} else if (bounds.grantMs === null) {
|
|
167
177
|
clearTimeout(timer);
|
|
@@ -25,6 +25,8 @@ const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,lstart=,command=';
|
|
|
25
25
|
*/
|
|
26
26
|
const MAX_SAMPLED_LEAVES = 4;
|
|
27
27
|
const SAMPLE_SECONDS = 1;
|
|
28
|
+
/** How long a process in an uninterruptible kernel wait has its file-system calls traced. */
|
|
29
|
+
const FS_USAGE_SECONDS = 5;
|
|
28
30
|
/**
|
|
29
31
|
* A process census's limit. The stack samples run until the closing census is this far from the window's end.
|
|
30
32
|
*/
|
|
@@ -236,6 +238,31 @@ export function captureHangDiagnostics(
|
|
|
236
238
|
}),
|
|
237
239
|
);
|
|
238
240
|
};
|
|
241
|
+
/**
|
|
242
|
+
* A process in an uninterruptible kernel wait (state U) is in a file-system or device call no signal ends, and its
|
|
243
|
+
* stack stops at the system call. Its file-system calls over a few seconds (fs_usage) name the volume and files it
|
|
244
|
+
* waits on, which tells a fixture doing its I/O on a contended shared volume from one serialised behind one APFS
|
|
245
|
+
* operation. Bounded like the stack samples; one fs_usage sudo refuses is named as no trace.
|
|
246
|
+
*/
|
|
247
|
+
const kernelWaits = async (tree: readonly ObservedProcess[]): Promise<void> => {
|
|
248
|
+
const waiting = tree.filter((entry) => entry.stat.startsWith('U')).slice(0, MAX_SAMPLED_LEAVES);
|
|
249
|
+
const limitMs = endsAtMs - Date.now() - CENSUS_LIMIT_MS;
|
|
250
|
+
await Promise.all(
|
|
251
|
+
waiting.map(async ({ pid }) => {
|
|
252
|
+
const name = `fs_usage pid ${pid}`;
|
|
253
|
+
const args = ['-n', 'fs_usage', '-w', '-t', String(FS_USAGE_SECONDS), '-f', 'filesys', String(pid)];
|
|
254
|
+
const result = await probe(name, 'sudo', args, limitMs);
|
|
255
|
+
if (finished) return;
|
|
256
|
+
if (result.stdout.trim() !== '') {
|
|
257
|
+
lines.push(`${name} (${FS_USAGE_SECONDS}s of file-system calls):\n${result.stdout.trimEnd()}`);
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
lines.push(
|
|
261
|
+
`${name}: no trace; sudo -n fs_usage gave none, so the files this process waits on are not observed.`,
|
|
262
|
+
);
|
|
263
|
+
}),
|
|
264
|
+
);
|
|
265
|
+
};
|
|
239
266
|
const collect = async (): Promise<void> => {
|
|
240
267
|
const before = await census('ps before');
|
|
241
268
|
if (finished) return;
|
|
@@ -294,7 +321,7 @@ export function captureHangDiagnostics(
|
|
|
294
321
|
}
|
|
295
322
|
}
|
|
296
323
|
};
|
|
297
|
-
await Promise.all([openFiles(), stackSamples(before)]);
|
|
324
|
+
await Promise.all([openFiles(), stackSamples(before), kernelWaits(before)]);
|
|
298
325
|
if (finished) return;
|
|
299
326
|
} else {
|
|
300
327
|
lines.push(`Open-file census not provided on ${source.platform}; observations below name processes only.`);
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { tags } from 'typia';
|
|
2
|
+
|
|
1
3
|
/** Peer wait classes from specs/cowshed/05_gateway.md, "Deadlines". */
|
|
2
4
|
export const CONTROL_MS = 5_000;
|
|
3
5
|
export const HEARTBEAT_MS = 5_000;
|
|
@@ -9,7 +11,19 @@ export interface CpuLeaseProgress {
|
|
|
9
11
|
progress: number;
|
|
10
12
|
}
|
|
11
13
|
|
|
12
|
-
|
|
14
|
+
/**
|
|
15
|
+
* The CPU pool's queue acknowledgement, shared by its server and lease client. `server` is the pid of the process the
|
|
16
|
+
* pool runs in, so a wait that falls silent names the process to look at.
|
|
17
|
+
*/
|
|
18
|
+
export interface CpuLeaseQueued {
|
|
19
|
+
ok: true;
|
|
20
|
+
lease: 'queued';
|
|
21
|
+
server: number & tags.Type<'uint32'>;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
type CpuWaitState =
|
|
25
|
+
| { kind: 'sending' }
|
|
26
|
+
| { kind: 'waiting'; at: number; last: string; sequence: number; server: number | null };
|
|
13
27
|
|
|
14
28
|
/** A failed wait, not a failed command: expiry ends this client's admission only. */
|
|
15
29
|
export interface CpuWaitFailure {
|
|
@@ -17,6 +31,15 @@ export interface CpuWaitFailure {
|
|
|
17
31
|
boundMs: number;
|
|
18
32
|
elapsedMs: number;
|
|
19
33
|
last: string;
|
|
34
|
+
/** The last advancing heartbeat sequence the client read; 0 before the first. */
|
|
35
|
+
sequence: number;
|
|
36
|
+
/** The pool's pid as its queue acknowledgement named it; null before one, or from a pool that names none. */
|
|
37
|
+
server: number | null;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** An expired CPU wait as its client keeps it: the failure, and the socket of the pool it waited on. */
|
|
41
|
+
export interface CpuWaitRow extends CpuWaitFailure {
|
|
42
|
+
socket: string;
|
|
20
43
|
}
|
|
21
44
|
|
|
22
45
|
/** Pure CPU admission policy; its injected monotonic clock performs no I/O or scheduling. */
|
|
@@ -35,15 +58,18 @@ export class CpuLeaseWait {
|
|
|
35
58
|
sent(): void {
|
|
36
59
|
this.looked = this.now();
|
|
37
60
|
if (this.state.kind === 'sending') {
|
|
38
|
-
this.state = { kind: 'waiting', at: this.now(), last: 'request sent', sequence: 0 };
|
|
61
|
+
this.state = { kind: 'waiting', at: this.now(), last: 'request sent', sequence: 0, server: null };
|
|
39
62
|
}
|
|
40
63
|
}
|
|
41
64
|
|
|
42
|
-
/**
|
|
43
|
-
|
|
65
|
+
/**
|
|
66
|
+
* A queue entry proves actor progress, not a deadline for returning a token. It names the pool's pid when the pool
|
|
67
|
+
* says it.
|
|
68
|
+
*/
|
|
69
|
+
queued(server: number | null): void {
|
|
44
70
|
this.sent();
|
|
45
71
|
if (this.state.kind === 'waiting') {
|
|
46
|
-
this.state = { ...this.state, at: this.now(), last: 'queued' };
|
|
72
|
+
this.state = { ...this.state, at: this.now(), last: 'queued', server };
|
|
47
73
|
}
|
|
48
74
|
}
|
|
49
75
|
|
|
@@ -52,7 +78,7 @@ export class CpuLeaseWait {
|
|
|
52
78
|
this.sent();
|
|
53
79
|
if (this.state.kind !== 'waiting' || !Number.isSafeInteger(sequence) || sequence <= this.state.sequence)
|
|
54
80
|
return false;
|
|
55
|
-
this.state = {
|
|
81
|
+
this.state = { ...this.state, at: this.now(), last: `heartbeat ${sequence}`, sequence };
|
|
56
82
|
return true;
|
|
57
83
|
}
|
|
58
84
|
|
|
@@ -79,11 +105,12 @@ export class CpuLeaseWait {
|
|
|
79
105
|
/** No total queue age exists: an advancing heartbeat keeps the wait alive indefinitely. */
|
|
80
106
|
expired(): CpuWaitFailure | null {
|
|
81
107
|
if (this.remainingMs() > 0) return null;
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
elapsedMs:
|
|
86
|
-
|
|
87
|
-
};
|
|
108
|
+
const elapsedMs = this.now() - this.started;
|
|
109
|
+
if (this.state.kind === 'sending') {
|
|
110
|
+
const last = 'connecting or sending request';
|
|
111
|
+
return { code: 'TIMEOUT', boundMs: CONTROL_MS, elapsedMs, last, sequence: 0, server: null };
|
|
112
|
+
}
|
|
113
|
+
const { last, sequence, server } = this.state;
|
|
114
|
+
return { code: 'SILENT', boundMs: HEARTBEAT_SILENCE_MS, elapsedMs, last, sequence, server };
|
|
88
115
|
}
|
|
89
116
|
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { RunnerKind } from './runner-sizing.js';
|
|
2
|
+
import { shellQuote } from './verdict.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* mbx's machine-wide pool as a leaf runner meets it (specs/cowshed/05_gateway.md, "Host scheduling", "Parallelism inside
|
|
6
|
+
* a job"). With the mbx capability the canonical environment names the one pool (`MBX_SCHEDULER=1` beside the shared
|
|
7
|
+
* `MBX_CACHE_DIR`) and the job's priority in it; that pool is the host's CPU and memory ledger, so no Nx run or leaf
|
|
8
|
+
* serves a pool of its own beside it.
|
|
9
|
+
*/
|
|
10
|
+
export interface MbxPool {
|
|
11
|
+
readonly priority: 'normal' | 'low';
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** The pool the environment names, or `null` where the project has no mbx and cowshed's own pools apply unchanged. */
|
|
15
|
+
export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | null {
|
|
16
|
+
if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR) {
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
return { priority: environment.MBX_SCHEDULER_PRIORITY === 'normal' ? 'normal' : 'low' };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* How a leaf enters the pool once: a Cargo leaf runs through mbx, which admits its compiler and supported test work
|
|
24
|
+
* itself, so wrapping it would charge it twice; any other leaf holds `cpus` of the pool for its whole run through
|
|
25
|
+
* `mbx reserve`, `cpus` being the job's own count.
|
|
26
|
+
*/
|
|
27
|
+
export type MbxLeaf =
|
|
28
|
+
| { readonly kind: 'cargo' }
|
|
29
|
+
| { readonly kind: 'reserve'; readonly cpus: number; readonly priority: 'normal' | 'low' };
|
|
30
|
+
|
|
31
|
+
const CARGO_LEAVES: ReadonlySet<RunnerKind> = new Set<RunnerKind>(['cargo', 'cargo-mutants', 'nextest']);
|
|
32
|
+
|
|
33
|
+
export function mbxLeaf(pool: MbxPool, kind: RunnerKind, want: number): MbxLeaf {
|
|
34
|
+
return CARGO_LEAVES.has(kind)
|
|
35
|
+
? { kind: 'cargo' }
|
|
36
|
+
: { kind: 'reserve', cpus: Math.max(1, want), priority: pool.priority };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* `command` as the shell runs it under its leaf's admission. `mbx` is the canonical environment's, whose realpath
|
|
41
|
+
* cowshed's admission proved to be the host's pinned mbx.
|
|
42
|
+
*/
|
|
43
|
+
export function withMbxLeaf(command: string, leaf: MbxLeaf): string {
|
|
44
|
+
if (leaf.kind === 'cargo') {
|
|
45
|
+
return command;
|
|
46
|
+
}
|
|
47
|
+
return `mbx reserve --cpus ${leaf.cpus} --priority ${leaf.priority} -- /bin/sh -c ${shellQuote(command)}`;
|
|
48
|
+
}
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import { availableParallelism } from 'node:os';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
import { Worker } from 'node:worker_threads';
|
|
2
4
|
import { logger } from 'nx/src/devkit-exports.js';
|
|
3
5
|
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
4
|
-
import { type
|
|
6
|
+
import { type CloseOutcome, type CpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
7
|
+
import type { CpuPoolThreadData, CpuPoolThreadReport } from './cpu-pool-thread.js';
|
|
8
|
+
import { mbxPool } from './mbx-pool.js';
|
|
5
9
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
6
10
|
|
|
7
11
|
/**
|
|
@@ -18,42 +22,140 @@ import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
|
18
22
|
* A run that inherits a pool (an enclosing executor's) takes from that one and serves none. Otherwise its pool holds
|
|
19
23
|
* one token per core, or the inherited positive `BOUNDED_EXEC_CPU_TOKENS` when that is fewer: inside a cowshed job
|
|
20
24
|
* that count is the job's own, so the job's `--want` bounds the run.
|
|
25
|
+
*
|
|
26
|
+
* The pool runs on a thread of its own (`cpu-pool-thread.ts`). The process that serves it, the Nx daemon when installed
|
|
27
|
+
* plugins load in it or the Nx CLI without one, hashes the run's tasks and builds its project graph in synchronous calls
|
|
28
|
+
* on its main thread. In a gate on 2026-10-28 one `hashPlans` ran for 14.2 s, and sixteen runners queued on a pool on
|
|
29
|
+
* that thread failed `SILENT` together while it was alive (05_gateway.md, "Deadlines"). The thread's event loop runs the
|
|
30
|
+
* pool alone; this module starts it, publishes the environment it reports and closes it.
|
|
21
31
|
*/
|
|
22
32
|
|
|
23
33
|
/** The pools this process serves, by Nx run id, until that run's `postTasksExecution`. */
|
|
24
34
|
const pools = new Map<string, CpuPool>();
|
|
25
35
|
|
|
26
36
|
/**
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
37
|
+
* The pool thread's module, in the form this one was loaded in: compiled JavaScript when Nx loads the plugin, its
|
|
38
|
+
* TypeScript source under Bun. A Bun worker starts without its parent's preloads, so from source the thread takes the
|
|
39
|
+
* transform preload this package's source needs, the one its bunfig names; compiled, it needs none.
|
|
40
|
+
*/
|
|
41
|
+
const FROM_SOURCE = import.meta.url.endsWith('.ts');
|
|
42
|
+
const THREAD_ENTRY = new URL(FROM_SOURCE ? './cpu-pool-thread.ts' : './cpu-pool-thread.js', import.meta.url);
|
|
43
|
+
const THREAD_PRELOAD: { readonly preload?: string[] } = FROM_SOURCE
|
|
44
|
+
? { preload: [fileURLToPath(import.meta.resolve('@smoothbricks/validation/bun/preload'))] }
|
|
45
|
+
: {};
|
|
46
|
+
|
|
47
|
+
type ServedPool = { ok: true; pool: CpuPool } | { ok: false; error: string };
|
|
48
|
+
|
|
49
|
+
/** Where a pool thread is, and who waits on it: its start, its close, or nobody. */
|
|
50
|
+
type ThreadState =
|
|
51
|
+
| { readonly kind: 'starting'; readonly served: (served: ServedPool) => void }
|
|
52
|
+
| { readonly kind: 'serving' }
|
|
53
|
+
| { readonly kind: 'closing'; readonly closed: (outcome: CloseOutcome) => void }
|
|
54
|
+
| { readonly kind: 'ended'; readonly outcome: CloseOutcome };
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* The count of the pool a run is served, or null when it is served none. A run that inherits a pool (an enclosing
|
|
58
|
+
* executor's) takes from that one; with mbx, its pool is the host's one ledger, and a run pool beside it would count
|
|
59
|
+
* every leaf twice. Otherwise one token per core, or the inherited positive count when that is fewer.
|
|
60
|
+
*/
|
|
61
|
+
export function runPoolTokens(env: Readonly<NodeJS.ProcessEnv>, cores: number): number | null {
|
|
62
|
+
if (readInheritedCpuPool(env) !== null || mbxPool(env) !== null) {
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
const inherited = parseInheritedTokens(env[CPU_TOKENS_ENV]);
|
|
66
|
+
return inherited === null ? cores : Math.min(inherited, cores);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Serve `context`'s run a pool of `runPoolTokens` and publish it in `env`, which every task inherits, unless the run
|
|
71
|
+
* is served none. A pool that cannot be made is said, and the run's tasks are then sized only by the ceilings they
|
|
72
|
+
* inherit.
|
|
30
73
|
*/
|
|
31
74
|
export async function serveRunCpuPool(
|
|
32
75
|
context: Pick<PreTasksExecutionContext, 'id'>,
|
|
33
76
|
env: NodeJS.ProcessEnv = process.env,
|
|
34
77
|
cores: number = availableParallelism(),
|
|
35
78
|
): Promise<void> {
|
|
36
|
-
|
|
79
|
+
const tokens = runPoolTokens(env, cores);
|
|
80
|
+
if (tokens === null) {
|
|
37
81
|
return;
|
|
38
82
|
}
|
|
39
|
-
const
|
|
40
|
-
|
|
41
|
-
const made = await createCpuPool(
|
|
42
|
-
inherited === null ? cores : Math.min(inherited, cores),
|
|
43
|
-
new Promise<void>(() => {}),
|
|
44
|
-
);
|
|
45
|
-
if (!made.ok) {
|
|
83
|
+
const served = await servePoolThread(tokens);
|
|
84
|
+
if (!served.ok) {
|
|
46
85
|
logger.warn(
|
|
47
|
-
`bounded-exec: this run's CPU pool failed (${
|
|
86
|
+
`bounded-exec: this run's CPU pool failed (${served.error}); its tasks are sized only by the ceilings they inherit`,
|
|
48
87
|
);
|
|
49
88
|
return;
|
|
50
89
|
}
|
|
51
|
-
pools.set(context.id,
|
|
52
|
-
for (const [name, value] of Object.entries(
|
|
90
|
+
pools.set(context.id, served.pool);
|
|
91
|
+
for (const [name, value] of Object.entries(served.pool.environment)) {
|
|
53
92
|
env[name] = value;
|
|
54
93
|
}
|
|
55
94
|
}
|
|
56
95
|
|
|
96
|
+
/**
|
|
97
|
+
* Start a pool of `tokens` on a thread of its own, and wait until it serves or says why it cannot. The first `close()`
|
|
98
|
+
* closes the pool and ends the thread; every call returns that close's outcome. A thread that throws or exits before it
|
|
99
|
+
* has said how its pool closed fails whichever wait is open, a later close returns that failure, and a thread lost while
|
|
100
|
+
* its pool serves is said at once.
|
|
101
|
+
*/
|
|
102
|
+
function servePoolThread(tokens: number): Promise<ServedPool> {
|
|
103
|
+
const thread = new Worker(THREAD_ENTRY, { workerData: { tokens } satisfies CpuPoolThreadData, ...THREAD_PRELOAD });
|
|
104
|
+
return new Promise<ServedPool>((served) => {
|
|
105
|
+
let state: ThreadState = { kind: 'starting', served };
|
|
106
|
+
let closing: Promise<CloseOutcome> | undefined;
|
|
107
|
+
// The pool is done with its thread once it has said how it ended: nothing of the pool's is left to run there.
|
|
108
|
+
const finish = (outcome: CloseOutcome): ThreadState => {
|
|
109
|
+
const was = state;
|
|
110
|
+
state = { kind: 'ended', outcome };
|
|
111
|
+
void thread.terminate();
|
|
112
|
+
return was;
|
|
113
|
+
};
|
|
114
|
+
const lose = (why: string): void => {
|
|
115
|
+
if (state.kind === 'ended') {
|
|
116
|
+
// The exit that finishing the thread caused.
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
const error = `the CPU pool's thread ${why}`;
|
|
120
|
+
const was = finish({ ok: false, error });
|
|
121
|
+
if (was.kind === 'starting') {
|
|
122
|
+
was.served({ ok: false, error });
|
|
123
|
+
} else if (was.kind === 'closing') {
|
|
124
|
+
was.closed({ ok: false, error });
|
|
125
|
+
} else {
|
|
126
|
+
logger.warn(`bounded-exec: this run's CPU pool stopped serving: ${error}`);
|
|
127
|
+
}
|
|
128
|
+
};
|
|
129
|
+
const close = (): Promise<CloseOutcome> => {
|
|
130
|
+
closing ??=
|
|
131
|
+
state.kind === 'ended'
|
|
132
|
+
? Promise.resolve(state.outcome)
|
|
133
|
+
: new Promise<CloseOutcome>((closed) => {
|
|
134
|
+
state = { kind: 'closing', closed };
|
|
135
|
+
thread.postMessage(null);
|
|
136
|
+
});
|
|
137
|
+
return closing;
|
|
138
|
+
};
|
|
139
|
+
thread.on('message', (report: CpuPoolThreadReport) => {
|
|
140
|
+
const now = state;
|
|
141
|
+
if (now.kind === 'starting' && report.kind === 'serving') {
|
|
142
|
+
state = { kind: 'serving' };
|
|
143
|
+
now.served({ ok: true, pool: { environment: report.environment, close } });
|
|
144
|
+
} else if (now.kind === 'starting' && report.kind === 'failed') {
|
|
145
|
+
finish({ ok: false, error: report.error });
|
|
146
|
+
now.served({ ok: false, error: report.error });
|
|
147
|
+
} else if (now.kind === 'closing' && report.kind === 'closed') {
|
|
148
|
+
finish(report.outcome);
|
|
149
|
+
now.closed(report.outcome);
|
|
150
|
+
} else {
|
|
151
|
+
lose(`said ${report.kind} while ${now.kind}`);
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
thread.on('error', (error) => lose(`failed: ${error instanceof Error ? error.message : String(error)}`));
|
|
155
|
+
thread.on('exit', (code) => lose(`exited with code ${code}`));
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
|
|
57
159
|
/** Close the pool `context`'s run was served, if it was served one. */
|
|
58
160
|
export async function closeRunCpuPool(context: Pick<PostTasksExecutionContext, 'id'>): Promise<void> {
|
|
59
161
|
const pool = pools.get(context.id);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
|
+
import type { CpuWaitRow } from './lease-wait.js';
|
|
3
4
|
import { describeHolders, type LockWait, type Opener } from './lock-wait.js';
|
|
4
5
|
|
|
5
6
|
/**
|
|
@@ -79,6 +80,12 @@ export interface BoundedExecRecord {
|
|
|
79
80
|
readonly want: number;
|
|
80
81
|
readonly waitedMs: number;
|
|
81
82
|
};
|
|
83
|
+
/**
|
|
84
|
+
* The parent CPU pool's wait that expired and refused the run: its code, bound and elapsed time, the last heartbeat
|
|
85
|
+
* sequence it read, and the pool's socket and pid (specs/cowshed/05_gateway.md, "Deadlines"). Absent when no wait
|
|
86
|
+
* expired; written for the reader of a refused run, never read back.
|
|
87
|
+
*/
|
|
88
|
+
readonly cpuWait?: CpuWaitRow;
|
|
82
89
|
}
|
|
83
90
|
|
|
84
91
|
/** The Nx task a run executes as. Nx gives an executor no task id, only its parts. */
|
|
@@ -350,7 +357,7 @@ export async function withJunitReport(command: string, directory: string): Promi
|
|
|
350
357
|
return command.replace(NEXTEST_RUN, () => `nextest run --tool-config-file ${shellQuote(`smoo-report:${config}`)}`);
|
|
351
358
|
}
|
|
352
359
|
|
|
353
|
-
function shellQuote(value: string): string {
|
|
360
|
+
export function shellQuote(value: string): string {
|
|
354
361
|
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
355
362
|
}
|
|
356
363
|
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the emit changes about the project's own options: JavaScript and declarations from one non-incremental program.
|
|
3
|
+
* They ride the command line onto the project's tsconfig. A synthesized config would have to sit beside that tsconfig,
|
|
4
|
+
* because `${configDir}` and `paths` anchor to the directory of the config that consumes them, and a transient file there
|
|
5
|
+
* changes the directory listing a concurrent ttsc transform of another project proves while it compiles; that
|
|
6
|
+
* transform then refuses with `graph/accessible-entries-changed`.
|
|
7
|
+
*
|
|
8
|
+
* The one definition of how a package's TypeScript dist is emitted: the typescript-emit executor passes these, and so
|
|
9
|
+
* does every bootstrap that emits a dist before Nx can run (tooling/ttsc-emit.ts, the shell entry). A bootstrap that
|
|
10
|
+
* emitted incrementally wrote the tsconfig's tsBuildInfoFile into dist, a file no Nx output produces, and a release
|
|
11
|
+
* refused the verified dist at its merge boundary for it.
|
|
12
|
+
*/
|
|
13
|
+
export const EMIT_OVERRIDES: readonly string[] = [
|
|
14
|
+
'--composite',
|
|
15
|
+
'false',
|
|
16
|
+
'--declaration',
|
|
17
|
+
'true',
|
|
18
|
+
'--declarationMap',
|
|
19
|
+
'true',
|
|
20
|
+
'--emitDeclarationOnly',
|
|
21
|
+
'false',
|
|
22
|
+
'--incremental',
|
|
23
|
+
'false',
|
|
24
|
+
];
|
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
writeBuildInputs,
|
|
16
16
|
} from '@smoothbricks/build-inputs';
|
|
17
17
|
import { readJsonFile } from 'nx/src/devkit-exports.js';
|
|
18
|
+
import { EMIT_OVERRIDES } from './emit-overrides.js';
|
|
18
19
|
import type { TypeScriptEmitOptions } from './schema.js';
|
|
19
20
|
|
|
20
21
|
interface TypeScriptEmitContext {
|
|
@@ -39,26 +40,6 @@ export interface CompilerInvocation {
|
|
|
39
40
|
|
|
40
41
|
export type CompilerRunner = (invocation: CompilerInvocation) => Promise<boolean>;
|
|
41
42
|
|
|
42
|
-
/**
|
|
43
|
-
* What the emit changes about the project's own options: JavaScript and declarations from one non-incremental program.
|
|
44
|
-
* They ride the command line onto the project's tsconfig. A synthesized config would have to sit beside that tsconfig,
|
|
45
|
-
* because `${configDir}` and `paths` anchor to the directory of the config that consumes them, and a transient file there
|
|
46
|
-
* changes the directory listing a concurrent ttsc transform of another project proves while it compiles; that
|
|
47
|
-
* transform then refuses with `graph/accessible-entries-changed`.
|
|
48
|
-
*/
|
|
49
|
-
const EMIT_OVERRIDES = [
|
|
50
|
-
'--composite',
|
|
51
|
-
'false',
|
|
52
|
-
'--declaration',
|
|
53
|
-
'true',
|
|
54
|
-
'--declarationMap',
|
|
55
|
-
'true',
|
|
56
|
-
'--emitDeclarationOnly',
|
|
57
|
-
'false',
|
|
58
|
-
'--incremental',
|
|
59
|
-
'false',
|
|
60
|
-
];
|
|
61
|
-
|
|
62
43
|
export default function typescriptEmitExecutor(
|
|
63
44
|
options: TypeScriptEmitOptions,
|
|
64
45
|
context: TypeScriptEmitContext,
|
|
@@ -211,6 +211,12 @@ const managedFiles: ManagedFileDescriptor[] = [
|
|
|
211
211
|
target: 'tooling/git-hooks/pre-push.sh',
|
|
212
212
|
executable: true,
|
|
213
213
|
},
|
|
214
|
+
{
|
|
215
|
+
kind: 'raw',
|
|
216
|
+
source: 'tooling/git-reword-commit',
|
|
217
|
+
target: 'tooling/git-reword-commit',
|
|
218
|
+
executable: true,
|
|
219
|
+
},
|
|
214
220
|
{
|
|
215
221
|
kind: 'raw',
|
|
216
222
|
source: 'git-format-staged.yml',
|
|
@@ -616,10 +616,11 @@ function yamlLinesForStep(step: PublishWorkflowStep, options: PublishWorkflowDef
|
|
|
616
616
|
' #',
|
|
617
617
|
' # `nx` cannot do it: the project graph loads this very plugin, so',
|
|
618
618
|
' # `nx build nx-plugin` deadlocks on the executor it is building.',
|
|
619
|
-
' # ttsc needs only the package tsconfig --
|
|
620
|
-
' #
|
|
619
|
+
' # ttsc needs only the package tsconfig -- the bootstrap the shell entry',
|
|
620
|
+
' # runs (tooling/direnv/enter-shell.ts) -- and emits as the typescript-emit',
|
|
621
|
+
' # executor does (EMIT_OVERRIDES), so dist holds no build info.',
|
|
621
622
|
' working-directory: packages/nx-plugin',
|
|
622
|
-
' run: ttsc
|
|
623
|
+
' run: bun ../../tooling/ttsc-emit.ts',
|
|
623
624
|
];
|
|
624
625
|
case PublishWorkflowStepKind.RepairPendingReleases:
|
|
625
626
|
return [
|
|
@@ -686,10 +687,12 @@ function yamlLinesForStep(step: PublishWorkflowStep, options: PublishWorkflowDef
|
|
|
686
687
|
// Scoped to the release's candidates: the build phase and the packed-package
|
|
687
688
|
// checks cover what ships. Unscoped, validate compiled every project - on a
|
|
688
689
|
// release touching no Rust that was 3m30 of the cowshed CLI on the critical
|
|
689
|
-
// path, for a binary the release did not contain.
|
|
690
|
+
// path, for a binary the release did not contain. The publish job repairs
|
|
691
|
+
// pending releases before it publishes, so a pin at a tagged version the
|
|
692
|
+
// registry still lacks is published by then (--repairs-pending).
|
|
690
693
|
return conditionalRunStep(
|
|
691
694
|
step,
|
|
692
|
-
`smoo monorepo validate --projects "${githubExpression('steps.version.outputs.projects')}"`,
|
|
695
|
+
`smoo monorepo validate --projects "${githubExpression('steps.version.outputs.projects')}" --repairs-pending`,
|
|
693
696
|
);
|
|
694
697
|
case PublishWorkflowStepKind.TagRelease:
|
|
695
698
|
return tagReleaseStepLines(step.name);
|
|
@@ -835,14 +838,14 @@ function renderSingleJobPublishWorkflowSteps(
|
|
|
835
838
|
lines.push('');
|
|
836
839
|
if (producer && step.kind === PublishWorkflowStepKind.Checkout) {
|
|
837
840
|
// A declared producer always runs, even when the graph selects no platform family.
|
|
838
|
-
lines.push(
|
|
841
|
+
lines.push(` # Step ${stepNumber++}`, ...platformProducerPreflightStepLines(producer), '');
|
|
839
842
|
}
|
|
840
843
|
if (
|
|
841
844
|
step.kind === PublishWorkflowStepKind.Build &&
|
|
842
845
|
(hasMacosPlatformTargets(options) || hasLinuxPlatformTargets(options))
|
|
843
846
|
) {
|
|
844
847
|
lines.push(
|
|
845
|
-
|
|
848
|
+
` # Step ${stepNumber++}`,
|
|
846
849
|
' - name: 🐧 Build supplemental Linux targets',
|
|
847
850
|
" if: steps.version.outputs.mode != 'none'",
|
|
848
851
|
' run:',
|
|
@@ -856,7 +859,7 @@ function renderSingleJobPublishWorkflowSteps(
|
|
|
856
859
|
);
|
|
857
860
|
if (crossOnThisRunner) {
|
|
858
861
|
lines.push(
|
|
859
|
-
|
|
862
|
+
` # Step ${stepNumber++}`,
|
|
860
863
|
' - name: 🍎 Build selected macOS and iOS release outputs',
|
|
861
864
|
" if: steps.version.outputs.mode != 'none'",
|
|
862
865
|
' run:',
|
|
@@ -871,7 +874,7 @@ function renderSingleJobPublishWorkflowSteps(
|
|
|
871
874
|
}
|
|
872
875
|
}
|
|
873
876
|
}
|
|
874
|
-
return mergeProducerStepEnvironment(lines, producer).join('\n').trimEnd()
|
|
877
|
+
return `${mergeProducerStepEnvironment(lines, producer).join('\n').trimEnd()}\n`;
|
|
875
878
|
}
|
|
876
879
|
|
|
877
880
|
function renderPlatformPublishWorkflowYaml(options: PublishWorkflowDefinitionOptions): string {
|
|
@@ -1156,10 +1159,11 @@ function renderMacosPlatformSteps(options: PublishWorkflowDefinitionOptions): st
|
|
|
1156
1159
|
' #',
|
|
1157
1160
|
' # `nx` cannot do it: the project graph loads this very plugin, so',
|
|
1158
1161
|
' # `nx build nx-plugin` deadlocks on the executor it is building.',
|
|
1159
|
-
' # ttsc needs only the package tsconfig --
|
|
1160
|
-
' #
|
|
1162
|
+
' # ttsc needs only the package tsconfig -- the bootstrap the shell entry',
|
|
1163
|
+
' # runs (tooling/direnv/enter-shell.ts) -- and emits as the typescript-emit',
|
|
1164
|
+
' # executor does (EMIT_OVERRIDES), so dist holds no build info.',
|
|
1161
1165
|
' working-directory: packages/nx-plugin',
|
|
1162
|
-
' run: ttsc
|
|
1166
|
+
' run: bun ../../tooling/ttsc-emit.ts',
|
|
1163
1167
|
);
|
|
1164
1168
|
}
|
|
1165
1169
|
lines.push(
|
|
@@ -1303,10 +1307,11 @@ function renderFinalLinuxPublishSteps(options: PublishWorkflowDefinitionOptions)
|
|
|
1303
1307
|
' #',
|
|
1304
1308
|
' # `nx` cannot do it: the project graph loads this very plugin, so',
|
|
1305
1309
|
' # `nx build nx-plugin` deadlocks on the executor it is building.',
|
|
1306
|
-
' # ttsc needs only the package tsconfig --
|
|
1307
|
-
' #
|
|
1310
|
+
' # ttsc needs only the package tsconfig -- the bootstrap the shell entry',
|
|
1311
|
+
' # runs (tooling/direnv/enter-shell.ts) -- and emits as the typescript-emit',
|
|
1312
|
+
' # executor does (EMIT_OVERRIDES), so dist holds no build info.',
|
|
1308
1313
|
' working-directory: packages/nx-plugin',
|
|
1309
|
-
' run: ttsc
|
|
1314
|
+
' run: bun ../../tooling/ttsc-emit.ts',
|
|
1310
1315
|
);
|
|
1311
1316
|
}
|
|
1312
1317
|
lines.push(
|