@smoothbricks/nx-plugin 0.7.11 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -25
- package/dist/.build-inputs.json +25 -17
- package/dist/built-output-packages.d.ts +27 -0
- package/dist/built-output-packages.d.ts.map +1 -0
- package/dist/built-output-packages.js +97 -0
- package/dist/cross-check-policy.d.ts +13 -1
- package/dist/cross-check-policy.d.ts.map +1 -1
- package/dist/cross-check-policy.js +13 -1
- package/dist/executors/bounded-exec/admission-queue.d.ts +68 -0
- package/dist/executors/bounded-exec/admission-queue.d.ts.map +1 -0
- package/dist/executors/bounded-exec/admission-queue.js +131 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts +14 -0
- package/dist/executors/bounded-exec/cpu-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/cpu-grant.js +18 -0
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +26 -24
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +88 -72
- package/dist/executors/bounded-exec/executor.d.ts +5 -7
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +69 -133
- package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
- package/dist/executors/bounded-exec/host-memory.js +6 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
- package/dist/executors/bounded-exec/memory-retry.d.ts +39 -0
- package/dist/executors/bounded-exec/memory-retry.d.ts.map +1 -0
- package/dist/executors/bounded-exec/memory-retry.js +33 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts +43 -0
- package/dist/executors/bounded-exec/parent-grant.d.ts.map +1 -0
- package/dist/executors/bounded-exec/parent-grant.js +37 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts +17 -0
- package/dist/executors/bounded-exec/peak-charge.d.ts.map +1 -0
- package/dist/executors/bounded-exec/peak-charge.js +20 -0
- package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +44 -8
- package/dist/executors/bounded-exec/pool-environment.d.ts +25 -0
- package/dist/executors/bounded-exec/pool-environment.d.ts.map +1 -0
- package/dist/executors/bounded-exec/pool-environment.js +34 -0
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +2 -2
- package/dist/executors/bounded-exec/verdict.d.ts +5 -18
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +12 -16
- package/dist/index.d.ts +4 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +62 -21
- package/dist/managed-files/ci-workflow.d.ts +29 -5
- package/dist/managed-files/ci-workflow.d.ts.map +1 -1
- package/dist/managed-files/ci-workflow.js +111 -16
- package/dist/managed-files/context.d.ts.map +1 -1
- package/dist/managed-files/context.js +1 -0
- package/dist/managed-files/files.d.ts +3 -1
- package/dist/managed-files/files.d.ts.map +1 -1
- package/dist/managed-files/files.js +2 -0
- package/dist/managed-files/publish-workflow.d.ts +6 -1
- package/dist/managed-files/publish-workflow.d.ts.map +1 -1
- package/dist/managed-files/publish-workflow.js +16 -11
- package/dist/owned-installation.d.ts.map +1 -1
- package/dist/owned-installation.js +48 -14
- package/dist/typecheck-route.d.ts +28 -0
- package/dist/typecheck-route.d.ts.map +1 -0
- package/dist/typecheck-route.js +109 -0
- package/dist/workspace-manifest.d.ts +50 -5
- package/dist/workspace-manifest.d.ts.map +1 -1
- package/executors.json +10 -0
- package/managed/raw/patches/nx@24.0.3.patch +16 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +93 -8
- package/managed/raw/tooling/direnv/apple-developer.sh +8 -1
- package/managed/raw/tooling/direnv/compiler-variables.sh +1 -1
- package/managed/raw/tooling/direnv/github-actions-bootstrap.sh +4 -2
- package/managed/raw/tooling/direnv/secret-references.ts +19 -5
- package/managed/raw/tooling/git-hooks/pre-push.sh +20 -10
- package/managed/templates/github/actions/forgejo-artifact.cjs +9 -9
- package/managed/templates/github/actions/setup-devenv/action.yml +53 -0
- package/package.json +2 -2
- package/src/built-output-packages.ts +125 -0
- package/src/cross-check-policy.ts +14 -1
- package/src/executors/bounded-exec/admission-queue.ts +163 -0
- package/src/executors/bounded-exec/cpu-grant.ts +32 -0
- package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
- package/src/executors/bounded-exec/cpu-pool.ts +123 -58
- package/src/executors/bounded-exec/executor.ts +93 -168
- package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
- package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
- package/src/executors/bounded-exec/host-memory.ts +25 -0
- package/src/executors/bounded-exec/memory-retry.ts +63 -0
- package/src/executors/bounded-exec/parent-grant.ts +75 -0
- package/src/executors/bounded-exec/peak-charge.ts +25 -0
- package/src/executors/bounded-exec/pool-admission.ts +66 -11
- package/src/executors/bounded-exec/pool-environment.ts +45 -0
- package/src/executors/bounded-exec/run-cpu-pool.ts +5 -4
- package/src/executors/bounded-exec/verdict.ts +17 -25
- package/src/index.ts +70 -21
- package/src/managed-files/ci-workflow.ts +143 -16
- package/src/managed-files/context.ts +1 -0
- package/src/managed-files/files.ts +5 -0
- package/src/managed-files/publish-workflow.ts +26 -10
- package/src/owned-installation.ts +63 -14
- package/src/typecheck-route.ts +64 -0
- package/src/workspace-manifest.ts +56 -5
- package/test-timings.json +141 -137
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { type TokenGrant, takeParentCpuTokens } from './cpu-pool.js';
|
|
2
|
+
import type { CpuWaitRow } from './lease-wait.js';
|
|
3
|
+
import type { InheritedCpuPool } from './pool-environment.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* What a run asks its parent's CPU pool and says of the answer. How many tokens the pool lends, how long the run waits
|
|
7
|
+
* and what it is charged are values that differ between runs of one task key; what a token count does to the command is
|
|
8
|
+
* `runner-sizing.ts`'s, keyed (README, "Bounded exec key").
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/** Tokens a parent pool lent, held until released. */
|
|
12
|
+
export type GrantedTokens = Extract<TokenGrant, { granted: true }>;
|
|
13
|
+
|
|
14
|
+
/** The parent pool's answer to a run: a grant, a refusal that fails the run, or neither, and what to say of it. */
|
|
15
|
+
export interface ParentGrant {
|
|
16
|
+
readonly grant: GrantedTokens | null;
|
|
17
|
+
/** A pool that answered and refused is a live budget, and the run fails rather than run beside it. */
|
|
18
|
+
readonly refusal: string | null;
|
|
19
|
+
readonly wait: CpuWaitRow | undefined;
|
|
20
|
+
/** What the run says, in order. */
|
|
21
|
+
readonly said: readonly string[];
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** What a run asks for: its count, what its task's last recorded peak charges it, and whether it runs alone. */
|
|
25
|
+
export interface Ask {
|
|
26
|
+
readonly want: number;
|
|
27
|
+
readonly charge: number | null;
|
|
28
|
+
/** The tokens the task was lent when its recorded peak was measured. */
|
|
29
|
+
readonly parallelism: number | null;
|
|
30
|
+
readonly alone: boolean;
|
|
31
|
+
readonly task: string | undefined;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Ask the parent pool for the run's tokens. A run that inherits no pool runs as asked and says nothing; a pool whose
|
|
36
|
+
* endpoint is gone (its holder exited) lends nothing, and the run goes on as asked.
|
|
37
|
+
*/
|
|
38
|
+
export async function askParentPool(inherited: InheritedCpuPool | null, ask: Ask): Promise<ParentGrant> {
|
|
39
|
+
if (inherited === null) {
|
|
40
|
+
return { grant: null, refusal: null, wait: undefined, said: [] };
|
|
41
|
+
}
|
|
42
|
+
const answer = await takeParentCpuTokens(inherited, ask.want, ask.charge, {
|
|
43
|
+
parallelism: ask.parallelism ?? undefined,
|
|
44
|
+
alone: ask.alone ? true : undefined,
|
|
45
|
+
task: ask.task,
|
|
46
|
+
});
|
|
47
|
+
if (answer.granted) {
|
|
48
|
+
return { grant: answer, refusal: null, wait: undefined, said: [] };
|
|
49
|
+
}
|
|
50
|
+
if (answer.cause === 'absent') {
|
|
51
|
+
return {
|
|
52
|
+
grant: null,
|
|
53
|
+
refusal: null,
|
|
54
|
+
wait: undefined,
|
|
55
|
+
said: [`cowshed: cpu-pool parent pool absent: ${answer.reason}; running the command as asked\n`],
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
return { grant: null, refusal: answer.reason, wait: answer.wait, said: [] };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** What a run says once its grant is in hand. */
|
|
62
|
+
export function describeGrant(
|
|
63
|
+
grant: GrantedTokens,
|
|
64
|
+
run: { waitedMs: number; want: number; runner: string; charge: number | null },
|
|
65
|
+
): string[] {
|
|
66
|
+
const said = [
|
|
67
|
+
`cowshed: cpu-pool wait done elapsed=${run.waitedMs}ms status=ok tokens=${grant.tokens}/${run.want} runner=${run.runner} memory=${run.charge ?? 'none'}${grant.charged !== null && grant.charged !== run.charge ? ` charged=${grant.charged}` : ''}\n`,
|
|
68
|
+
];
|
|
69
|
+
if (grant.oversized !== null) {
|
|
70
|
+
said.push(
|
|
71
|
+
`cowshed: cpu-pool granted while nothing else holds memory: this task's last peak of ${run.charge} bytes alone exceeds the run's ${grant.oversized.memory} bytes\n`,
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
return said;
|
|
75
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { readRecord, VERDICT_FILE } from './verdict.js';
|
|
3
|
+
|
|
4
|
+
/** What the task's last record charges its next run: the peak and the parallelism it was measured under. */
|
|
5
|
+
export type LastPeak =
|
|
6
|
+
| { ok: true; peakBytes: number | null; parallelism: number | null }
|
|
7
|
+
| { ok: false; error: string };
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The peak the task's last record in `directory` names: its charge in its run's pool. Null when the task has no record
|
|
11
|
+
* or its record no peak; a record that cannot be read is an error naming it, never a charge of nothing. It decides what a
|
|
12
|
+
* pool charges a run, a value that differs between runs of one task key, so the key leaves it out (README, "Bounded
|
|
13
|
+
* exec key").
|
|
14
|
+
*/
|
|
15
|
+
export async function readLastPeak(directory: string): Promise<LastPeak> {
|
|
16
|
+
try {
|
|
17
|
+
const record = await readRecord(directory);
|
|
18
|
+
return { ok: true, peakBytes: record?.peakBytes ?? null, parallelism: record?.cpu?.tokens ?? null };
|
|
19
|
+
} catch (error) {
|
|
20
|
+
return {
|
|
21
|
+
ok: false,
|
|
22
|
+
error: `the task's last record ${join(directory, VERDICT_FILE)} cannot be read: ${error instanceof Error ? error.message : String(error)}`,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
}
|
|
@@ -7,58 +7,113 @@
|
|
|
7
7
|
* Every grant needs both. The head takes `min(want, free)` tokens once one is free, and only while the charges of the
|
|
8
8
|
* leaves holding grants plus its own fit the run's memory; a head that cannot be granted holds up every request behind
|
|
9
9
|
* it, and no request holds part of a grant while it waits for the rest. A leaf is charged its task's last recorded
|
|
10
|
-
* memory peak, and nothing while its task has none. A
|
|
11
|
-
*
|
|
10
|
+
* memory peak, and nothing while its task has none. A runner's memory grows with the parallelism it is lent (a Vitest
|
|
11
|
+
* worker is a process of its own), so a peak measured while the leaf was lent `parallelism` tokens is charged in
|
|
12
|
+
* proportion to the tokens it is lent now when that is more, never less: the baseline a leaf holds however few workers
|
|
13
|
+
* it has makes a smaller grant no cheaper. When its whole ask does not fit, the head is lent the most tokens whose
|
|
14
|
+
* charge does. A leaf whose own charge alone exceeds the run's memory could never fit, so it is granted once nothing
|
|
15
|
+
* else holds memory, on one token when its peak names the parallelism it was measured under, and the grant says so.
|
|
16
|
+
*
|
|
17
|
+
* A leaf the kernel killed for memory, run again (`alone`), is granted only while nothing holds a grant, ahead of the
|
|
18
|
+
* leaves behind it, and while it runs nothing else is granted: a leaf that was killed beside others runs by itself.
|
|
12
19
|
*/
|
|
13
20
|
|
|
14
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* What a queued request asks: whole tokens, at least one, its charge in bytes (0 for a task with no peak), and the
|
|
23
|
+
* tokens that charge was measured under. Without them the charge is flat: it is the same however many tokens are lent.
|
|
24
|
+
*/
|
|
15
25
|
export interface PoolAsk {
|
|
16
26
|
readonly want: number;
|
|
17
27
|
readonly memory: number;
|
|
28
|
+
readonly parallelism?: number | undefined;
|
|
29
|
+
/** A leaf the kernel killed for memory, run again: granted while nothing else holds a grant, and alone. */
|
|
30
|
+
readonly alone?: boolean | undefined;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Who holds a grant when the pool decides: how many leaves, and whether one of them runs alone. */
|
|
34
|
+
export interface Holding {
|
|
35
|
+
readonly holders: number;
|
|
36
|
+
readonly alone: boolean;
|
|
18
37
|
}
|
|
19
38
|
|
|
39
|
+
const NOTHING_HELD: Holding = { holders: 0, alone: false };
|
|
20
40
|
/** The run's memory a pool charges against, and the bytes its holders are charged; null for a pool that charges none. */
|
|
21
41
|
export interface MemoryLedger {
|
|
22
42
|
readonly capacity: number;
|
|
23
43
|
readonly held: number;
|
|
24
44
|
}
|
|
25
45
|
|
|
26
|
-
/**
|
|
46
|
+
/**
|
|
47
|
+
* One request's grant: the request, its tokens, the bytes it is charged at those tokens, and whether its charge alone
|
|
48
|
+
* exceeds the run's memory.
|
|
49
|
+
*/
|
|
27
50
|
export interface PoolGrant<Ask extends PoolAsk> {
|
|
28
51
|
readonly request: Ask;
|
|
29
52
|
readonly tokens: number;
|
|
53
|
+
readonly charge: number;
|
|
30
54
|
readonly oversized: boolean;
|
|
31
55
|
}
|
|
32
56
|
|
|
33
57
|
/**
|
|
34
58
|
* The grants of the queue's first requests, in its order, and what the first request left waiting waits for: free
|
|
35
|
-
* tokens (`cpu`)
|
|
59
|
+
* tokens (`cpu`), the run's memory (`memory`) or the grants others hold (`alone`). `waiting` is null when every request was granted.
|
|
36
60
|
*/
|
|
37
61
|
export interface Admission<Ask extends PoolAsk> {
|
|
38
62
|
readonly grants: readonly PoolGrant<Ask>[];
|
|
39
|
-
readonly waiting: 'cpu' | 'memory' | null;
|
|
63
|
+
readonly waiting: 'cpu' | 'memory' | 'alone' | null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The bytes `ask` is charged when it is lent `tokens`: its peak, scaled up when that is more than it was measured under. */
|
|
67
|
+
export function chargeAt(ask: PoolAsk, tokens: number): number {
|
|
68
|
+
const measured = ask.parallelism;
|
|
69
|
+
return measured === undefined || tokens <= measured ? ask.memory : Math.ceil((ask.memory * tokens) / measured);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The most tokens, up to `most`, whose charge fits `room` bytes; none when even the peak itself does not. */
|
|
73
|
+
function tokensFitting(ask: PoolAsk, most: number, room: number): number {
|
|
74
|
+
if (ask.memory > room) {
|
|
75
|
+
return 0;
|
|
76
|
+
}
|
|
77
|
+
const measured = ask.parallelism;
|
|
78
|
+
if (measured === undefined || ask.memory === 0 || most <= measured) {
|
|
79
|
+
return most;
|
|
80
|
+
}
|
|
81
|
+
return Math.min(most, Math.floor((room * measured) / ask.memory));
|
|
40
82
|
}
|
|
41
83
|
|
|
42
84
|
export function admit<Ask extends PoolAsk>(
|
|
43
85
|
queue: readonly Ask[],
|
|
44
86
|
available: number,
|
|
45
87
|
memory: MemoryLedger | null,
|
|
88
|
+
holding: Holding = NOTHING_HELD,
|
|
46
89
|
): Admission<Ask> {
|
|
47
90
|
const grants: PoolGrant<Ask>[] = [];
|
|
91
|
+
if (holding.alone) {
|
|
92
|
+
return { grants, waiting: queue.length === 0 ? null : 'alone' };
|
|
93
|
+
}
|
|
48
94
|
let free = available;
|
|
49
95
|
let held = memory?.held ?? 0;
|
|
50
96
|
for (const request of queue) {
|
|
97
|
+
if (request.alone === true && holding.holders + grants.length > 0) {
|
|
98
|
+
return { grants, waiting: 'alone' };
|
|
99
|
+
}
|
|
51
100
|
const oversized = memory !== null && request.memory > memory.capacity;
|
|
52
|
-
|
|
101
|
+
const room = memory === null ? Number.POSITIVE_INFINITY : memory.capacity - held;
|
|
102
|
+
if (request.memory > room && !(oversized && held === 0)) {
|
|
53
103
|
return { grants, waiting: 'memory' };
|
|
54
104
|
}
|
|
55
|
-
const
|
|
56
|
-
if (
|
|
105
|
+
const most = Math.min(request.want, free);
|
|
106
|
+
if (most === 0) {
|
|
57
107
|
return { grants, waiting: 'cpu' };
|
|
58
108
|
}
|
|
109
|
+
const tokens = oversized ? (request.parallelism === undefined ? most : 1) : tokensFitting(request, most, room);
|
|
110
|
+
const charge = chargeAt(request, tokens);
|
|
59
111
|
free -= tokens;
|
|
60
|
-
held +=
|
|
61
|
-
grants.push({ request, tokens, oversized });
|
|
112
|
+
held += charge;
|
|
113
|
+
grants.push({ request, tokens, charge, oversized });
|
|
114
|
+
if (request.alone === true) {
|
|
115
|
+
return { grants, waiting: grants.length < queue.length ? 'alone' : null };
|
|
116
|
+
}
|
|
62
117
|
}
|
|
63
118
|
return { grants, waiting: null };
|
|
64
119
|
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* What a CPU pool puts in a command's environment (cpu-pool.ts serves the pool; this module decides what the command
|
|
5
|
+
* sees of it, so it is keyed: executors.json lists cpu-pool.js as output-neutral, which it may only be while it writes
|
|
6
|
+
* no variable the command reads).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/** The environment name a child reads its parent's pool endpoint from; its count is `CPU_TOKENS_ENV`. */
|
|
10
|
+
export const CPU_POOL_ENV = 'BOUNDED_EXEC_CPU_POOL';
|
|
11
|
+
|
|
12
|
+
/** An inherited pool descriptor; only a grant from its endpoint proves a live allocation. */
|
|
13
|
+
export interface InheritedCpuPool {
|
|
14
|
+
readonly socket: string;
|
|
15
|
+
/** The count the parent names beside its endpoint. */
|
|
16
|
+
readonly tokens: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* The inherited descriptor, or null when malformed or absent. A missing or bare count, or an empty
|
|
21
|
+
* or NUL-bearing endpoint, proves no allocation; the count is a positive whole number. A valid
|
|
22
|
+
* descriptor still needs a live endpoint grant.
|
|
23
|
+
*/
|
|
24
|
+
export function readInheritedCpuPool(environment: Readonly<NodeJS.ProcessEnv>): InheritedCpuPool | null {
|
|
25
|
+
const socket = environment[CPU_POOL_ENV];
|
|
26
|
+
const tokens = parseInheritedTokens(environment[CPU_TOKENS_ENV]);
|
|
27
|
+
if (socket === undefined || socket.length === 0 || socket.includes('\0') || tokens === null) {
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
return { socket, tokens };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The environment a child of a pool inherits to reach it: its endpoint, the count the pool holds, and its make
|
|
35
|
+
* jobserver, which Cargo and make read from the fifo; `-j` with a jobserver names no count of its own.
|
|
36
|
+
*/
|
|
37
|
+
export function poolEnvironment(socketPath: string, tokens: number, fifoPath: string): Record<string, string> {
|
|
38
|
+
const jobserverFlags = `-j --jobserver-auth=fifo:${fifoPath}`;
|
|
39
|
+
return {
|
|
40
|
+
[CPU_POOL_ENV]: socketPath,
|
|
41
|
+
[CPU_TOKENS_ENV]: String(tokens),
|
|
42
|
+
CARGO_MAKEFLAGS: jobserverFlags,
|
|
43
|
+
MAKEFLAGS: jobserverFlags,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
@@ -3,10 +3,11 @@ import { fileURLToPath } from 'node:url';
|
|
|
3
3
|
import { Worker } from 'node:worker_threads';
|
|
4
4
|
import { logger } from 'nx/src/devkit-exports.js';
|
|
5
5
|
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
6
|
-
import {
|
|
6
|
+
import type { CloseOutcome, CpuPool } from './cpu-pool.js';
|
|
7
7
|
import type { CpuPoolThreadData, CpuPoolThreadReport } from './cpu-pool-thread.js';
|
|
8
|
-
import { type RunMemoryReading, RunMemoryUnreadable, readRunMemory } from './host-memory.js';
|
|
8
|
+
import { type RunMemory, type RunMemoryReading, RunMemoryUnreadable, readRunMemory } from './host-memory.js';
|
|
9
9
|
import { mbxPool } from './mbx-pool.js';
|
|
10
|
+
import { readInheritedCpuPool } from './pool-environment.js';
|
|
10
11
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
11
12
|
|
|
12
13
|
/**
|
|
@@ -91,7 +92,7 @@ export async function serveRunCpuPool(
|
|
|
91
92
|
if (!memory.ok) {
|
|
92
93
|
throw new RunMemoryUnreadable(memory.error);
|
|
93
94
|
}
|
|
94
|
-
const served = await servePoolThread(tokens, memory.memory
|
|
95
|
+
const served = await servePoolThread(tokens, memory.memory);
|
|
95
96
|
if (!served.ok) {
|
|
96
97
|
logger.warn(
|
|
97
98
|
`bounded-exec: this run's CPU pool failed (${served.error}); its tasks are sized only by the ceilings they inherit`,
|
|
@@ -110,7 +111,7 @@ export async function serveRunCpuPool(
|
|
|
110
111
|
* has said how its pool closed fails whichever wait is open, a later close returns that failure, and a thread lost while
|
|
111
112
|
* its pool serves is said at once.
|
|
112
113
|
*/
|
|
113
|
-
function servePoolThread(tokens: number, memory:
|
|
114
|
+
function servePoolThread(tokens: number, memory: RunMemory): Promise<ServedPool> {
|
|
114
115
|
const thread = new Worker(THREAD_ENTRY, {
|
|
115
116
|
workerData: { tokens, memory } satisfies CpuPoolThreadData,
|
|
116
117
|
...THREAD_PRELOAD,
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import typia from 'typia';
|
|
4
|
+
import type { CpuGrantRecord } from './cpu-grant.js';
|
|
5
|
+
import { parseCpuGrant } from './cpu-grant.js';
|
|
4
6
|
import type { CpuWaitRow } from './lease-wait.js';
|
|
5
7
|
import { describeHolders, type LockWait, type Opener } from './lock-wait.js';
|
|
6
8
|
import type { MbxQueuedRequest } from './mbx-queue.js';
|
|
@@ -87,14 +89,11 @@ export interface BoundedExecRecord {
|
|
|
87
89
|
readonly verdict: BoundedExecVerdict;
|
|
88
90
|
/**
|
|
89
91
|
* The parent CPU pool's grant the run started under: tokens granted of those asked for, and
|
|
90
|
-
* how long it waited for the pool. Absent when it ran ungranted
|
|
91
|
-
*
|
|
92
|
+
* how long it waited for the pool. Absent when it ran ungranted. Its tokens are the parallelism the run's
|
|
93
|
+
* `peakBytes` was measured under, which the task's next run reads (`readLastPeak`, peak-charge.ts) to charge that peak in proportion
|
|
94
|
+
* to the parallelism it is lent.
|
|
92
95
|
*/
|
|
93
|
-
readonly cpu?:
|
|
94
|
-
readonly tokens: number;
|
|
95
|
-
readonly want: number;
|
|
96
|
-
readonly waitedMs: number;
|
|
97
|
-
};
|
|
96
|
+
readonly cpu?: CpuGrantRecord;
|
|
98
97
|
/**
|
|
99
98
|
* The parent CPU pool's wait that expired and refused the run: its code, bound and elapsed time, the last heartbeat
|
|
100
99
|
* sequence it read, and the pool's socket and pid (specs/cowshed/05_gateway.md, "Deadlines"). Absent when no wait
|
|
@@ -495,23 +494,6 @@ export async function readRecord(directory: string): Promise<BoundedExecRecord |
|
|
|
495
494
|
return parseRecord(JSON.parse(text));
|
|
496
495
|
}
|
|
497
496
|
|
|
498
|
-
/**
|
|
499
|
-
* The peak the task's last record in `directory` names: its charge in its run's pool. Null when the task has no record
|
|
500
|
-
* or its record no peak; a record that cannot be read is an error naming it, never a charge of nothing.
|
|
501
|
-
*/
|
|
502
|
-
export async function readLastPeak(
|
|
503
|
-
directory: string,
|
|
504
|
-
): Promise<{ ok: true; peakBytes: number | null } | { ok: false; error: string }> {
|
|
505
|
-
try {
|
|
506
|
-
return { ok: true, peakBytes: (await readRecord(directory))?.peakBytes ?? null };
|
|
507
|
-
} catch (error) {
|
|
508
|
-
return {
|
|
509
|
-
ok: false,
|
|
510
|
-
error: `the task's last record ${join(directory, VERDICT_FILE)} cannot be read: ${error instanceof Error ? error.message : String(error)}`,
|
|
511
|
-
};
|
|
512
|
-
}
|
|
513
|
-
}
|
|
514
|
-
|
|
515
497
|
/** The JUnit report a command wrote into `directory`, or `null` when it wrote none. */
|
|
516
498
|
export async function readJunitReport(
|
|
517
499
|
directory: string,
|
|
@@ -540,7 +522,17 @@ function parseRecord(value: unknown): BoundedExecRecord {
|
|
|
540
522
|
if (!(peakBytes === null || (typeof peakBytes === 'number' && Number.isSafeInteger(peakBytes) && peakBytes >= 0))) {
|
|
541
523
|
throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
|
|
542
524
|
}
|
|
543
|
-
|
|
525
|
+
const cpu = parseCpuGrant(value.cpu);
|
|
526
|
+
if (cpu === 'malformed') {
|
|
527
|
+
throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
|
|
528
|
+
}
|
|
529
|
+
return {
|
|
530
|
+
task: value.task,
|
|
531
|
+
hash: value.hash,
|
|
532
|
+
verdict: parseVerdict(value.verdict),
|
|
533
|
+
...(cpu === undefined ? {} : { cpu }),
|
|
534
|
+
peakBytes,
|
|
535
|
+
};
|
|
544
536
|
}
|
|
545
537
|
|
|
546
538
|
function parseVerdict(value: unknown): BoundedExecVerdict {
|
package/src/index.ts
CHANGED
|
@@ -25,6 +25,7 @@ import { AggregateCreateNodesError } from 'nx/src/project-graph/error-types.js';
|
|
|
25
25
|
import { parse as parseToml } from 'smol-toml';
|
|
26
26
|
import { BOUNDED_TEST_IDLE_TIMEOUT_MS, BOUNDED_TEST_KILL_AFTER_MS } from './bounded-test-policy.js';
|
|
27
27
|
import { refuseOverlappingOutputs } from './build-state-output-policy.js';
|
|
28
|
+
import { builtOutputLockfileDependencies, builtOutputLockfileNodes } from './built-output-packages.js';
|
|
28
29
|
import { bunLockDependencies, bunLockInput, bunLockNode } from './bun-lock-scope.js';
|
|
29
30
|
import {
|
|
30
31
|
CARGO_CLOSURE_INPUT,
|
|
@@ -104,6 +105,7 @@ import {
|
|
|
104
105
|
} from './test-timings.js';
|
|
105
106
|
import { assertUpstreamBasePrecondition } from './test-walk-policy.js';
|
|
106
107
|
import { type NativeTools, nativeToolCommand, nativeToolDependsOn, resolveNativeTools } from './tool-bins.js';
|
|
108
|
+
import { typecheckRoute } from './typecheck-route.js';
|
|
107
109
|
import {
|
|
108
110
|
isBuildOutputTargetName,
|
|
109
111
|
isTestNamespaceTargetName,
|
|
@@ -130,15 +132,14 @@ const TYPESCRIPT_PATCH_INPUTS = ['{workspaceRoot}/patches/**/*', '!{workspaceRoo
|
|
|
130
132
|
const WORKSPACE_TSCONFIG_BASE_INPUT = '{workspaceRoot}/tsconfig.base.json';
|
|
131
133
|
|
|
132
134
|
/**
|
|
133
|
-
* The native compiler a typecheck runs
|
|
134
|
-
* from a pinned source revision and exports its path. The typecheck targets run it
|
|
135
|
-
* not through ttsc: ttsc auto-loads a dependency's transform plugin (typia) and then checks
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
* variable: falling back would check with another compiler under the same key.
|
|
135
|
+
* The native compiler a typecheck runs unless a dependency declares a dialect (typecheck-route.ts), apart from the one
|
|
136
|
+
* emit runs: devenv.smoo.nix builds it from a pinned source revision and exports its path. The typecheck targets run it
|
|
137
|
+
* directly, not through ttsc: ttsc auto-loads a dependency's transform plugin (typia) and then checks inside that
|
|
138
|
+
* plugin's host, which links ttsc's own compiler and never runs the pinned one. Transform diagnostics stay enforced
|
|
139
|
+
* where the transform runs: for the lib program in `tsc-js`'s emit, for test files when the test runner transforms
|
|
140
|
+
* them. A test file no test executes has its types checked here but its transform calls checked nowhere. Hashed by the
|
|
141
|
+
* store path, which carries the revision, so a compiler bump misses the cache. The command refuses an unset variable:
|
|
142
|
+
* falling back would check with another compiler under the same key.
|
|
142
143
|
*/
|
|
143
144
|
const TYPECHECK_COMPILER_ENV = 'SMOO_TYPECHECK_TSC';
|
|
144
145
|
const TYPECHECK_COMPILER_INPUT = { env: TYPECHECK_COMPILER_ENV };
|
|
@@ -146,6 +147,29 @@ function typecheckCommand(args: string): string {
|
|
|
146
147
|
return `"\${${TYPECHECK_COMPILER_ENV}:?is unset, enter the devenv shell (devenv.smoo.nix builds the typecheck compiler)}" ${args}`;
|
|
147
148
|
}
|
|
148
149
|
|
|
150
|
+
/** How one project's typecheck targets run: their command, the compiler input they hash, and what they wait for. */
|
|
151
|
+
interface TypecheckTargets {
|
|
152
|
+
readonly command: (args: string) => string;
|
|
153
|
+
readonly inputs: readonly NonNullable<TargetConfiguration['inputs']>[number][];
|
|
154
|
+
readonly dependsOn: readonly string[];
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* A project's typecheck targets. A project whose direct dependency declares a dialect typechecks through ttsc, whose
|
|
159
|
+
* linked plugin lowers the dialect and withdraws the checker's refusals of it; its compiler is the installed ttsc,
|
|
160
|
+
* which bun.lock and the patches in its check inputs already key, and it waits for the targets the declaring package
|
|
161
|
+
* names to prepare the plugin binaries ttsc loads. Every other project runs the pinned native compiler.
|
|
162
|
+
*/
|
|
163
|
+
function typecheckTargets(packageJsonPath: string, packageJson: PackageJson): TypecheckTargets {
|
|
164
|
+
const route = typecheckRoute(packageJsonPath, [
|
|
165
|
+
...Object.keys(packageJson.dependencies ?? {}),
|
|
166
|
+
...Object.keys(packageJson.devDependencies ?? {}),
|
|
167
|
+
]);
|
|
168
|
+
return route.declaredBy.length === 0
|
|
169
|
+
? { command: typecheckCommand, inputs: [TYPECHECK_COMPILER_INPUT], dependsOn: [] }
|
|
170
|
+
: { command: (args) => `ttsc ${args}`, inputs: [], dependsOn: route.dependsOn };
|
|
171
|
+
}
|
|
172
|
+
|
|
149
173
|
/**
|
|
150
174
|
* The release-rewritten root manifest hashed by field instead of by byte, for
|
|
151
175
|
* a check that only reads it.
|
|
@@ -1131,12 +1155,13 @@ function supportsJsonInputs(): boolean {
|
|
|
1131
1155
|
type CreateNodesHandler = CreateNodesV2[1];
|
|
1132
1156
|
|
|
1133
1157
|
function createNodesHandler(hostPlatform: NapiPlatform | null): CreateNodesHandler {
|
|
1134
|
-
return async (projectConfigurationFiles,
|
|
1158
|
+
return async (projectConfigurationFiles, options, context) => {
|
|
1135
1159
|
const results: CreateNodesResultV2 = [];
|
|
1136
1160
|
const errors: Array<[file: string | null, error: Error]> = [];
|
|
1137
1161
|
let cargoWorkspaces: CargoWorkspace[];
|
|
1138
1162
|
let eslint: EslintWorkspace | null;
|
|
1139
1163
|
let nativeTools: NativeTools | null;
|
|
1164
|
+
let lockfileNodes: Record<string, ProjectGraphExternalNode>;
|
|
1140
1165
|
try {
|
|
1141
1166
|
cargoWorkspaces = await resolveCargoWorkspaces(projectConfigurationFiles, context.workspaceRoot, hostPlatform);
|
|
1142
1167
|
eslint = await eslintWorkspace(context.workspaceRoot);
|
|
@@ -1149,6 +1174,7 @@ function createNodesHandler(hostPlatform: NapiPlatform | null): CreateNodesHandl
|
|
|
1149
1174
|
EXECUTABLE_PLATFORM_DIRECTORY[hostPlatform.targetFamily][hostPlatform.architecture],
|
|
1150
1175
|
scriptcTargetName(hostPlatform),
|
|
1151
1176
|
);
|
|
1177
|
+
lockfileNodes = builtOutputLockfileNodes(options, context);
|
|
1152
1178
|
} catch (error) {
|
|
1153
1179
|
throw new AggregateCreateNodesError([[null, error instanceof Error ? error : new Error(String(error))]], results);
|
|
1154
1180
|
}
|
|
@@ -1185,6 +1211,20 @@ function createNodesHandler(hostPlatform: NapiPlatform | null): CreateNodesHandl
|
|
|
1185
1211
|
}),
|
|
1186
1212
|
);
|
|
1187
1213
|
|
|
1214
|
+
// The root manifest's result carries the lockfile's package nodes, as it carries the root's lock-scope node.
|
|
1215
|
+
if (Object.keys(lockfileNodes).length > 0) {
|
|
1216
|
+
const rootIndex = results.findIndex(([file]) => file === 'package.json');
|
|
1217
|
+
const rootResult = results[rootIndex];
|
|
1218
|
+
if (rootResult === undefined) {
|
|
1219
|
+
errors.push([
|
|
1220
|
+
null,
|
|
1221
|
+
new Error('builtOutputPackages needs a root package.json to publish the lockfile nodes on'),
|
|
1222
|
+
]);
|
|
1223
|
+
} else {
|
|
1224
|
+
const [file, result] = rootResult;
|
|
1225
|
+
results[rootIndex] = [file, { ...result, externalNodes: { ...lockfileNodes, ...result.externalNodes } }];
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1188
1228
|
if (errors.length > 0) {
|
|
1189
1229
|
throw new AggregateCreateNodesError(errors, results);
|
|
1190
1230
|
}
|
|
@@ -1241,8 +1281,14 @@ export const postTasksExecution: PostTasksExecution<ProducerReceiptsOptions> = a
|
|
|
1241
1281
|
await postCargoRetention(options, context);
|
|
1242
1282
|
};
|
|
1243
1283
|
|
|
1244
|
-
/**
|
|
1245
|
-
|
|
1284
|
+
/**
|
|
1285
|
+
* The lock-scope nodes' edges: each bun node's to the packages and members its manifest resolves; and, for a workspace
|
|
1286
|
+
* that names built-output packages, the lockfile's package edges (`built-output-packages.ts`).
|
|
1287
|
+
*/
|
|
1288
|
+
export const createDependencies: CreateDependencies = (options, context) => [
|
|
1289
|
+
...bunLockDependencies(context),
|
|
1290
|
+
...builtOutputLockfileDependencies(options, context),
|
|
1291
|
+
];
|
|
1246
1292
|
|
|
1247
1293
|
/** Nx invokes this only after merging inferred/declarative targets and every plugin's project dependencies. */
|
|
1248
1294
|
export const createMetadata: CreateMetadata = (graph, _options, context) => {
|
|
@@ -1255,6 +1301,7 @@ export default { createNodesV2, createDependencies, createMetadata, postTasksExe
|
|
|
1255
1301
|
|
|
1256
1302
|
interface PackageJson {
|
|
1257
1303
|
bin?: unknown;
|
|
1304
|
+
dependencies?: Record<string, unknown>;
|
|
1258
1305
|
devDependencies?: Record<string, unknown>;
|
|
1259
1306
|
name?: string;
|
|
1260
1307
|
napi?: Record<string, unknown>;
|
|
@@ -1559,6 +1606,7 @@ async function createProjectTargets(
|
|
|
1559
1606
|
const projectRoot = dirname(packageJsonPath);
|
|
1560
1607
|
const absoluteProjectRoot = join(workspaceRoot, projectRoot);
|
|
1561
1608
|
const packageJson = await readPackageJson(join(workspaceRoot, packageJsonPath));
|
|
1609
|
+
const typecheck = typecheckTargets(join(workspaceRoot, packageJsonPath), packageJson);
|
|
1562
1610
|
const projectName = packageJson.nx?.name ?? packageJson.name;
|
|
1563
1611
|
if (typeof projectName !== 'string' || projectName.length === 0) {
|
|
1564
1612
|
throw new Error(`${packageJsonPath} must declare a non-empty package or nx project name`);
|
|
@@ -1669,13 +1717,13 @@ async function createProjectTargets(
|
|
|
1669
1717
|
selfProduction,
|
|
1670
1718
|
dependencyProduction,
|
|
1671
1719
|
...checkInputs,
|
|
1672
|
-
|
|
1720
|
+
...typecheck.inputs,
|
|
1673
1721
|
'{projectRoot}/tsconfig.lib.json',
|
|
1674
1722
|
],
|
|
1675
1723
|
outputs: [],
|
|
1676
|
-
dependsOn: ['^*-js', ...(cargoWasmConfig ? ['cargo-wasm'] : [])],
|
|
1724
|
+
dependsOn: ['^*-js', ...(cargoWasmConfig ? ['cargo-wasm'] : []), ...typecheck.dependsOn],
|
|
1677
1725
|
options: {
|
|
1678
|
-
command:
|
|
1726
|
+
command: typecheck.command('-p tsconfig.lib.json --noEmit'),
|
|
1679
1727
|
cwd: projectRoot,
|
|
1680
1728
|
},
|
|
1681
1729
|
};
|
|
@@ -1693,12 +1741,12 @@ async function createProjectTargets(
|
|
|
1693
1741
|
selfDefault,
|
|
1694
1742
|
dependencyProduction,
|
|
1695
1743
|
...checkInputs,
|
|
1696
|
-
|
|
1744
|
+
...typecheck.inputs,
|
|
1697
1745
|
'{projectRoot}/tsconfig.test.json',
|
|
1698
1746
|
],
|
|
1699
|
-
dependsOn: ['typecheck'],
|
|
1747
|
+
dependsOn: ['typecheck', ...typecheck.dependsOn],
|
|
1700
1748
|
options: {
|
|
1701
|
-
command:
|
|
1749
|
+
command: typecheck.command('-p tsconfig.test.json --noEmit'),
|
|
1702
1750
|
cwd: projectRoot,
|
|
1703
1751
|
},
|
|
1704
1752
|
};
|
|
@@ -1706,7 +1754,7 @@ async function createProjectTargets(
|
|
|
1706
1754
|
executor: 'nx:run-commands',
|
|
1707
1755
|
continuous: true,
|
|
1708
1756
|
options: {
|
|
1709
|
-
command:
|
|
1757
|
+
command: typecheck.command('-p tsconfig.test.json --noEmit --watch'),
|
|
1710
1758
|
cwd: projectRoot,
|
|
1711
1759
|
},
|
|
1712
1760
|
};
|
|
@@ -2296,9 +2344,9 @@ async function createProjectTargets(
|
|
|
2296
2344
|
// Test configs may resolve the package's own `./index.js` through dist, so the
|
|
2297
2345
|
// current-project declaration output is rebuilt first, not only the
|
|
2298
2346
|
// dependency-project outputs reached through typecheck.
|
|
2299
|
-
targets['typecheck-tests'].dependsOn = ['tsc-js', 'typecheck'];
|
|
2347
|
+
targets['typecheck-tests'].dependsOn = ['tsc-js', 'typecheck', ...typecheck.dependsOn];
|
|
2300
2348
|
} else if (targets.build) {
|
|
2301
|
-
targets['typecheck-tests'].dependsOn = ['build'];
|
|
2349
|
+
targets['typecheck-tests'].dependsOn = ['build', ...typecheck.dependsOn];
|
|
2302
2350
|
}
|
|
2303
2351
|
}
|
|
2304
2352
|
|
|
@@ -2913,6 +2961,7 @@ async function readPackageJson(packageJsonPath: string): Promise<PackageJson> {
|
|
|
2913
2961
|
const rawNx = isRecord(parsed.nx) ? parsed.nx : undefined;
|
|
2914
2962
|
return {
|
|
2915
2963
|
...('bin' in parsed ? { bin: parsed.bin } : {}),
|
|
2964
|
+
...(isRecord(parsed.dependencies) ? { dependencies: parsed.dependencies } : {}),
|
|
2916
2965
|
...(isRecord(parsed.devDependencies) ? { devDependencies: parsed.devDependencies } : {}),
|
|
2917
2966
|
...(typeof parsed.name === 'string' ? { name: parsed.name } : {}),
|
|
2918
2967
|
...(isRecord(parsed.napi) ? { napi: parsed.napi } : {}),
|