@smoothbricks/nx-plugin 0.7.10 → 0.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -21
- package/dist/.build-inputs.json +9 -9
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +25 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +86 -23
- package/dist/executors/bounded-exec/executor.d.ts +5 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +44 -5
- package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
- package/dist/executors/bounded-exec/host-memory.js +6 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
- package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +44 -8
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +4 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +19 -2
- package/managed/raw/patches/nx@24.0.3.patch +16 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +189 -2
- package/package.json +1 -1
- package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
- package/src/executors/bounded-exec/cpu-pool.ts +121 -25
- package/src/executors/bounded-exec/executor.ts +71 -5
- package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
- package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
- package/src/executors/bounded-exec/host-memory.ts +25 -0
- package/src/executors/bounded-exec/pool-admission.ts +66 -11
- package/src/executors/bounded-exec/run-cpu-pool.ts +3 -3
- package/src/executors/bounded-exec/verdict.ts +25 -5
- package/test-timings.json +2 -0
|
@@ -18,7 +18,27 @@ export interface HostMemory {
|
|
|
18
18
|
* listing and its read adds nothing, and a root that has exited holds nothing.
|
|
19
19
|
*/
|
|
20
20
|
treeResidentBytes(root: number): Promise<number>;
|
|
21
|
+
/**
|
|
22
|
+
* What the run's bound holds that the kernel cannot take back, read now: its anonymous and shared memory. `bound` is
|
|
23
|
+
* the `source` of the run's memory (`RunMemory`): the cgroup's `memory.max`, whose `memory.stat` is read, or the
|
|
24
|
+
* host's `MemTotal`, whose `AnonPages` and `Shmem` are. The pool says it on every grant, so what the leaves are not
|
|
25
|
+
* charged for (the baseline, a tmpfs) and what a suite holds beyond its record are measured, not guessed.
|
|
26
|
+
*/
|
|
27
|
+
liveUsage(bound: string): LiveUsageReading;
|
|
28
|
+
}
|
|
29
|
+
/** What a bound holds live, in bytes, and the file it was read from. */
|
|
30
|
+
export interface LiveUsage {
|
|
31
|
+
readonly anon: number;
|
|
32
|
+
readonly shmem: number;
|
|
33
|
+
readonly source: string;
|
|
21
34
|
}
|
|
35
|
+
export type LiveUsageReading = {
|
|
36
|
+
readonly ok: true;
|
|
37
|
+
readonly usage: LiveUsage;
|
|
38
|
+
} | {
|
|
39
|
+
readonly ok: false;
|
|
40
|
+
readonly reason: string;
|
|
41
|
+
};
|
|
22
42
|
/** The run's memory in bytes, and where it was read: the file or sysctl that holds it. */
|
|
23
43
|
export interface RunMemory {
|
|
24
44
|
readonly bytes: number;
|
|
@@ -51,4 +71,6 @@ export declare class RunMemoryUnreadable extends Error {
|
|
|
51
71
|
export declare const HOST_MEMORY: HostMemory | null;
|
|
52
72
|
/** The run's memory on this platform, or why it cannot be read. */
|
|
53
73
|
export declare function readRunMemory(): Promise<RunMemoryReading>;
|
|
74
|
+
/** The run's bound live on this platform, or why it cannot be read: a pool says it, never leaves it out. */
|
|
75
|
+
export declare function readLiveUsage(bound: string): LiveUsageReading;
|
|
54
76
|
//# sourceMappingURL=host-memory.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host-memory.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3C;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IACtD;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"host-memory.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.ts"],"names":[],"mappings":"AAGA;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,mEAAmE;IACnE,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC3C;;;OAGG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,CAAC;IACtD;;;;;OAKG;IACH,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACjD;;;;;OAKG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAAC;CAC5C;AAED,wEAAwE;AACxE,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAA;CAAE,GAChD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpD,0FAA0F;AAC1F,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,4FAA4F;AAC5F,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,SAAS,EAAE,iBAAiB,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,IAAI,EAAE,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,sBAAsB,CAAC;IACjF,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAA;CAAE,CAAC;AAE3D,sFAAsF;AACtF,qBAAa,mBAAoB,SAAQ,KAAK;IAChC,QAAQ,CAAC,KAAK,EAAE,cAAc;IAA1C,YAAqB,KAAK,EAAE,cAAc,EAKzC;CACF;AAID,kGAAkG;AAClG,eAAO,MAAM,WAAW,EAAE,UAAU,GAAG,IAA0C,CAAC;AAElF,mEAAmE;AACnE,wBAAgB,aAAa,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAazD;AAED,4GAA4G;AAC5G,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAI7D"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host-memory.darwin.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.darwin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,kBAAkB,CAAC;AAqIrE,eAAO,MAAM,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"host-memory.darwin.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.darwin.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAoB,MAAM,kBAAkB,CAAC;AAqIrE,eAAO,MAAM,YAAY,EAAE,UAM1B,CAAC"}
|
|
@@ -99,4 +99,10 @@ async function treeResidentBytes(root) {
|
|
|
99
99
|
}
|
|
100
100
|
return total;
|
|
101
101
|
}
|
|
102
|
-
export const darwinMemory = {
|
|
102
|
+
export const darwinMemory = {
|
|
103
|
+
readRunMemory,
|
|
104
|
+
peakBytes: (maxRss) => maxRss,
|
|
105
|
+
treeResidentBytes,
|
|
106
|
+
// macOS has no cgroup, and its anonymous and shared pages are not read from a file the pool may open.
|
|
107
|
+
liveUsage: () => ({ ok: false, reason: 'macOS keeps no anonymous and shared memory of a run to read' }),
|
|
108
|
+
};
|
|
@@ -27,3 +27,9 @@ export function readRunMemory() {
|
|
|
27
27
|
}
|
|
28
28
|
return HOST_MEMORY.readRunMemory();
|
|
29
29
|
}
|
|
30
|
+
/** The run's bound live on this platform, or why it cannot be read: a pool says it, never leaves it out. */
|
|
31
|
+
export function readLiveUsage(bound) {
|
|
32
|
+
return HOST_MEMORY === null
|
|
33
|
+
? { ok: false, reason: `${process.platform} has no reader of live memory` }
|
|
34
|
+
: HOST_MEMORY.liveUsage(bound);
|
|
35
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { HostMemory, RunMemoryReading } from './host-memory.js';
|
|
1
|
+
import type { HostMemory, LiveUsageReading, RunMemoryReading } from './host-memory.js';
|
|
2
2
|
/** A file's text, or the errno code and message its read failed with. */
|
|
3
3
|
export type FileRead = {
|
|
4
4
|
readonly ok: true;
|
|
@@ -10,5 +10,7 @@ export type FileRead = {
|
|
|
10
10
|
};
|
|
11
11
|
/** The run's memory, reading every file through `read`. */
|
|
12
12
|
export declare function linuxRunMemory(read: (path: string) => Promise<FileRead>): Promise<RunMemoryReading>;
|
|
13
|
+
/** The memory a bound holds live: `memory.stat`'s `anon` and `shmem` beside a cgroup's `memory.max`, meminfo's `AnonPages` and `Shmem` for the host's. */
|
|
14
|
+
export declare function linuxLiveUsage(read: (path: string) => FileRead, bound: string): LiveUsageReading;
|
|
13
15
|
export declare const linuxMemory: HostMemory;
|
|
14
16
|
//# sourceMappingURL=host-memory.linux.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"host-memory.linux.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.linux.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"host-memory.linux.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/host-memory.linux.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAA6B,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAclH,yEAAyE;AACzE,MAAM,MAAM,QAAQ,GAChB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;CAAE,GAC5C;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AA2B5E,2DAA2D;AAC3D,wBAAsB,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,QAAQ,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,CA8CzG;AAWD,0JAA0J;AAC1J,wBAAgB,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,QAAQ,EAAE,KAAK,EAAE,MAAM,GAAG,gBAAgB,CAuBhG;AAmDD,eAAO,MAAM,WAAW,EAAE,UAKzB,CAAC"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
1
2
|
import { readFile } from 'node:fs/promises';
|
|
2
3
|
import { posix } from 'node:path';
|
|
3
4
|
import { childPids } from '../../process-children.linux.js';
|
|
@@ -93,6 +94,42 @@ async function readText(path) {
|
|
|
93
94
|
return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
|
|
94
95
|
}
|
|
95
96
|
}
|
|
97
|
+
/** The memory a bound holds live: `memory.stat`'s `anon` and `shmem` beside a cgroup's `memory.max`, meminfo's `AnonPages` and `Shmem` for the host's. */
|
|
98
|
+
export function linuxLiveUsage(read, bound) {
|
|
99
|
+
const cgroupLimit = '/memory.max';
|
|
100
|
+
if (bound.endsWith(cgroupLimit)) {
|
|
101
|
+
const file = `${bound.slice(0, -cgroupLimit.length)}/memory.stat`;
|
|
102
|
+
const stat = read(file);
|
|
103
|
+
if (!stat.ok)
|
|
104
|
+
return { ok: false, reason: `${file}: ${stat.code}: ${stat.message}` };
|
|
105
|
+
const [anon, shmem] = [/^anon (\d+)$/m.exec(stat.text)?.[1], /^shmem (\d+)$/m.exec(stat.text)?.[1]];
|
|
106
|
+
return anon === undefined || shmem === undefined
|
|
107
|
+
? { ok: false, reason: `${file} names no anon and shmem in bytes` }
|
|
108
|
+
: { ok: true, usage: { anon: Number(anon), shmem: Number(shmem), source: file } };
|
|
109
|
+
}
|
|
110
|
+
if (bound === `${MEMINFO} MemTotal`) {
|
|
111
|
+
const info = read(MEMINFO);
|
|
112
|
+
if (!info.ok)
|
|
113
|
+
return { ok: false, reason: `${MEMINFO}: ${info.code}: ${info.message}` };
|
|
114
|
+
const [anon, shmem] = [
|
|
115
|
+
/^AnonPages:\s+(\d+) kB$/m.exec(info.text)?.[1],
|
|
116
|
+
/^Shmem:\s+(\d+) kB$/m.exec(info.text)?.[1],
|
|
117
|
+
];
|
|
118
|
+
return anon === undefined || shmem === undefined
|
|
119
|
+
? { ok: false, reason: `${MEMINFO} names no AnonPages and Shmem in kB` }
|
|
120
|
+
: { ok: true, usage: { anon: Number(anon) * 1024, shmem: Number(shmem) * 1024, source: MEMINFO } };
|
|
121
|
+
}
|
|
122
|
+
return { ok: false, reason: `${bound} is no cgroup memory.max or meminfo bound` };
|
|
123
|
+
}
|
|
124
|
+
function readTextSync(path) {
|
|
125
|
+
try {
|
|
126
|
+
return { ok: true, text: readFileSync(path, 'utf8') };
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
const code = error instanceof Error && 'code' in error && typeof error.code === 'string' ? error.code : 'EIO';
|
|
130
|
+
return { ok: false, code, message: error instanceof Error ? error.message : String(error) };
|
|
131
|
+
}
|
|
132
|
+
}
|
|
96
133
|
/** Bun before 1.4.0 reports Linux's `ru_maxrss` as the kernel does, in KiB; from 1.4.0 it reports bytes. */
|
|
97
134
|
function peakBytes(maxRss, bunVersion) {
|
|
98
135
|
const version = /^(\d+)\.(\d+)\./.exec(bunVersion);
|
|
@@ -134,4 +171,9 @@ async function residentBytes(pid) {
|
|
|
134
171
|
const kib = /^VmRSS:\s+(\d+) kB$/m.exec(status)?.[1];
|
|
135
172
|
return kib === undefined ? 0 : Number(kib) * 1024;
|
|
136
173
|
}
|
|
137
|
-
export const linuxMemory = {
|
|
174
|
+
export const linuxMemory = {
|
|
175
|
+
readRunMemory: () => linuxRunMemory(readText),
|
|
176
|
+
peakBytes,
|
|
177
|
+
treeResidentBytes,
|
|
178
|
+
liveUsage: (bound) => linuxLiveUsage(readTextSync, bound),
|
|
179
|
+
};
|
|
@@ -7,32 +7,56 @@
|
|
|
7
7
|
* Every grant needs both. The head takes `min(want, free)` tokens once one is free, and only while the charges of the
|
|
8
8
|
* leaves holding grants plus its own fit the run's memory; a head that cannot be granted holds up every request behind
|
|
9
9
|
* it, and no request holds part of a grant while it waits for the rest. A leaf is charged its task's last recorded
|
|
10
|
-
* memory peak, and nothing while its task has none. A
|
|
11
|
-
*
|
|
10
|
+
* memory peak, and nothing while its task has none. A runner's memory grows with the parallelism it is lent (a Vitest
|
|
11
|
+
* worker is a process of its own), so a peak measured while the leaf was lent `parallelism` tokens is charged in
|
|
12
|
+
* proportion to the tokens it is lent now when that is more, never less: the baseline a leaf holds however few workers
|
|
13
|
+
* it has makes a smaller grant no cheaper. When its whole ask does not fit, the head is lent the most tokens whose
|
|
14
|
+
* charge does. A leaf whose own charge alone exceeds the run's memory could never fit, so it is granted once nothing
|
|
15
|
+
* else holds memory, on one token when its peak names the parallelism it was measured under, and the grant says so.
|
|
16
|
+
*
|
|
17
|
+
* A leaf the kernel killed for memory, run again (`alone`), is granted only while nothing holds a grant, ahead of the
|
|
18
|
+
* leaves behind it, and while it runs nothing else is granted: a leaf that was killed beside others runs by itself.
|
|
19
|
+
*/
|
|
20
|
+
/**
|
|
21
|
+
* What a queued request asks: whole tokens, at least one, its charge in bytes (0 for a task with no peak), and the
|
|
22
|
+
* tokens that charge was measured under. Without them the charge is flat: it is the same however many tokens are lent.
|
|
12
23
|
*/
|
|
13
|
-
/** What a queued request asks: whole tokens, at least one, and its charge in bytes (0 for a task with no peak). */
|
|
14
24
|
export interface PoolAsk {
|
|
15
25
|
readonly want: number;
|
|
16
26
|
readonly memory: number;
|
|
27
|
+
readonly parallelism?: number | undefined;
|
|
28
|
+
/** A leaf the kernel killed for memory, run again: granted while nothing else holds a grant, and alone. */
|
|
29
|
+
readonly alone?: boolean | undefined;
|
|
30
|
+
}
|
|
31
|
+
/** Who holds a grant when the pool decides: how many leaves, and whether one of them runs alone. */
|
|
32
|
+
export interface Holding {
|
|
33
|
+
readonly holders: number;
|
|
34
|
+
readonly alone: boolean;
|
|
17
35
|
}
|
|
18
36
|
/** The run's memory a pool charges against, and the bytes its holders are charged; null for a pool that charges none. */
|
|
19
37
|
export interface MemoryLedger {
|
|
20
38
|
readonly capacity: number;
|
|
21
39
|
readonly held: number;
|
|
22
40
|
}
|
|
23
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* One request's grant: the request, its tokens, the bytes it is charged at those tokens, and whether its charge alone
|
|
43
|
+
* exceeds the run's memory.
|
|
44
|
+
*/
|
|
24
45
|
export interface PoolGrant<Ask extends PoolAsk> {
|
|
25
46
|
readonly request: Ask;
|
|
26
47
|
readonly tokens: number;
|
|
48
|
+
readonly charge: number;
|
|
27
49
|
readonly oversized: boolean;
|
|
28
50
|
}
|
|
29
51
|
/**
|
|
30
52
|
* The grants of the queue's first requests, in its order, and what the first request left waiting waits for: free
|
|
31
|
-
* tokens (`cpu`)
|
|
53
|
+
* tokens (`cpu`), the run's memory (`memory`) or the grants others hold (`alone`). `waiting` is null when every request was granted.
|
|
32
54
|
*/
|
|
33
55
|
export interface Admission<Ask extends PoolAsk> {
|
|
34
56
|
readonly grants: readonly PoolGrant<Ask>[];
|
|
35
|
-
readonly waiting: 'cpu' | 'memory' | null;
|
|
57
|
+
readonly waiting: 'cpu' | 'memory' | 'alone' | null;
|
|
36
58
|
}
|
|
37
|
-
|
|
59
|
+
/** The bytes `ask` is charged when it is lent `tokens`: its peak, scaled up when that is more than it was measured under. */
|
|
60
|
+
export declare function chargeAt(ask: PoolAsk, tokens: number): number;
|
|
61
|
+
export declare function admit<Ask extends PoolAsk>(queue: readonly Ask[], available: number, memory: MemoryLedger | null, holding?: Holding): Admission<Ask>;
|
|
38
62
|
//# sourceMappingURL=pool-admission.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pool-admission.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/pool-admission.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"pool-admission.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/pool-admission.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH;;;GAGG;AACH,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1C,2GAA2G;IAC3G,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACtC;AAED,oGAAoG;AACpG,MAAM,WAAW,OAAO;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;CACzB;AAGD,yHAAyH;AACzH,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,GAAG,SAAS,OAAO;IAC5C,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS,CAAC,GAAG,SAAS,OAAO;IAC5C,QAAQ,CAAC,MAAM,EAAE,SAAS,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;IAC3C,QAAQ,CAAC,OAAO,EAAE,KAAK,GAAG,QAAQ,GAAG,OAAO,GAAG,IAAI,CAAC;CACrD;AAED,6HAA6H;AAC7H,wBAAgB,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAG7D;AAcD,wBAAgB,KAAK,CAAC,GAAG,SAAS,OAAO,EACvC,KAAK,EAAE,SAAS,GAAG,EAAE,EACrB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,YAAY,GAAG,IAAI,EAC3B,OAAO,GAAE,OAAsB,GAC9B,SAAS,CAAC,GAAG,CAAC,CA8BhB"}
|
|
@@ -7,25 +7,61 @@
|
|
|
7
7
|
* Every grant needs both. The head takes `min(want, free)` tokens once one is free, and only while the charges of the
|
|
8
8
|
* leaves holding grants plus its own fit the run's memory; a head that cannot be granted holds up every request behind
|
|
9
9
|
* it, and no request holds part of a grant while it waits for the rest. A leaf is charged its task's last recorded
|
|
10
|
-
* memory peak, and nothing while its task has none. A
|
|
11
|
-
*
|
|
10
|
+
* memory peak, and nothing while its task has none. A runner's memory grows with the parallelism it is lent (a Vitest
|
|
11
|
+
* worker is a process of its own), so a peak measured while the leaf was lent `parallelism` tokens is charged in
|
|
12
|
+
* proportion to the tokens it is lent now when that is more, never less: the baseline a leaf holds however few workers
|
|
13
|
+
* it has makes a smaller grant no cheaper. When its whole ask does not fit, the head is lent the most tokens whose
|
|
14
|
+
* charge does. A leaf whose own charge alone exceeds the run's memory could never fit, so it is granted once nothing
|
|
15
|
+
* else holds memory, on one token when its peak names the parallelism it was measured under, and the grant says so.
|
|
16
|
+
*
|
|
17
|
+
* A leaf the kernel killed for memory, run again (`alone`), is granted only while nothing holds a grant, ahead of the
|
|
18
|
+
* leaves behind it, and while it runs nothing else is granted: a leaf that was killed beside others runs by itself.
|
|
12
19
|
*/
|
|
13
|
-
|
|
20
|
+
const NOTHING_HELD = { holders: 0, alone: false };
|
|
21
|
+
/** The bytes `ask` is charged when it is lent `tokens`: its peak, scaled up when that is more than it was measured under. */
|
|
22
|
+
export function chargeAt(ask, tokens) {
|
|
23
|
+
const measured = ask.parallelism;
|
|
24
|
+
return measured === undefined || tokens <= measured ? ask.memory : Math.ceil((ask.memory * tokens) / measured);
|
|
25
|
+
}
|
|
26
|
+
/** The most tokens, up to `most`, whose charge fits `room` bytes; none when even the peak itself does not. */
|
|
27
|
+
function tokensFitting(ask, most, room) {
|
|
28
|
+
if (ask.memory > room) {
|
|
29
|
+
return 0;
|
|
30
|
+
}
|
|
31
|
+
const measured = ask.parallelism;
|
|
32
|
+
if (measured === undefined || ask.memory === 0 || most <= measured) {
|
|
33
|
+
return most;
|
|
34
|
+
}
|
|
35
|
+
return Math.min(most, Math.floor((room * measured) / ask.memory));
|
|
36
|
+
}
|
|
37
|
+
export function admit(queue, available, memory, holding = NOTHING_HELD) {
|
|
14
38
|
const grants = [];
|
|
39
|
+
if (holding.alone) {
|
|
40
|
+
return { grants, waiting: queue.length === 0 ? null : 'alone' };
|
|
41
|
+
}
|
|
15
42
|
let free = available;
|
|
16
43
|
let held = memory?.held ?? 0;
|
|
17
44
|
for (const request of queue) {
|
|
45
|
+
if (request.alone === true && holding.holders + grants.length > 0) {
|
|
46
|
+
return { grants, waiting: 'alone' };
|
|
47
|
+
}
|
|
18
48
|
const oversized = memory !== null && request.memory > memory.capacity;
|
|
19
|
-
|
|
49
|
+
const room = memory === null ? Number.POSITIVE_INFINITY : memory.capacity - held;
|
|
50
|
+
if (request.memory > room && !(oversized && held === 0)) {
|
|
20
51
|
return { grants, waiting: 'memory' };
|
|
21
52
|
}
|
|
22
|
-
const
|
|
23
|
-
if (
|
|
53
|
+
const most = Math.min(request.want, free);
|
|
54
|
+
if (most === 0) {
|
|
24
55
|
return { grants, waiting: 'cpu' };
|
|
25
56
|
}
|
|
57
|
+
const tokens = oversized ? (request.parallelism === undefined ? most : 1) : tokensFitting(request, most, room);
|
|
58
|
+
const charge = chargeAt(request, tokens);
|
|
26
59
|
free -= tokens;
|
|
27
|
-
held +=
|
|
28
|
-
grants.push({ request, tokens, oversized });
|
|
60
|
+
held += charge;
|
|
61
|
+
grants.push({ request, tokens, charge, oversized });
|
|
62
|
+
if (request.alone === true) {
|
|
63
|
+
return { grants, waiting: grants.length < queue.length ? 'alone' : null };
|
|
64
|
+
}
|
|
29
65
|
}
|
|
30
66
|
return { grants, waiting: null };
|
|
31
67
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/run-cpu-pool.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AAGtH,OAAO,
|
|
1
|
+
{"version":3,"file":"run-cpu-pool.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/run-cpu-pool.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,yBAAyB,EAAE,wBAAwB,EAAE,MAAM,4CAA4C,CAAC;AAGtH,OAAO,EAAkB,KAAK,gBAAgB,EAAsC,MAAM,kBAAkB,CAAC;AAqD7G;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAM5F;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE,IAAI,CAAC,wBAAwB,EAAE,IAAI,CAAC,EAC7C,GAAG,GAAE,MAAM,CAAC,UAAwB,EACpC,KAAK,GAAE,MAA+B,EACtC,UAAU,GAAE,MAAM,OAAO,CAAC,gBAAgB,CAAiB,GAC1D,OAAO,CAAC,IAAI,CAAC,CAoBf;AAoED,uEAAuE;AACvE,wBAAsB,eAAe,CAAC,OAAO,EAAE,IAAI,CAAC,yBAAyB,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAUnG"}
|
|
@@ -70,7 +70,7 @@ export async function serveRunCpuPool(context, env = process.env, cores = availa
|
|
|
70
70
|
if (!memory.ok) {
|
|
71
71
|
throw new RunMemoryUnreadable(memory.error);
|
|
72
72
|
}
|
|
73
|
-
const served = await servePoolThread(tokens, memory.memory
|
|
73
|
+
const served = await servePoolThread(tokens, memory.memory);
|
|
74
74
|
if (!served.ok) {
|
|
75
75
|
logger.warn(`bounded-exec: this run's CPU pool failed (${served.error}); its tasks are sized only by the ceilings they inherit`);
|
|
76
76
|
return;
|
|
@@ -90,8 +90,9 @@ export interface BoundedExecRecord {
|
|
|
90
90
|
readonly verdict: BoundedExecVerdict;
|
|
91
91
|
/**
|
|
92
92
|
* The parent CPU pool's grant the run started under: tokens granted of those asked for, and
|
|
93
|
-
* how long it waited for the pool. Absent when it ran ungranted
|
|
94
|
-
*
|
|
93
|
+
* how long it waited for the pool. Absent when it ran ungranted. Its tokens are the parallelism the run's
|
|
94
|
+
* `peakBytes` was measured under, which the task's next run reads (`readLastPeak`) to charge that peak in proportion
|
|
95
|
+
* to the parallelism it is lent.
|
|
95
96
|
*/
|
|
96
97
|
readonly cpu?: {
|
|
97
98
|
readonly tokens: number;
|
|
@@ -237,6 +238,7 @@ export declare function readRecord(directory: string): Promise<BoundedExecRecord
|
|
|
237
238
|
export declare function readLastPeak(directory: string): Promise<{
|
|
238
239
|
ok: true;
|
|
239
240
|
peakBytes: number | null;
|
|
241
|
+
parallelism: number | null;
|
|
240
242
|
} | {
|
|
241
243
|
ok: false;
|
|
242
244
|
error: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/verdict.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAmB,KAAK,QAAQ,EAAe,MAAM,gBAAgB,CAAC;AAC7E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAGhE;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,MAAM,kBAAkB,GAC1B;IAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,GAC9B,CAAC;IAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,kBAAkB,CAAC,GAChF;IACE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,GACD;IAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACxF;IACE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,8FAA8F;IAC9F,QAAQ,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,CAAC;CACzC,GACD,CAAC;IAAE,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GAAG,eAAe,CAAC,GAClH;IAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,CAAC;AAEjG,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;CAChD;AAGD;;;;GAIG;AACH,MAAM,MAAM,eAAe,GACvB;IACE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvH,uHAAuH;AACvH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAAC;CACzC;AAED,uEAAuE;AACvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC
|
|
1
|
+
{"version":3,"file":"verdict.d.ts","sourceRoot":"","sources":["../../../src/executors/bounded-exec/verdict.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAmB,KAAK,QAAQ,EAAe,MAAM,gBAAgB,CAAC;AAC7E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAGhE;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,MAAM,kBAAkB,GAC1B;IAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAA;CAAE,GAC9B,CAAC;IAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,kBAAkB,CAAC,GAChF;IACE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,GACD;IAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GACxF;IACE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,8FAA8F;IAC9F,QAAQ,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,CAAC;CACzC,GACD,CAAC;IAAE,QAAQ,CAAC,OAAO,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,GAAG,eAAe,CAAC,GAClH;IAAE,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,CAAC;AAEjG,2FAA2F;AAC3F,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,SAAS,gBAAgB,EAAE,CAAC;CAChD;AAGD;;;;GAIG;AACH,MAAM,MAAM,eAAe,GACvB;IACE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,GACD;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvH,uHAAuH;AACvH,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAAC;CACzC;AAED,uEAAuE;AACvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IACrC;;;;;OAKG;IACH,QAAQ,CAAC,GAAG,CAAC,EAAE;QACb,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC3B,CAAC;IACF;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,UAAU,CAAC;IAC9B;;;;;OAKG;IACH,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,sFAAsF;AACtF,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B;AAED,8FAA8F;AAC9F,eAAO,MAAM,gBAAgB,uBAAuB,CAAC;AA2BrD,wEAAwE;AACxE,MAAM,WAAW,aAAa;IAC5B,oDAAoD;IACpD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,mGAAmG;IACnG,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;CACpC;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,CAwCxD;AAED,gGAAgG;AAChG,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,gBAAgB,EAAE,CAwD/D;AACD,+GAA+G;AAC/G,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,sHAAsH;AACtH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,EAAE,CAa3D;AAyCD;;;;;;;;GAQG;AACH,MAAM,MAAM,YAAY,GACpB;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACtD;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,CAAC;AAE3F,+CAA+C;AAC/C,MAAM,WAAW,MAAM;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI,CAAC;IACrC,qFAAqF;IACrF,QAAQ,CAAC,MAAM,EAAE,aAAa,GAAG,IAAI,CAAC;IACtC,uEAAuE;IACvE,QAAQ,CAAC,OAAO,EAAE,eAAe,GAAG,IAAI,CAAC;IACzC,0GAA0G;IAC1G,QAAQ,CAAC,SAAS,EAAE,SAAS,QAAQ,EAAE,CAAC;IACxC,uFAAuF;IACvF,QAAQ,CAAC,cAAc,CAAC,EAAE,kBAAkB,GAAG,IAAI,CAAC;CACrD;AAED;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,kBAAkB,CAgCxD;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,IAAI,GAAG,kBAAkB,CAG1G;AAED,+FAA+F;AAC/F,wBAAgB,aAAa,CAAC,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAE3E;AAED,eAAO,MAAM,YAAY,iBAAiB,CAAC;AAC3C,eAAO,MAAM,UAAU,eAAe,CAAC;AAGvC;;;;;;;;GAQG;AACH,eAAO,MAAM,QAAQ,QAAoE,CAAC;AAC1F,eAAO,MAAM,WAAW,QAA4C,CAAC;AACrE,eAAO,MAAM,OAAO,QAAsC,CAAC;AAE3D;;;;GAIG;AACH,eAAO,MAAM,cAAc,QAAwF,CAAC;AAEpH;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAkBzF;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEhD;AAED,wFAAwF;AACxF,wBAAsB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAQzE;AAED,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAK7F;AAED,0FAA0F;AAC1F,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC,CAWrF;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAU5G;AAED,uFAAuF;AACvF,wBAAsB,eAAe,CACnC,SAAS,EAAE,MAAM,EACjB,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,gBAAgB,EAAE,KAAK,IAAI,GACzD,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAS/B;AAwJD,0FAA0F;AAC1F,MAAM,MAAM,UAAU,GAClB;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,kBAAkB,EAAE;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;CAAE,GACpH;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9E;;;;GAIG;AACH,wBAAsB,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CA6BrG;AAED,kFAAkF;AAClF,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,kBAAkB,GAAG,MAAM,CAQvE;AACD,wBAAgB,eAAe,CAAC,OAAO,EAAE,kBAAkB,GAAG,MAAM,CA2BnE;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,eAAe,GAAG,MAAM,CAgBjE"}
|
|
@@ -327,7 +327,8 @@ export async function readRecord(directory) {
|
|
|
327
327
|
*/
|
|
328
328
|
export async function readLastPeak(directory) {
|
|
329
329
|
try {
|
|
330
|
-
|
|
330
|
+
const record = await readRecord(directory);
|
|
331
|
+
return { ok: true, peakBytes: record?.peakBytes ?? null, parallelism: record?.cpu?.tokens ?? null };
|
|
331
332
|
}
|
|
332
333
|
catch (error) {
|
|
333
334
|
return {
|
|
@@ -361,7 +362,23 @@ function parseRecord(value) {
|
|
|
361
362
|
if (!(peakBytes === null || (typeof peakBytes === 'number' && Number.isSafeInteger(peakBytes) && peakBytes >= 0))) {
|
|
362
363
|
throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
|
|
363
364
|
}
|
|
364
|
-
|
|
365
|
+
const cpu = value.cpu;
|
|
366
|
+
if (cpu === undefined) {
|
|
367
|
+
return { task: value.task, hash: value.hash, verdict: parseVerdict(value.verdict), peakBytes };
|
|
368
|
+
}
|
|
369
|
+
if (!isObject(cpu) ||
|
|
370
|
+
!(typeof cpu.tokens === 'number' && Number.isSafeInteger(cpu.tokens) && cpu.tokens >= 1) ||
|
|
371
|
+
!(typeof cpu.want === 'number' && Number.isSafeInteger(cpu.want) && cpu.want >= 1) ||
|
|
372
|
+
!(typeof cpu.waitedMs === 'number' && Number.isFinite(cpu.waitedMs) && cpu.waitedMs >= 0)) {
|
|
373
|
+
throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
|
|
374
|
+
}
|
|
375
|
+
return {
|
|
376
|
+
task: value.task,
|
|
377
|
+
hash: value.hash,
|
|
378
|
+
verdict: parseVerdict(value.verdict),
|
|
379
|
+
cpu: { tokens: cpu.tokens, want: cpu.want, waitedMs: cpu.waitedMs },
|
|
380
|
+
peakBytes,
|
|
381
|
+
};
|
|
365
382
|
}
|
|
366
383
|
function parseVerdict(value) {
|
|
367
384
|
if (isObject(value)) {
|
|
@@ -4000,6 +4000,22 @@ index 2000bbcd5d707c5435c2bc47573528d7a31ae606..524a82cae46a6028baceaa985097fe59
|
|
|
4000
4000
|
}
|
|
4001
4001
|
function setErrorTimeout(timeoutMs, errorMessage) {
|
|
4002
4002
|
if (environment.NX_PLUGIN_NO_TIMEOUTS === 'true') {
|
|
4003
|
+
diff --git a/dist/src/project-graph/plugins/loaded-nx-plugin.js b/dist/src/project-graph/plugins/loaded-nx-plugin.js
|
|
4004
|
+
index 9fa6b993ac4dbe918c163092c12f77f87eaccb27..d93538310c4dc2622cde88cd9da983bcc1e3e63c 100644
|
|
4005
|
+
--- a/dist/src/project-graph/plugins/loaded-nx-plugin.js
|
|
4006
|
+
+++ b/dist/src/project-graph/plugins/loaded-nx-plugin.js
|
|
4007
|
+
@@ -85,7 +85,10 @@ class LoadedNxPlugin {
|
|
4008
|
+
this.preTasksExecution = async (context) => {
|
|
4009
|
+
const updates = {};
|
|
4010
|
+
let originalEnv = process.env;
|
|
4011
|
+
- if ((0, enabled_1.isIsolationEnabled)() || (0, client_1.isDaemonEnabled)()) {
|
|
4012
|
+
+ // A plugin worker records the hook's writes whatever the isolation setting and the daemon say: the
|
|
4013
|
+
+ // patched Nx loads a local plugin in one under every setting, and these recorded writes are how the
|
|
4014
|
+
+ // hook's environment reaches the client that applies it.
|
|
4015
|
+
+ if ((0, enabled_1.isIsolationEnabled)() || (0, client_1.isDaemonEnabled)() || global.NX_PLUGIN_WORKER) {
|
|
4016
|
+
process.env = new Proxy(originalEnv, {
|
|
4017
|
+
set: (target, key, value) => {
|
|
4018
|
+
target[key] = value;
|
|
4003
4019
|
diff --git a/dist/src/project-graph/plugins/local-plugin-source.d.ts b/dist/src/project-graph/plugins/local-plugin-source.d.ts
|
|
4004
4020
|
new file mode 100644
|
|
4005
4021
|
index 0000000000000000000000000000000000000000..1a84106509d0e05a2f0d6c595b79ecc781ff21d1
|
|
@@ -73,6 +73,130 @@ index 0000000000000000000000000000000000000000..97d1522f7979b488dc62b86cc1d78d72
|
|
|
73
73
|
+ t.Fatalf("forced emit did not rewrite the runtime import: %s", output["index.js"])
|
|
74
74
|
+ }
|
|
75
75
|
+}
|
|
76
|
+
diff --git a/driver/emit_plugin.go b/driver/emit_plugin.go
|
|
77
|
+
index 3793b3a406c2bf2643567a755d34e66a26eac5c8..3c690ac3d9a99be7914ad9aed76e5890d236ea9b 100644
|
|
78
|
+
--- a/driver/emit_plugin.go
|
|
79
|
+
+++ b/driver/emit_plugin.go
|
|
80
|
+
@@ -427,7 +427,13 @@ func (p *Program) EmitWithPluginTransformers(transforms []PluginTransform, write
|
|
81
|
+
return diagnostics, err
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
- return diagnostics, output.flush()
|
|
85
|
+
+ if err := output.flush(); err != nil {
|
|
86
|
+
+ return diagnostics, err
|
|
87
|
+
+ }
|
|
88
|
+
+ // Files linked plugins reported writing beside these outputs follow them
|
|
89
|
+
+ // through the host's writer, so the host lists them too (ReportHostOutput).
|
|
90
|
+
+ _, err = p.writeHostOutputs(writeFile)
|
|
91
|
+
+ return diagnostics, err
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// writePluginEmitOutput writes one artifact of the hand-assembled emit, passing
|
|
95
|
+
diff --git a/driver/plugins.go b/driver/plugins.go
|
|
96
|
+
index 1778e250d80d1316e108f1f36b6e1a38724922d5..da2ec288afa9a4ae8ceebc1728f6119bae676c4a 100644
|
|
97
|
+
--- a/driver/plugins.go
|
|
98
|
+
+++ b/driver/plugins.go
|
|
99
|
+
@@ -169,6 +169,7 @@ type PluginContext struct {
|
|
100
|
+
reportFileDependencyRejected func(string)
|
|
101
|
+
reportFileComplete func(string)
|
|
102
|
+
reportEveryFileComplete func()
|
|
103
|
+
+ reportHostOutput func(string)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// ReportObservationIncomplete declares that this hook's input population could
|
|
107
|
+
@@ -195,6 +196,24 @@ func (ctx PluginContext) ReportObservationIncomplete() {
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
+// ReportHostOutput declares a file this hook wrote beside the compiler's
|
|
112
|
+
+// outputs, such as a census or a manifest in outDir. A later emit of this
|
|
113
|
+
+// generation hands the file's bytes as they are then to the host's writer,
|
|
114
|
+
+// after the compiler's own outputs, so every host lists it with them: a caller
|
|
115
|
+
+// that keeps exactly the files an emit lists (a build that prunes its outDir by
|
|
116
|
+
+// that list) keeps it too. A relative file is resolved against Cwd. An emit
|
|
117
|
+
+// fails when a reported file cannot be read. A context without a driver
|
|
118
|
+
+// callback ignores the report.
|
|
119
|
+
+func (ctx PluginContext) ReportHostOutput(file string) {
|
|
120
|
+
+ if ctx.reportHostOutput == nil || strings.TrimSpace(file) == "" {
|
|
121
|
+
+ return
|
|
122
|
+
+ }
|
|
123
|
+
+ if !filepath.IsAbs(file) {
|
|
124
|
+
+ file = filepath.Join(ctx.Cwd, file)
|
|
125
|
+
+ }
|
|
126
|
+
+ ctx.reportHostOutput(filepath.Clean(file))
|
|
127
|
+
+}
|
|
128
|
+
+
|
|
129
|
+
// ReportHostInputHash declares the exact file state consumed by a native
|
|
130
|
+
// plugin. hash is a lowercase SHA-256 digest for an observed file and nil for
|
|
131
|
+
// a missing candidate. Conflicting observations are retained as host inputs
|
|
132
|
+
@@ -511,6 +530,7 @@ type linkedPluginState struct {
|
|
133
|
+
declarations *pluginFileDeclarations
|
|
134
|
+
entries []PluginEntry
|
|
135
|
+
inputs *pluginHostInputScopes
|
|
136
|
+
+ outputs *pluginHostOutputs
|
|
137
|
+
tsconfig string
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
@@ -800,6 +820,7 @@ func loadLinkedPluginState(cwd, tsconfigPath string) (linkedPluginState, error)
|
|
141
|
+
cwd: cwd,
|
|
142
|
+
declarations: newPluginFileDeclarations(),
|
|
143
|
+
inputs: newPluginHostInputScopes(),
|
|
144
|
+
+ outputs: newPluginHostOutputs(),
|
|
145
|
+
tsconfig: tsconfigPath,
|
|
146
|
+
}, nil
|
|
147
|
+
}
|
|
148
|
+
@@ -812,6 +833,7 @@ func loadLinkedPluginState(cwd, tsconfigPath string) (linkedPluginState, error)
|
|
149
|
+
declarations: newPluginFileDeclarations(),
|
|
150
|
+
entries: entries,
|
|
151
|
+
inputs: newPluginHostInputScopes(),
|
|
152
|
+
+ outputs: newPluginHostOutputs(),
|
|
153
|
+
tsconfig: tsconfigPath,
|
|
154
|
+
}, nil
|
|
155
|
+
}
|
|
156
|
+
@@ -931,7 +953,43 @@ func (state linkedPluginState) contextAt(index int, entry PluginEntry) PluginCon
|
|
157
|
+
reportFileDependencyRejected: declaration.rejectDependency,
|
|
158
|
+
reportFileComplete: declaration.addComplete,
|
|
159
|
+
reportEveryFileComplete: declaration.completeEveryFile,
|
|
160
|
+
+ reportHostOutput: state.outputs.add,
|
|
161
|
+
+ }
|
|
162
|
+
+}
|
|
163
|
+
+
|
|
164
|
+
+// pluginHostOutputs keeps the files linked plugins reported writing beside the
|
|
165
|
+
+// compiler's outputs in this generation (ReportHostOutput), each once, in the
|
|
166
|
+
+// order first reported.
|
|
167
|
+
+type pluginHostOutputs struct {
|
|
168
|
+
+ mu sync.Mutex
|
|
169
|
+
+ seen map[string]struct{}
|
|
170
|
+
+ files []string
|
|
171
|
+
+}
|
|
172
|
+
+
|
|
173
|
+
+func newPluginHostOutputs() *pluginHostOutputs {
|
|
174
|
+
+ return &pluginHostOutputs{seen: map[string]struct{}{}}
|
|
175
|
+
+}
|
|
176
|
+
+
|
|
177
|
+
+func (outputs *pluginHostOutputs) add(file string) {
|
|
178
|
+
+ if outputs == nil {
|
|
179
|
+
+ return
|
|
180
|
+
+ }
|
|
181
|
+
+ outputs.mu.Lock()
|
|
182
|
+
+ defer outputs.mu.Unlock()
|
|
183
|
+
+ if _, ok := outputs.seen[file]; ok {
|
|
184
|
+
+ return
|
|
185
|
+
+ }
|
|
186
|
+
+ outputs.seen[file] = struct{}{}
|
|
187
|
+
+ outputs.files = append(outputs.files, file)
|
|
188
|
+
+}
|
|
189
|
+
+
|
|
190
|
+
+func (outputs *pluginHostOutputs) list() []string {
|
|
191
|
+
+ if outputs == nil {
|
|
192
|
+
+ return nil
|
|
193
|
+
}
|
|
194
|
+
+ outputs.mu.Lock()
|
|
195
|
+
+ defer outputs.mu.Unlock()
|
|
196
|
+
+ return append([]string(nil), outputs.files...)
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// hostInputs returns the exact native configuration inputs reported in this
|
|
76
200
|
diff --git a/driver/program.go b/driver/program.go
|
|
77
201
|
index f6ee81fb1af41deea022b92e838a7c4665eb9778..112faf913ce38b1f4624b38db37839370b9c5e8b 100644
|
|
78
202
|
--- a/driver/program.go
|
|
@@ -91,10 +215,73 @@ index f6ee81fb1af41deea022b92e838a7c4665eb9778..112faf913ce38b1f4624b38db3783937
|
|
|
91
215
|
|
|
92
216
|
// forceNoEmit sets noEmit so the program type-checks without writing files.
|
|
93
217
|
diff --git a/driver/rewrite.go b/driver/rewrite.go
|
|
94
|
-
index 79be74d24e879da216ad680d23ecfeb5f11a29db..
|
|
218
|
+
index 79be74d24e879da216ad680d23ecfeb5f11a29db..6db9dddba05bd50e13ed92535ca442c3a4f0b28a 100644
|
|
95
219
|
--- a/driver/rewrite.go
|
|
96
220
|
+++ b/driver/rewrite.go
|
|
97
|
-
@@ -
|
|
221
|
+
@@ -204,6 +204,9 @@ func (p *Program) EmitAllRaw(writeFile shimcompiler.WriteFile) (*shimcompiler.Em
|
|
222
|
+
result := p.emitProgram(shimcompiler.EmitOptions{
|
|
223
|
+
WriteFile: wf,
|
|
224
|
+
})
|
|
225
|
+
+ if err := p.listHostOutputs(result, writeFile); err != nil {
|
|
226
|
+
+ return result, p.convertProgramDiagnostics(result.Diagnostics), err
|
|
227
|
+
+ }
|
|
228
|
+
return result, p.convertProgramDiagnostics(result.Diagnostics), nil
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
@@ -271,9 +274,52 @@ func (p *Program) emit(rs *RewriteSet, target *ast.SourceFile, writeFile shimcom
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
result := p.emitProgram(shimcompiler.TargetedEmitOptions(target, wf))
|
|
235
|
+
+ if target == nil {
|
|
236
|
+
+ if err := p.listHostOutputs(result, writeFile); err != nil {
|
|
237
|
+
+ return result, p.convertProgramDiagnostics(result.Diagnostics), err
|
|
238
|
+
+ }
|
|
239
|
+
+ }
|
|
240
|
+
return result, p.convertProgramDiagnostics(result.Diagnostics), nil
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
+// listHostOutputs hands the files linked plugins reported writing beside a
|
|
244
|
+
+// whole-program emit's outputs to writeFile and lists them among the result's
|
|
245
|
+
+// EmittedFiles, unless the emit wrote nothing (noEmit, or noEmitOnError
|
|
246
|
+
+// withholding a failed program).
|
|
247
|
+
+func (p *Program) listHostOutputs(result *shimcompiler.EmitResult, writeFile shimcompiler.WriteFile) error {
|
|
248
|
+
+ if result == nil || result.EmitSkipped || p.TSProgram.Options().NoEmit.IsTrue() {
|
|
249
|
+
+ return nil
|
|
250
|
+
+ }
|
|
251
|
+
+ handed, err := p.writeHostOutputs(writeFile)
|
|
252
|
+
+ result.EmittedFiles = append(result.EmittedFiles, handed...)
|
|
253
|
+
+ return err
|
|
254
|
+
+}
|
|
255
|
+
+
|
|
256
|
+
+// writeHostOutputs hands each file a linked plugin reported writing beside the
|
|
257
|
+
+// compiler's outputs (PluginContext.ReportHostOutput) to writeFile, with its
|
|
258
|
+
+// bytes as they are now, after the compiler's own outputs, so a host lists it
|
|
259
|
+
+// with them. It answers the files handed, and fails on one it cannot read: a
|
|
260
|
+
+// plugin reported an output it did not leave.
|
|
261
|
+
+func (p *Program) writeHostOutputs(writeFile shimcompiler.WriteFile) ([]string, error) {
|
|
262
|
+
+ var handed []string
|
|
263
|
+
+ for _, file := range p.plugins.outputs.list() {
|
|
264
|
+
+ text, err := os.ReadFile(file)
|
|
265
|
+
+ if err != nil {
|
|
266
|
+
+ return handed, fmt.Errorf("ttsc driver: a linked plugin reported the host output %s, which cannot be read: %w", file, err)
|
|
267
|
+
+ }
|
|
268
|
+
+ if writeFile != nil {
|
|
269
|
+
+ err = writeFile(shimtspath.RootedFilePathFromAbsolute(file), string(text), nil)
|
|
270
|
+
+ } else {
|
|
271
|
+
+ err = DefaultWriteFile(file, string(text))
|
|
272
|
+
+ }
|
|
273
|
+
+ if err != nil {
|
|
274
|
+
+ return handed, err
|
|
275
|
+
+ }
|
|
276
|
+
+ handed = append(handed, file)
|
|
277
|
+
+ }
|
|
278
|
+
+ return handed, nil
|
|
279
|
+
+}
|
|
280
|
+
+
|
|
281
|
+
// emitProgram runs one whole-program or single-file emit, taking tsgo's
|
|
282
|
+
// incremental lane when the resolved compiler options ask for build
|
|
283
|
+
// information.
|
|
284
|
+
@@ -317,22 +363,28 @@ func (p *Program) emitsBuildInfo() bool {
|
|
98
285
|
}
|
|
99
286
|
|
|
100
287
|
// DefaultWriteFile is the default disk writer used when EmitAll's caller does not
|