@smoothbricks/nx-plugin 0.7.10 → 0.7.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -21
- package/dist/.build-inputs.json +9 -9
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
- package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.d.ts +25 -10
- package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/cpu-pool.js +86 -23
- package/dist/executors/bounded-exec/executor.d.ts +5 -0
- package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
- package/dist/executors/bounded-exec/executor.js +44 -5
- package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
- package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
- package/dist/executors/bounded-exec/host-memory.js +6 -0
- package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
- package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
- package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
- package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
- package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
- package/dist/executors/bounded-exec/pool-admission.js +44 -8
- package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
- package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
- package/dist/executors/bounded-exec/verdict.d.ts +4 -2
- package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
- package/dist/executors/bounded-exec/verdict.js +19 -2
- package/managed/raw/patches/nx@24.0.3.patch +16 -0
- package/managed/raw/patches/ttsc@0.34.1.patch +189 -2
- package/package.json +1 -1
- package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
- package/src/executors/bounded-exec/cpu-pool.ts +121 -25
- package/src/executors/bounded-exec/executor.ts +71 -5
- package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
- package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
- package/src/executors/bounded-exec/host-memory.ts +25 -0
- package/src/executors/bounded-exec/pool-admission.ts +66 -11
- package/src/executors/bounded-exec/run-cpu-pool.ts +3 -3
- package/src/executors/bounded-exec/verdict.ts +25 -5
- package/test-timings.json +2 -0
|
@@ -7,58 +7,113 @@
|
|
|
7
7
|
* Every grant needs both. The head takes `min(want, free)` tokens once one is free, and only while the charges of the
|
|
8
8
|
* leaves holding grants plus its own fit the run's memory; a head that cannot be granted holds up every request behind
|
|
9
9
|
* it, and no request holds part of a grant while it waits for the rest. A leaf is charged its task's last recorded
|
|
10
|
-
* memory peak, and nothing while its task has none. A
|
|
11
|
-
*
|
|
10
|
+
* memory peak, and nothing while its task has none. A runner's memory grows with the parallelism it is lent (a Vitest
|
|
11
|
+
* worker is a process of its own), so a peak measured while the leaf was lent `parallelism` tokens is charged in
|
|
12
|
+
* proportion to the tokens it is lent now when that is more, never less: the baseline a leaf holds however few workers
|
|
13
|
+
* it has makes a smaller grant no cheaper. When its whole ask does not fit, the head is lent the most tokens whose
|
|
14
|
+
* charge does. A leaf whose own charge alone exceeds the run's memory could never fit, so it is granted once nothing
|
|
15
|
+
* else holds memory, on one token when its peak names the parallelism it was measured under, and the grant says so.
|
|
16
|
+
*
|
|
17
|
+
* A leaf the kernel killed for memory, run again (`alone`), is granted only while nothing holds a grant, ahead of the
|
|
18
|
+
* leaves behind it, and while it runs nothing else is granted: a leaf that was killed beside others runs by itself.
|
|
12
19
|
*/
|
|
13
20
|
|
|
14
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* What a queued request asks: whole tokens, at least one, its charge in bytes (0 for a task with no peak), and the
|
|
23
|
+
* tokens that charge was measured under. Without them the charge is flat: it is the same however many tokens are lent.
|
|
24
|
+
*/
|
|
15
25
|
export interface PoolAsk {
|
|
16
26
|
readonly want: number;
|
|
17
27
|
readonly memory: number;
|
|
28
|
+
readonly parallelism?: number | undefined;
|
|
29
|
+
/** A leaf the kernel killed for memory, run again: granted while nothing else holds a grant, and alone. */
|
|
30
|
+
readonly alone?: boolean | undefined;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** Who holds a grant when the pool decides: how many leaves, and whether one of them runs alone. */
|
|
34
|
+
export interface Holding {
|
|
35
|
+
readonly holders: number;
|
|
36
|
+
readonly alone: boolean;
|
|
18
37
|
}
|
|
19
38
|
|
|
39
|
+
const NOTHING_HELD: Holding = { holders: 0, alone: false };
|
|
20
40
|
/** The run's memory a pool charges against, and the bytes its holders are charged; null for a pool that charges none. */
|
|
21
41
|
export interface MemoryLedger {
|
|
22
42
|
readonly capacity: number;
|
|
23
43
|
readonly held: number;
|
|
24
44
|
}
|
|
25
45
|
|
|
26
|
-
/**
|
|
46
|
+
/**
|
|
47
|
+
* One request's grant: the request, its tokens, the bytes it is charged at those tokens, and whether its charge alone
|
|
48
|
+
* exceeds the run's memory.
|
|
49
|
+
*/
|
|
27
50
|
export interface PoolGrant<Ask extends PoolAsk> {
|
|
28
51
|
readonly request: Ask;
|
|
29
52
|
readonly tokens: number;
|
|
53
|
+
readonly charge: number;
|
|
30
54
|
readonly oversized: boolean;
|
|
31
55
|
}
|
|
32
56
|
|
|
33
57
|
/**
|
|
34
58
|
* The grants of the queue's first requests, in its order, and what the first request left waiting waits for: free
|
|
35
|
-
* tokens (`cpu`)
|
|
59
|
+
* tokens (`cpu`), the run's memory (`memory`) or the grants others hold (`alone`). `waiting` is null when every request was granted.
|
|
36
60
|
*/
|
|
37
61
|
export interface Admission<Ask extends PoolAsk> {
|
|
38
62
|
readonly grants: readonly PoolGrant<Ask>[];
|
|
39
|
-
readonly waiting: 'cpu' | 'memory' | null;
|
|
63
|
+
readonly waiting: 'cpu' | 'memory' | 'alone' | null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** The bytes `ask` is charged when it is lent `tokens`: its peak, scaled up when that is more than it was measured under. */
|
|
67
|
+
export function chargeAt(ask: PoolAsk, tokens: number): number {
|
|
68
|
+
const measured = ask.parallelism;
|
|
69
|
+
return measured === undefined || tokens <= measured ? ask.memory : Math.ceil((ask.memory * tokens) / measured);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The most tokens, up to `most`, whose charge fits `room` bytes; none when even the peak itself does not. */
|
|
73
|
+
function tokensFitting(ask: PoolAsk, most: number, room: number): number {
|
|
74
|
+
if (ask.memory > room) {
|
|
75
|
+
return 0;
|
|
76
|
+
}
|
|
77
|
+
const measured = ask.parallelism;
|
|
78
|
+
if (measured === undefined || ask.memory === 0 || most <= measured) {
|
|
79
|
+
return most;
|
|
80
|
+
}
|
|
81
|
+
return Math.min(most, Math.floor((room * measured) / ask.memory));
|
|
40
82
|
}
|
|
41
83
|
|
|
42
84
|
export function admit<Ask extends PoolAsk>(
|
|
43
85
|
queue: readonly Ask[],
|
|
44
86
|
available: number,
|
|
45
87
|
memory: MemoryLedger | null,
|
|
88
|
+
holding: Holding = NOTHING_HELD,
|
|
46
89
|
): Admission<Ask> {
|
|
47
90
|
const grants: PoolGrant<Ask>[] = [];
|
|
91
|
+
if (holding.alone) {
|
|
92
|
+
return { grants, waiting: queue.length === 0 ? null : 'alone' };
|
|
93
|
+
}
|
|
48
94
|
let free = available;
|
|
49
95
|
let held = memory?.held ?? 0;
|
|
50
96
|
for (const request of queue) {
|
|
97
|
+
if (request.alone === true && holding.holders + grants.length > 0) {
|
|
98
|
+
return { grants, waiting: 'alone' };
|
|
99
|
+
}
|
|
51
100
|
const oversized = memory !== null && request.memory > memory.capacity;
|
|
52
|
-
|
|
101
|
+
const room = memory === null ? Number.POSITIVE_INFINITY : memory.capacity - held;
|
|
102
|
+
if (request.memory > room && !(oversized && held === 0)) {
|
|
53
103
|
return { grants, waiting: 'memory' };
|
|
54
104
|
}
|
|
55
|
-
const
|
|
56
|
-
if (
|
|
105
|
+
const most = Math.min(request.want, free);
|
|
106
|
+
if (most === 0) {
|
|
57
107
|
return { grants, waiting: 'cpu' };
|
|
58
108
|
}
|
|
109
|
+
const tokens = oversized ? (request.parallelism === undefined ? most : 1) : tokensFitting(request, most, room);
|
|
110
|
+
const charge = chargeAt(request, tokens);
|
|
59
111
|
free -= tokens;
|
|
60
|
-
held +=
|
|
61
|
-
grants.push({ request, tokens, oversized });
|
|
112
|
+
held += charge;
|
|
113
|
+
grants.push({ request, tokens, charge, oversized });
|
|
114
|
+
if (request.alone === true) {
|
|
115
|
+
return { grants, waiting: grants.length < queue.length ? 'alone' : null };
|
|
116
|
+
}
|
|
62
117
|
}
|
|
63
118
|
return { grants, waiting: null };
|
|
64
119
|
}
|
|
@@ -5,7 +5,7 @@ import { logger } from 'nx/src/devkit-exports.js';
|
|
|
5
5
|
import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
|
|
6
6
|
import { type CloseOutcome, type CpuPool, readInheritedCpuPool } from './cpu-pool.js';
|
|
7
7
|
import type { CpuPoolThreadData, CpuPoolThreadReport } from './cpu-pool-thread.js';
|
|
8
|
-
import { type RunMemoryReading, RunMemoryUnreadable, readRunMemory } from './host-memory.js';
|
|
8
|
+
import { type RunMemory, type RunMemoryReading, RunMemoryUnreadable, readRunMemory } from './host-memory.js';
|
|
9
9
|
import { mbxPool } from './mbx-pool.js';
|
|
10
10
|
import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
|
|
11
11
|
|
|
@@ -91,7 +91,7 @@ export async function serveRunCpuPool(
|
|
|
91
91
|
if (!memory.ok) {
|
|
92
92
|
throw new RunMemoryUnreadable(memory.error);
|
|
93
93
|
}
|
|
94
|
-
const served = await servePoolThread(tokens, memory.memory
|
|
94
|
+
const served = await servePoolThread(tokens, memory.memory);
|
|
95
95
|
if (!served.ok) {
|
|
96
96
|
logger.warn(
|
|
97
97
|
`bounded-exec: this run's CPU pool failed (${served.error}); its tasks are sized only by the ceilings they inherit`,
|
|
@@ -110,7 +110,7 @@ export async function serveRunCpuPool(
|
|
|
110
110
|
* has said how its pool closed fails whichever wait is open, a later close returns that failure, and a thread lost while
|
|
111
111
|
* its pool serves is said at once.
|
|
112
112
|
*/
|
|
113
|
-
function servePoolThread(tokens: number, memory:
|
|
113
|
+
function servePoolThread(tokens: number, memory: RunMemory): Promise<ServedPool> {
|
|
114
114
|
const thread = new Worker(THREAD_ENTRY, {
|
|
115
115
|
workerData: { tokens, memory } satisfies CpuPoolThreadData,
|
|
116
116
|
...THREAD_PRELOAD,
|
|
@@ -87,8 +87,9 @@ export interface BoundedExecRecord {
|
|
|
87
87
|
readonly verdict: BoundedExecVerdict;
|
|
88
88
|
/**
|
|
89
89
|
* The parent CPU pool's grant the run started under: tokens granted of those asked for, and
|
|
90
|
-
* how long it waited for the pool. Absent when it ran ungranted
|
|
91
|
-
*
|
|
90
|
+
* how long it waited for the pool. Absent when it ran ungranted. Its tokens are the parallelism the run's
|
|
91
|
+
* `peakBytes` was measured under, which the task's next run reads (`readLastPeak`) to charge that peak in proportion
|
|
92
|
+
* to the parallelism it is lent.
|
|
92
93
|
*/
|
|
93
94
|
readonly cpu?: {
|
|
94
95
|
readonly tokens: number;
|
|
@@ -501,9 +502,10 @@ export async function readRecord(directory: string): Promise<BoundedExecRecord |
|
|
|
501
502
|
*/
|
|
502
503
|
export async function readLastPeak(
|
|
503
504
|
directory: string,
|
|
504
|
-
): Promise<{ ok: true; peakBytes: number | null } | { ok: false; error: string }> {
|
|
505
|
+
): Promise<{ ok: true; peakBytes: number | null; parallelism: number | null } | { ok: false; error: string }> {
|
|
505
506
|
try {
|
|
506
|
-
|
|
507
|
+
const record = await readRecord(directory);
|
|
508
|
+
return { ok: true, peakBytes: record?.peakBytes ?? null, parallelism: record?.cpu?.tokens ?? null };
|
|
507
509
|
} catch (error) {
|
|
508
510
|
return {
|
|
509
511
|
ok: false,
|
|
@@ -540,7 +542,25 @@ function parseRecord(value: unknown): BoundedExecRecord {
|
|
|
540
542
|
if (!(peakBytes === null || (typeof peakBytes === 'number' && Number.isSafeInteger(peakBytes) && peakBytes >= 0))) {
|
|
541
543
|
throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
|
|
542
544
|
}
|
|
543
|
-
|
|
545
|
+
const cpu = value.cpu;
|
|
546
|
+
if (cpu === undefined) {
|
|
547
|
+
return { task: value.task, hash: value.hash, verdict: parseVerdict(value.verdict), peakBytes };
|
|
548
|
+
}
|
|
549
|
+
if (
|
|
550
|
+
!isObject(cpu) ||
|
|
551
|
+
!(typeof cpu.tokens === 'number' && Number.isSafeInteger(cpu.tokens) && cpu.tokens >= 1) ||
|
|
552
|
+
!(typeof cpu.want === 'number' && Number.isSafeInteger(cpu.want) && cpu.want >= 1) ||
|
|
553
|
+
!(typeof cpu.waitedMs === 'number' && Number.isFinite(cpu.waitedMs) && cpu.waitedMs >= 0)
|
|
554
|
+
) {
|
|
555
|
+
throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
|
|
556
|
+
}
|
|
557
|
+
return {
|
|
558
|
+
task: value.task,
|
|
559
|
+
hash: value.hash,
|
|
560
|
+
verdict: parseVerdict(value.verdict),
|
|
561
|
+
cpu: { tokens: cpu.tokens, want: cpu.want, waitedMs: cpu.waitedMs },
|
|
562
|
+
peakBytes,
|
|
563
|
+
};
|
|
544
564
|
}
|
|
545
565
|
|
|
546
566
|
function parseVerdict(value: unknown): BoundedExecVerdict {
|
package/test-timings.json
CHANGED
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
"src/owned-installation.test.ts": 28757,
|
|
11
11
|
"src/index.test.ts": 26774,
|
|
12
12
|
"src/executors/bounded-exec/executor.test.ts": 25285,
|
|
13
|
+
"src/executors/bounded-exec/pool-sim.property.test.ts": 23356,
|
|
13
14
|
"src/ensure-built-receipt.test.ts": 21873,
|
|
14
15
|
"src/nx-daemon-idle-policy.test.ts": 18587,
|
|
15
16
|
"src/cargo-retention.test.ts": 16827,
|
|
@@ -24,6 +25,7 @@
|
|
|
24
25
|
"src/typecheck-test-policy.test.ts": 5606,
|
|
25
26
|
"src/bounded-test-policy.test.ts": 5412,
|
|
26
27
|
"src/nx-daemon-start-never.test.ts": 5356,
|
|
28
|
+
"src/nx-local-plugin-hook-env.test.ts": 5300,
|
|
27
29
|
"src/executors/bounded-exec/ram-temp.test.ts": 5259,
|
|
28
30
|
"src/task-results.test.ts": 5234,
|
|
29
31
|
"src/nx-dot-input-policy.test.ts": 5127,
|