@smoothbricks/nx-plugin 0.7.10 → 0.7.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +30 -21
  2. package/dist/.build-inputs.json +9 -9
  3. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts +5 -4
  4. package/dist/executors/bounded-exec/cpu-pool-thread.d.ts.map +1 -1
  5. package/dist/executors/bounded-exec/cpu-pool.d.ts +25 -10
  6. package/dist/executors/bounded-exec/cpu-pool.d.ts.map +1 -1
  7. package/dist/executors/bounded-exec/cpu-pool.js +86 -23
  8. package/dist/executors/bounded-exec/executor.d.ts +5 -0
  9. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  10. package/dist/executors/bounded-exec/executor.js +44 -5
  11. package/dist/executors/bounded-exec/host-memory.d.ts +22 -0
  12. package/dist/executors/bounded-exec/host-memory.d.ts.map +1 -1
  13. package/dist/executors/bounded-exec/host-memory.darwin.d.ts.map +1 -1
  14. package/dist/executors/bounded-exec/host-memory.darwin.js +7 -1
  15. package/dist/executors/bounded-exec/host-memory.js +6 -0
  16. package/dist/executors/bounded-exec/host-memory.linux.d.ts +3 -1
  17. package/dist/executors/bounded-exec/host-memory.linux.d.ts.map +1 -1
  18. package/dist/executors/bounded-exec/host-memory.linux.js +43 -1
  19. package/dist/executors/bounded-exec/pool-admission.d.ts +31 -7
  20. package/dist/executors/bounded-exec/pool-admission.d.ts.map +1 -1
  21. package/dist/executors/bounded-exec/pool-admission.js +44 -8
  22. package/dist/executors/bounded-exec/run-cpu-pool.d.ts.map +1 -1
  23. package/dist/executors/bounded-exec/run-cpu-pool.js +1 -1
  24. package/dist/executors/bounded-exec/verdict.d.ts +4 -2
  25. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  26. package/dist/executors/bounded-exec/verdict.js +19 -2
  27. package/managed/raw/patches/nx@24.0.3.patch +16 -0
  28. package/managed/raw/patches/ttsc@0.34.1.patch +189 -2
  29. package/package.json +1 -1
  30. package/src/executors/bounded-exec/cpu-pool-thread.ts +5 -4
  31. package/src/executors/bounded-exec/cpu-pool.ts +121 -25
  32. package/src/executors/bounded-exec/executor.ts +71 -5
  33. package/src/executors/bounded-exec/host-memory.darwin.ts +7 -1
  34. package/src/executors/bounded-exec/host-memory.linux.ts +42 -2
  35. package/src/executors/bounded-exec/host-memory.ts +25 -0
  36. package/src/executors/bounded-exec/pool-admission.ts +66 -11
  37. package/src/executors/bounded-exec/run-cpu-pool.ts +3 -3
  38. package/src/executors/bounded-exec/verdict.ts +25 -5
  39. package/test-timings.json +2 -0
@@ -7,58 +7,113 @@
7
7
  * Every grant needs both. The head takes `min(want, free)` tokens once one is free, and only while the charges of the
8
8
  * leaves holding grants plus its own fit the run's memory; a head that cannot be granted holds up every request behind
9
9
  * it, and no request holds part of a grant while it waits for the rest. A leaf is charged its task's last recorded
10
- * memory peak, and nothing while its task has none. A leaf whose own charge alone exceeds the run's memory could never
11
- * fit, so it is granted once nothing else holds memory, and the grant says so.
10
+ * memory peak, and nothing while its task has none. A runner's memory grows with the parallelism it is lent (a Vitest
11
+ * worker is a process of its own), so a peak measured while the leaf was lent `parallelism` tokens is charged in
12
+ * proportion to the tokens it is lent now when that is more, never less: the baseline a leaf holds however few workers
13
+ * it has makes a smaller grant no cheaper. When its whole ask does not fit, the head is lent the most tokens whose
14
+ * charge does. A leaf whose own charge alone exceeds the run's memory could never fit, so it is granted once nothing
15
+ * else holds memory, on one token when its peak names the parallelism it was measured under, and the grant says so.
16
+ *
17
+ * A leaf the kernel killed for memory, run again (`alone`), is granted only while nothing holds a grant, ahead of the
18
+ * leaves behind it, and while it runs nothing else is granted: a leaf that was killed beside others runs by itself.
12
19
  */
13
20
 
14
- /** What a queued request asks: whole tokens, at least one, and its charge in bytes (0 for a task with no peak). */
21
+ /**
22
+ * What a queued request asks: whole tokens, at least one, its charge in bytes (0 for a task with no peak), and the
23
+ * tokens that charge was measured under. Without them the charge is flat: it is the same however many tokens are lent.
24
+ */
15
25
  export interface PoolAsk {
16
26
  readonly want: number;
17
27
  readonly memory: number;
28
+ readonly parallelism?: number | undefined;
29
+ /** A leaf the kernel killed for memory, run again: granted while nothing else holds a grant, and alone. */
30
+ readonly alone?: boolean | undefined;
31
+ }
32
+
33
+ /** Who holds a grant when the pool decides: how many leaves, and whether one of them runs alone. */
34
+ export interface Holding {
35
+ readonly holders: number;
36
+ readonly alone: boolean;
18
37
  }
19
38
 
39
+ const NOTHING_HELD: Holding = { holders: 0, alone: false };
20
40
  /** The run's memory a pool charges against, and the bytes its holders are charged; null for a pool that charges none. */
21
41
  export interface MemoryLedger {
22
42
  readonly capacity: number;
23
43
  readonly held: number;
24
44
  }
25
45
 
26
- /** One request's grant: the request, its tokens, and whether its charge alone exceeds the run's memory. */
46
+ /**
47
+ * One request's grant: the request, its tokens, the bytes it is charged at those tokens, and whether its charge alone
48
+ * exceeds the run's memory.
49
+ */
27
50
  export interface PoolGrant<Ask extends PoolAsk> {
28
51
  readonly request: Ask;
29
52
  readonly tokens: number;
53
+ readonly charge: number;
30
54
  readonly oversized: boolean;
31
55
  }
32
56
 
33
57
  /**
34
58
  * The grants of the queue's first requests, in its order, and what the first request left waiting waits for: free
35
- * tokens (`cpu`) or the run's memory (`memory`). `waiting` is null when every request was granted.
59
+ * tokens (`cpu`), the run's memory (`memory`) or the grants others hold (`alone`). `waiting` is null when every request was granted.
36
60
  */
37
61
  export interface Admission<Ask extends PoolAsk> {
38
62
  readonly grants: readonly PoolGrant<Ask>[];
39
- readonly waiting: 'cpu' | 'memory' | null;
63
+ readonly waiting: 'cpu' | 'memory' | 'alone' | null;
64
+ }
65
+
66
+ /** The bytes `ask` is charged when it is lent `tokens`: its peak, scaled up when that is more than it was measured under. */
67
+ export function chargeAt(ask: PoolAsk, tokens: number): number {
68
+ const measured = ask.parallelism;
69
+ return measured === undefined || tokens <= measured ? ask.memory : Math.ceil((ask.memory * tokens) / measured);
70
+ }
71
+
72
+ /** The most tokens, up to `most`, whose charge fits `room` bytes; none when even the peak itself does not. */
73
+ function tokensFitting(ask: PoolAsk, most: number, room: number): number {
74
+ if (ask.memory > room) {
75
+ return 0;
76
+ }
77
+ const measured = ask.parallelism;
78
+ if (measured === undefined || ask.memory === 0 || most <= measured) {
79
+ return most;
80
+ }
81
+ return Math.min(most, Math.floor((room * measured) / ask.memory));
40
82
  }
41
83
 
42
84
  export function admit<Ask extends PoolAsk>(
43
85
  queue: readonly Ask[],
44
86
  available: number,
45
87
  memory: MemoryLedger | null,
88
+ holding: Holding = NOTHING_HELD,
46
89
  ): Admission<Ask> {
47
90
  const grants: PoolGrant<Ask>[] = [];
91
+ if (holding.alone) {
92
+ return { grants, waiting: queue.length === 0 ? null : 'alone' };
93
+ }
48
94
  let free = available;
49
95
  let held = memory?.held ?? 0;
50
96
  for (const request of queue) {
97
+ if (request.alone === true && holding.holders + grants.length > 0) {
98
+ return { grants, waiting: 'alone' };
99
+ }
51
100
  const oversized = memory !== null && request.memory > memory.capacity;
52
- if (memory !== null && held + request.memory > memory.capacity && !(oversized && held === 0)) {
101
+ const room = memory === null ? Number.POSITIVE_INFINITY : memory.capacity - held;
102
+ if (request.memory > room && !(oversized && held === 0)) {
53
103
  return { grants, waiting: 'memory' };
54
104
  }
55
- const tokens = Math.min(request.want, free);
56
- if (tokens === 0) {
105
+ const most = Math.min(request.want, free);
106
+ if (most === 0) {
57
107
  return { grants, waiting: 'cpu' };
58
108
  }
109
+ const tokens = oversized ? (request.parallelism === undefined ? most : 1) : tokensFitting(request, most, room);
110
+ const charge = chargeAt(request, tokens);
59
111
  free -= tokens;
60
- held += request.memory;
61
- grants.push({ request, tokens, oversized });
112
+ held += charge;
113
+ grants.push({ request, tokens, charge, oversized });
114
+ if (request.alone === true) {
115
+ return { grants, waiting: grants.length < queue.length ? 'alone' : null };
116
+ }
62
117
  }
63
118
  return { grants, waiting: null };
64
119
  }
@@ -5,7 +5,7 @@ import { logger } from 'nx/src/devkit-exports.js';
5
5
  import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src/project-graph/plugins/public-api.js';
6
6
  import { type CloseOutcome, type CpuPool, readInheritedCpuPool } from './cpu-pool.js';
7
7
  import type { CpuPoolThreadData, CpuPoolThreadReport } from './cpu-pool-thread.js';
8
- import { type RunMemoryReading, RunMemoryUnreadable, readRunMemory } from './host-memory.js';
8
+ import { type RunMemory, type RunMemoryReading, RunMemoryUnreadable, readRunMemory } from './host-memory.js';
9
9
  import { mbxPool } from './mbx-pool.js';
10
10
  import { CPU_TOKENS_ENV, parseInheritedTokens } from './runner-sizing.js';
11
11
 
@@ -91,7 +91,7 @@ export async function serveRunCpuPool(
91
91
  if (!memory.ok) {
92
92
  throw new RunMemoryUnreadable(memory.error);
93
93
  }
94
- const served = await servePoolThread(tokens, memory.memory.bytes);
94
+ const served = await servePoolThread(tokens, memory.memory);
95
95
  if (!served.ok) {
96
96
  logger.warn(
97
97
  `bounded-exec: this run's CPU pool failed (${served.error}); its tasks are sized only by the ceilings they inherit`,
@@ -110,7 +110,7 @@ export async function serveRunCpuPool(
110
110
  * has said how its pool closed fails whichever wait is open, a later close returns that failure, and a thread lost while
111
111
  * its pool serves is said at once.
112
112
  */
113
- function servePoolThread(tokens: number, memory: number): Promise<ServedPool> {
113
+ function servePoolThread(tokens: number, memory: RunMemory): Promise<ServedPool> {
114
114
  const thread = new Worker(THREAD_ENTRY, {
115
115
  workerData: { tokens, memory } satisfies CpuPoolThreadData,
116
116
  ...THREAD_PRELOAD,
@@ -87,8 +87,9 @@ export interface BoundedExecRecord {
87
87
  readonly verdict: BoundedExecVerdict;
88
88
  /**
89
89
  * The parent CPU pool's grant the run started under: tokens granted of those asked for, and
90
- * how long it waited for the pool. Absent when it ran ungranted; written for the reader of a slow
91
- * run, never read back.
90
+ * how long it waited for the pool. Absent when it ran ungranted. Its tokens are the parallelism the run's
91
+ * `peakBytes` was measured under, which the task's next run reads (`readLastPeak`) to charge that peak in proportion
92
+ * to the parallelism it is lent.
92
93
  */
93
94
  readonly cpu?: {
94
95
  readonly tokens: number;
@@ -501,9 +502,10 @@ export async function readRecord(directory: string): Promise<BoundedExecRecord |
501
502
  */
502
503
  export async function readLastPeak(
503
504
  directory: string,
504
- ): Promise<{ ok: true; peakBytes: number | null } | { ok: false; error: string }> {
505
+ ): Promise<{ ok: true; peakBytes: number | null; parallelism: number | null } | { ok: false; error: string }> {
505
506
  try {
506
- return { ok: true, peakBytes: (await readRecord(directory))?.peakBytes ?? null };
507
+ const record = await readRecord(directory);
508
+ return { ok: true, peakBytes: record?.peakBytes ?? null, parallelism: record?.cpu?.tokens ?? null };
507
509
  } catch (error) {
508
510
  return {
509
511
  ok: false,
@@ -540,7 +542,25 @@ function parseRecord(value: unknown): BoundedExecRecord {
540
542
  if (!(peakBytes === null || (typeof peakBytes === 'number' && Number.isSafeInteger(peakBytes) && peakBytes >= 0))) {
541
543
  throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
542
544
  }
543
- return { task: value.task, hash: value.hash, verdict: parseVerdict(value.verdict), peakBytes };
545
+ const cpu = value.cpu;
546
+ if (cpu === undefined) {
547
+ return { task: value.task, hash: value.hash, verdict: parseVerdict(value.verdict), peakBytes };
548
+ }
549
+ if (
550
+ !isObject(cpu) ||
551
+ !(typeof cpu.tokens === 'number' && Number.isSafeInteger(cpu.tokens) && cpu.tokens >= 1) ||
552
+ !(typeof cpu.want === 'number' && Number.isSafeInteger(cpu.want) && cpu.want >= 1) ||
553
+ !(typeof cpu.waitedMs === 'number' && Number.isFinite(cpu.waitedMs) && cpu.waitedMs >= 0)
554
+ ) {
555
+ throw new Error(`not a bounded-exec record: ${JSON.stringify(value)}`);
556
+ }
557
+ return {
558
+ task: value.task,
559
+ hash: value.hash,
560
+ verdict: parseVerdict(value.verdict),
561
+ cpu: { tokens: cpu.tokens, want: cpu.want, waitedMs: cpu.waitedMs },
562
+ peakBytes,
563
+ };
544
564
  }
545
565
 
546
566
  function parseVerdict(value: unknown): BoundedExecVerdict {
package/test-timings.json CHANGED
@@ -10,6 +10,7 @@
10
10
  "src/owned-installation.test.ts": 28757,
11
11
  "src/index.test.ts": 26774,
12
12
  "src/executors/bounded-exec/executor.test.ts": 25285,
13
+ "src/executors/bounded-exec/pool-sim.property.test.ts": 23356,
13
14
  "src/ensure-built-receipt.test.ts": 21873,
14
15
  "src/nx-daemon-idle-policy.test.ts": 18587,
15
16
  "src/cargo-retention.test.ts": 16827,
@@ -24,6 +25,7 @@
24
25
  "src/typecheck-test-policy.test.ts": 5606,
25
26
  "src/bounded-test-policy.test.ts": 5412,
26
27
  "src/nx-daemon-start-never.test.ts": 5356,
28
+ "src/nx-local-plugin-hook-env.test.ts": 5300,
27
29
  "src/executors/bounded-exec/ram-temp.test.ts": 5259,
28
30
  "src/task-results.test.ts": 5234,
29
31
  "src/nx-dot-input-policy.test.ts": 5127,