@smoothbricks/nx-plugin 0.7.16 → 0.7.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/README.md +210 -64
  2. package/dist/.build-inputs.json +28 -23
  3. package/dist/bun-lock-scope.d.ts +128 -20
  4. package/dist/bun-lock-scope.d.ts.map +1 -1
  5. package/dist/bun-lock-scope.js +267 -44
  6. package/dist/cargo-closure-input.d.ts +8 -6
  7. package/dist/cargo-closure-input.d.ts.map +1 -1
  8. package/dist/cargo-closure-input.js +14 -11
  9. package/dist/cargo-source-hash.d.ts +23 -4
  10. package/dist/cargo-source-hash.d.ts.map +1 -1
  11. package/dist/cargo-source-hash.js +80 -14
  12. package/dist/cargo-workspace.d.ts +20 -2
  13. package/dist/cargo-workspace.d.ts.map +1 -1
  14. package/dist/cargo-workspace.js +42 -22
  15. package/dist/compile-environment.d.ts +8 -0
  16. package/dist/compile-environment.d.ts.map +1 -0
  17. package/dist/compile-environment.js +80 -0
  18. package/dist/dependency-manifests.d.ts +13 -0
  19. package/dist/dependency-manifests.d.ts.map +1 -0
  20. package/dist/dependency-manifests.js +20 -0
  21. package/dist/ensure-built-receipt.d.ts +3 -2
  22. package/dist/ensure-built-receipt.d.ts.map +1 -1
  23. package/dist/ensure-built-receipt.js +3 -2
  24. package/dist/eslint-config-closure.d.ts +8 -3
  25. package/dist/eslint-config-closure.d.ts.map +1 -1
  26. package/dist/eslint-config-closure.js +24 -5
  27. package/dist/executors/bounded-exec/bounds.d.ts +12 -1
  28. package/dist/executors/bounded-exec/bounds.d.ts.map +1 -1
  29. package/dist/executors/bounded-exec/bounds.js +8 -4
  30. package/dist/executors/bounded-exec/cpu-rise.d.ts +73 -0
  31. package/dist/executors/bounded-exec/cpu-rise.d.ts.map +1 -0
  32. package/dist/executors/bounded-exec/cpu-rise.js +111 -0
  33. package/dist/executors/bounded-exec/executor.d.ts +1 -1
  34. package/dist/executors/bounded-exec/executor.d.ts.map +1 -1
  35. package/dist/executors/bounded-exec/executor.js +54 -22
  36. package/dist/executors/bounded-exec/hang-diagnostics.d.ts +8 -0
  37. package/dist/executors/bounded-exec/hang-diagnostics.d.ts.map +1 -1
  38. package/dist/executors/bounded-exec/hang-diagnostics.js +66 -6
  39. package/dist/executors/bounded-exec/mbx-pool.d.ts +16 -2
  40. package/dist/executors/bounded-exec/mbx-pool.d.ts.map +1 -1
  41. package/dist/executors/bounded-exec/mbx-pool.js +18 -2
  42. package/dist/executors/bounded-exec/mbx-reservations.d.ts +5 -2
  43. package/dist/executors/bounded-exec/mbx-reservations.d.ts.map +1 -1
  44. package/dist/executors/bounded-exec/mbx-reservations.js +5 -3
  45. package/dist/executors/bounded-exec/progress-channel.d.ts +38 -0
  46. package/dist/executors/bounded-exec/progress-channel.d.ts.map +1 -0
  47. package/dist/executors/bounded-exec/progress-channel.js +113 -0
  48. package/dist/executors/bounded-exec/progress.d.ts +2 -1
  49. package/dist/executors/bounded-exec/progress.d.ts.map +1 -1
  50. package/dist/executors/bounded-exec/ram-temp.d.ts +24 -1
  51. package/dist/executors/bounded-exec/ram-temp.d.ts.map +1 -1
  52. package/dist/executors/bounded-exec/ram-temp.js +123 -16
  53. package/dist/executors/bounded-exec/schema.d.ts +5 -0
  54. package/dist/executors/bounded-exec/schema.d.ts.map +1 -1
  55. package/dist/executors/bounded-exec/schema.js +17 -1
  56. package/dist/executors/bounded-exec/verdict.d.ts +13 -6
  57. package/dist/executors/bounded-exec/verdict.d.ts.map +1 -1
  58. package/dist/executors/bounded-exec/verdict.js +34 -9
  59. package/dist/index.d.ts.map +1 -1
  60. package/dist/index.js +216 -97
  61. package/dist/managed-files/files.d.ts +3 -3
  62. package/dist/managed-files/files.js +3 -3
  63. package/dist/managed-files/generator.d.ts +5 -1
  64. package/dist/managed-files/generator.d.ts.map +1 -1
  65. package/dist/managed-files/generator.js +22 -7
  66. package/dist/managed-files/render-memo.d.ts +13 -0
  67. package/dist/managed-files/render-memo.d.ts.map +1 -0
  68. package/dist/managed-files/render-memo.js +94 -0
  69. package/dist/test-program.d.ts +11 -6
  70. package/dist/test-program.d.ts.map +1 -1
  71. package/dist/test-program.js +39 -35
  72. package/dist/tool-bins.d.ts +12 -0
  73. package/dist/tool-bins.d.ts.map +1 -1
  74. package/dist/tool-bins.js +31 -6
  75. package/dist/typecheck-route.d.ts +3 -4
  76. package/dist/typecheck-route.d.ts.map +1 -1
  77. package/dist/typecheck-route.js +4 -15
  78. package/dist/typecheck-test-policy.d.ts +10 -1
  79. package/dist/typecheck-test-policy.d.ts.map +1 -1
  80. package/dist/typecheck-test-policy.js +40 -24
  81. package/dist/workspace-index.d.ts +14 -1
  82. package/dist/workspace-index.d.ts.map +1 -1
  83. package/dist/workspace-index.js +36 -4
  84. package/managed/raw/patches/nx@24.0.3.patch +703 -119
  85. package/managed/raw/patches/{ttsc@0.34.1.patch → ttsc@0.36.3.patch} +728 -71
  86. package/managed/raw/tooling/direnv/developer-links.ts +17 -3
  87. package/managed/raw/tooling/direnv/devenv.smoo.nix +1 -1
  88. package/managed/templates/github/actions/save-nix-devenv/action.yml +19 -1
  89. package/managed/templates/github/actions/setup-devenv/action.yml +2 -0
  90. package/package.json +5 -4
  91. package/src/bun-lock-scope.ts +293 -40
  92. package/src/cargo-closure-input.ts +19 -11
  93. package/src/cargo-source-hash.ts +103 -13
  94. package/src/cargo-workspace.ts +59 -20
  95. package/src/compile-environment.ts +35 -0
  96. package/src/dependency-manifests.ts +30 -0
  97. package/src/ensure-built-receipt.ts +3 -2
  98. package/src/eslint-config-closure.ts +27 -6
  99. package/src/executors/bounded-exec/bounds.ts +13 -4
  100. package/src/executors/bounded-exec/cpu-rise.ts +156 -0
  101. package/src/executors/bounded-exec/executor.ts +65 -21
  102. package/src/executors/bounded-exec/hang-diagnostics.ts +88 -6
  103. package/src/executors/bounded-exec/mbx-pool.ts +24 -3
  104. package/src/executors/bounded-exec/mbx-reservations.ts +7 -5
  105. package/src/executors/bounded-exec/progress-channel.ts +129 -0
  106. package/src/executors/bounded-exec/progress.ts +2 -1
  107. package/src/executors/bounded-exec/ram-temp.ts +137 -17
  108. package/src/executors/bounded-exec/schema.ts +19 -0
  109. package/src/executors/bounded-exec/verdict.ts +44 -16
  110. package/src/index.ts +272 -106
  111. package/src/managed-files/files.ts +3 -3
  112. package/src/managed-files/generator.ts +21 -6
  113. package/src/managed-files/render-memo.ts +135 -0
  114. package/src/test-program.ts +36 -41
  115. package/src/tool-bins.ts +35 -6
  116. package/src/typecheck-route.ts +5 -14
  117. package/src/typecheck-test-policy.ts +49 -24
  118. package/src/workspace-index.ts +40 -5
  119. package/test-timings.json +6 -0
@@ -17,6 +17,7 @@ import { AdmissionObservation, type QueueHost } from './admission-queue.js';
17
17
  import { type BoundEvent, type BoundKind, type Bounds, earliestBound, stepBounds, UNARMED } from './bounds.js';
18
18
  import { DIAGNOSTIC_CONTEXT_ENV, DIAGNOSTIC_MODULE_ENV, grantMemoryEnvironment } from './child-exit-diagnostics.js';
19
19
  import { type CpuPool, createCpuPool } from './cpu-pool.js';
20
+ import type { CpuRise } from './cpu-rise.js';
20
21
  import { GATEWAY_SOCKET } from './gateway-lease.js';
21
22
  import type { HangDiagnostics } from './hang-diagnostics.js';
22
23
  import { captureHangDiagnostics, HANG_CAPTURE_MS } from './hang-diagnostics.js';
@@ -47,6 +48,7 @@ import { askParentPool, describeGrant } from './parent-grant.js';
47
48
  import { type LastPeak, readLastPeak } from './peak-charge.js';
48
49
  import { CPU_POOL_ENV, type InheritedCpuPool, readInheritedCpuPool } from './pool-environment.js';
49
50
  import { createRunProgress, withProgressReport, withTestResultReport } from './progress.js';
51
+ import { PROGRESS_SOCKET_ENV, type ProgressChannel, serveProgressChannel } from './progress-channel.js';
50
52
  import {
51
53
  createHostCommands,
52
54
  describeCoverageGap,
@@ -62,7 +64,7 @@ import {
62
64
  import { createRunnerLifecycle, withFixtureLifecycle, withRunnerLifecycle } from './runner-lifecycle.js';
63
65
  import { CPU_TOKENS_ENV, runnerDemand, sizeRunner, sizeRunnerToGrant } from './runner-sizing.js';
64
66
  import { checkRuntimePin, describeRuntimePinRefusal } from './runtime-pin.js';
65
- import type { BoundedExecOptions } from './schema.js';
67
+ import { type BoundedExecOptions, boundedCommand } from './schema.js';
66
68
  import { createTestResultWriter, type TestResultWriter } from './test-result-writer.js';
67
69
  import {
68
70
  type BoundedExecRecord,
@@ -137,6 +139,8 @@ interface RunState {
137
139
  noSpace: Promise<NoSpaceSighting> | null;
138
140
  /** The lock waits the expiry's snapshot named: empty unless that capture completed. Judged with the expiry. */
139
141
  lockWaits: readonly LockWait[];
142
+ /** The processes the expiry's snapshot saw gain CPU time: empty unless that capture completed. Judged with the expiry. */
143
+ cpuRising: readonly CpuRise[];
140
144
  }
141
145
 
142
146
  export interface ProcessTreeKiller {
@@ -341,7 +345,7 @@ async function runCommand(
341
345
  if (directory !== null) {
342
346
  await clearTaskDirectory(directory);
343
347
  }
344
- const requested = buildCommand(options);
348
+ const requested = boundedCommand(options);
345
349
  const timeoutMs = options.timeoutMs;
346
350
  const idleTimeoutMs = options.idleTimeoutMs;
347
351
  const killAfterMs = options.killAfterMs ?? DEFAULT_KILL_AFTER_MS;
@@ -361,6 +365,7 @@ async function runCommand(
361
365
  retireError: null,
362
366
  noSpace: null,
363
367
  lockWaits: [],
368
+ cpuRising: [],
364
369
  };
365
370
  const appendStdout = (chunk: Buffer | string): void => {
366
371
  const text = chunk.toString();
@@ -609,6 +614,17 @@ async function runCommand(
609
614
  let pool: CpuPool | null = null;
610
615
  let reservation: Reservation | null = null;
611
616
  const releaseReservation = (): void => reservation?.release();
617
+ // The run's progress channel (progress-channel.ts), served when the run has an idle bound for its records to restart,
618
+ // and closed once its command is gone. A channel that leaves its socket behind says so; the run's verdict stands.
619
+ let progressChannel: ProgressChannel | null = null;
620
+ let progressClosed: Promise<void> | null = null;
621
+ const closeProgress = (): Promise<void> => {
622
+ progressClosed ??= (async () => {
623
+ const left = (await progressChannel?.close()) ?? null;
624
+ if (left !== null) appendStderr(`bounded-exec: ${left}\n`);
625
+ })();
626
+ return progressClosed;
627
+ };
612
628
  let cpuReleased: Promise<void> | null = null;
613
629
  const releaseCpu = (): Promise<void> => {
614
630
  cpuReleased ??= (async () => {
@@ -642,6 +658,7 @@ async function runCommand(
642
658
  operational: boolean,
643
659
  noSpace: NoSpaceSighting | null,
644
660
  lockWaits: readonly LockWait[],
661
+ cpuRising: readonly CpuRise[],
645
662
  peakBytes: number | null,
646
663
  ): Promise<void> => {
647
664
  if (context.record === undefined) {
@@ -677,6 +694,7 @@ async function runCommand(
677
694
  report,
678
695
  noSpace,
679
696
  lockWaits,
697
+ cpuRising,
680
698
  admissionQueue,
681
699
  });
682
700
  if (verdict.outcome === 'bound' && verdict.bound === 'test') {
@@ -695,9 +713,10 @@ async function runCommand(
695
713
  const refuse = async (message: string): Promise<BoundedExecResult> => {
696
714
  appendStderr(message);
697
715
  await releaseCpu();
716
+ await closeProgress();
698
717
  // The command never started, so its lease holds nothing of its own to keep.
699
718
  await releaseLease(false);
700
- await recordRun(1, null, Date.now() - asked, false, null, [], null);
719
+ await recordRun(1, null, Date.now() - asked, false, null, [], [], null);
701
720
  return { success: false, terminalOutput: outputChunks.join('') };
702
721
  };
703
722
  if (parentRefusal !== null) {
@@ -723,6 +742,29 @@ async function runCommand(
723
742
  }
724
743
  pool = created.pool;
725
744
  }
745
+ // A running test reports its own progress on the run's progress channel, and each record restarts the idle bound as a
746
+ // finished test does; a record that is not one fails the run as a broken reporter. A run with no idle bound has
747
+ // nothing for a record to restart and is served none. Only the command learns the socket, from its environment, so
748
+ // these handlers run after it is spawned and its bounds are wired below.
749
+ if (idleTimeoutMs !== undefined) {
750
+ const served = await serveProgressChannel({
751
+ moved: () => {
752
+ if (fixtureDone) stepTo({ kind: 'reported', at: Date.now(), testing: isTesting() });
753
+ },
754
+ invalid: (detail) => {
755
+ if (lifecycleFailure !== null || state.expiry !== null || state.settled) return;
756
+ lifecycleFailure = `progress channel: ${detail}`;
757
+ appendStderr(`bounded-exec: ${lifecycleFailure}\n`);
758
+ termination = terminate();
759
+ },
760
+ });
761
+ if (!served.ok) {
762
+ return refuse(
763
+ `bounded-exec: the run's progress channel could not be served (${served.error}); refusing to run the command\n`,
764
+ );
765
+ }
766
+ progressChannel = served.channel;
767
+ }
726
768
  const native = directory === null ? { command: sized.command, env: {} } : withTestResultReport(sized.command);
727
769
  const command =
728
770
  directory === null
@@ -740,6 +782,7 @@ async function runCommand(
740
782
  ...(pool?.environment ?? {}),
741
783
  ...(await grantMemoryEnvironment(grant?.charged ?? null, process.env)),
742
784
  ...(leaf?.kind === 'cargo' ? { MBX_GRANT_FD: String(MBX_GRANT_FD) } : {}),
785
+ ...(progressChannel === null ? {} : { [PROGRESS_SOCKET_ENV]: progressChannel.socket }),
743
786
  };
744
787
  // A fixture rides as segment 0 of the lifecycle. Without the channel (Windows) it is plain shell text ahead
745
788
  // of the command: the progress bound runs during it and its own ceiling does not. Under mbx a leaf that names its
@@ -845,7 +888,8 @@ async function runCommand(
845
888
  });
846
889
  }
847
890
 
848
- // Progress is a finished test, not output: a runner's SLOW warnings and a hung test's heartbeat keep printing.
891
+ // Progress in the output is a finished test, never output itself: a runner's SLOW warnings and a hung test's heartbeat
892
+ // keep printing. A running test reports its own progress on the progress channel instead.
849
893
  const progress = createRunProgress(
850
894
  command,
851
895
  resultWriter === null
@@ -1082,6 +1126,7 @@ async function runCommand(
1082
1126
  queue.stop();
1083
1127
  return killChildTree();
1084
1128
  })
1129
+ .finally(closeProgress)
1085
1130
  .finally(releaseCpu)
1086
1131
  .finally(() => releaseLease(true))
1087
1132
  .finally(releaseReservation)
@@ -1093,6 +1138,7 @@ async function runCommand(
1093
1138
  state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null,
1094
1139
  state.noSpace === null ? null : await state.noSpace,
1095
1140
  state.lockWaits,
1141
+ state.cpuRising,
1096
1142
  null,
1097
1143
  ),
1098
1144
  )
@@ -1145,7 +1191,7 @@ async function runCommand(
1145
1191
  const elapsedMs = Date.now() - startedAt;
1146
1192
  appendStderr(
1147
1193
  bound.kind === 'idle'
1148
- ? `\nCommand made no progress: no test finished for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
1194
+ ? `\nCommand made no progress: no test finished or reported progress for ${bound.limitMs}ms (idleTimeoutMs=${bound.limitMs}) after ${elapsedMs}ms of runtime; ${
1149
1195
  bound.tests.length > 0
1150
1196
  ? `still running: ${bound.tests.join(', ')}`
1151
1197
  : 'the runner named no test still running'
@@ -1161,6 +1207,7 @@ async function runCommand(
1161
1207
  if (bound.kind === 'idle') {
1162
1208
  const observed = idleCapture?.result;
1163
1209
  state.lockWaits = observed?.kind === 'captured' ? observed.snapshot.lockWaits : [];
1210
+ state.cpuRising = observed?.kind === 'captured' ? observed.snapshot.cpuRising : [];
1164
1211
  appendStderr(
1165
1212
  `Hang diagnostics: runner pid=${runner ?? 'unavailable'} pgid=${
1166
1213
  process.platform === 'win32' ? 'unavailable' : (child.pid ?? 'unavailable')
@@ -1365,8 +1412,9 @@ async function runCommand(
1365
1412
  clearTimeout(boundTimer);
1366
1413
  cancelCapture();
1367
1414
  removeSignalHandlers();
1368
- // The runner is gone. Its sub-pool closes first, so no descendant still draws on tokens the grant returns;
1369
- // then the grant goes back to its parent or the host.
1415
+ // The runner is gone, and with it every test that could report on the run's progress channel. Its sub-pool closes
1416
+ // first, so no descendant still draws on tokens the grant returns; then the grant goes back to its parent or the host.
1417
+ await closeProgress();
1370
1418
  await releaseCpu();
1371
1419
 
1372
1420
  const code = exitCode ?? signalToExitCode(exitSignal);
@@ -1414,7 +1462,16 @@ async function runCommand(
1414
1462
  // Broken resource ownership, a lost native lifecycle, or leftovers that could not be retired fail even if a shell
1415
1463
  // branch later exited 0: its verdict is an ordinary operational failure, never a successful or silently unbounded run.
1416
1464
  const operationalFailure = state.poolCloseError !== null || lifecycleFailure !== null || state.retireError !== null;
1417
- await recordRun(code, state.expiry, Date.now() - startedAt, operationalFailure, seen, state.lockWaits, peakBytes);
1465
+ await recordRun(
1466
+ code,
1467
+ state.expiry,
1468
+ Date.now() - startedAt,
1469
+ operationalFailure,
1470
+ seen,
1471
+ state.lockWaits,
1472
+ state.cpuRising,
1473
+ peakBytes,
1474
+ );
1418
1475
  const killedForMemory = killedFromOutside({
1419
1476
  sentSignal: state.sentSignal,
1420
1477
  bounded: state.expiry !== null,
@@ -1443,19 +1500,6 @@ function resolveCwd(cwd: string | undefined, root: string): string {
1443
1500
  return isAbsolute(cwd) ? cwd : join(root, cwd);
1444
1501
  }
1445
1502
 
1446
- function buildCommand(options: BoundedExecOptions): string {
1447
- const parts = [options.command];
1448
- if (Array.isArray(options.args)) {
1449
- parts.push(...options.args);
1450
- } else if (options.args) {
1451
- parts.push(options.args);
1452
- }
1453
- if (options.forwardAllArgs !== false && options.__unparsed__?.length) {
1454
- parts.push(...options.__unparsed__);
1455
- }
1456
- return parts.join(' ');
1457
- }
1458
-
1459
1503
  /** Owned scratch is the command's TMPDIR unless the target names one itself. */
1460
1504
  function mergeEnv(directory: string | null, env: Record<string, string>): NodeJS.ProcessEnv {
1461
1505
  const inherited = withoutPoolEnv(process.env);
@@ -1,6 +1,16 @@
1
1
  import { spawn } from 'node:child_process';
2
2
  import { describeTeardown, killProcessGroupNow, TEARDOWN_PATIENCE } from '../../process-group.js';
3
3
  import { type ProcessEntry, withDescendants } from '../../testing.js';
4
+ import {
5
+ CPU_RISING_CLASS,
6
+ type CpuRise,
7
+ type CpuTime,
8
+ cpuChange,
9
+ cpuRiseThresholdsMs,
10
+ describeCpuRise,
11
+ describeCpuThresholds,
12
+ parseCpuTime,
13
+ } from './cpu-rise.js';
4
14
  import {
5
15
  describeHolders,
6
16
  type LockWait,
@@ -16,7 +26,15 @@ import {
16
26
  */
17
27
  export const HANG_CAPTURE_MS = 10_000;
18
28
  const MAX_PROBE_BYTES = 4 * 1024 * 1024;
19
- const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,lstart=,command=';
29
+ const PS_FIELDS = 'pid=,ppid=,pgid=,stat=,%cpu=,etime=,cputime=,lstart=,command=';
30
+ /**
31
+ * The least time between the two process censuses, which are the snapshot's two CPU samples. A process is named
32
+ * after gaining 2 s where ps counts whole seconds and 0.5 s where it counts centiseconds (cpuRiseThresholdMs), so five
33
+ * seconds show a process at 40% or at 10% of a core. On Darwin the stack samples have usually spent most of it;
34
+ * elsewhere, or when sudo refuses them, the snapshot pauses, inside its window and early enough that the closing
35
+ * census keeps its own limit.
36
+ */
37
+ const CPU_INTERVAL_MS = 5_000;
20
38
  /**
21
39
  * Stack samples: `sample` suspends a process every millisecond for one second and reports its threads' call graph. The
22
40
  * runner and at most this many leaf processes are sampled, all at once. `-mayDie` is not passed: it reads symbols
@@ -58,6 +76,11 @@ export interface HangDiagnostics {
58
76
  outcome: 'holders' | 'none' | 'incomplete';
59
77
  /** Lock waits the snapshot names: empty unless the capture is complete, and always empty without the open-file census. */
60
78
  lockWaits: readonly LockWait[];
79
+ /**
80
+ * The processes of the tree whose CPU time rose between the two censuses, largest gain first: empty when none did, and
81
+ * when the closing census did not complete. A build inside the test, never a restart of the idle bound.
82
+ */
83
+ cpuRising: readonly CpuRise[];
61
84
  text: string;
62
85
  }
63
86
 
@@ -77,15 +100,31 @@ export type HangProbeResult =
77
100
  export interface HangDiagnosticHost {
78
101
  platform: NodeJS.Platform;
79
102
  probe(file: string, args: readonly string[], timeoutMs: number, signal: AbortSignal): Promise<HangProbeResult>;
103
+ /** Waits `ms`, or until `signal` aborts: the capture's end ends its own CPU interval. */
104
+ sleep(ms: number, signal: AbortSignal): Promise<void>;
80
105
  }
81
106
 
82
107
  interface ObservedProcess extends ProcessEntry {
83
108
  pgid: number;
84
109
  started: string;
110
+ cpu: CpuTime | null;
85
111
  raw: string;
86
112
  }
87
113
 
88
- const host: HangDiagnosticHost = { platform: process.platform, probe: runProbe };
114
+ const host: HangDiagnosticHost = { platform: process.platform, probe: runProbe, sleep: sleepUnlessAborted };
115
+
116
+ function sleepUnlessAborted(ms: number, signal: AbortSignal): Promise<void> {
117
+ return new Promise((resolve) => {
118
+ const done = (): void => {
119
+ clearTimeout(timer);
120
+ signal.removeEventListener('abort', done);
121
+ resolve();
122
+ };
123
+ const timer = setTimeout(done, ms);
124
+ signal.addEventListener('abort', done, { once: true });
125
+ if (signal.aborted) done();
126
+ });
127
+ }
89
128
 
90
129
  /** A pre-expiry observation, never an assertion that an open file is a contested lock. */
91
130
  export function captureHangDiagnostics(
@@ -104,6 +143,7 @@ export function captureHangDiagnostics(
104
143
  const during = (): string => (active.size === 0 ? 'process census' : [...active].join(', '));
105
144
  let finished = false;
106
145
  let waits: readonly LockWait[] = [];
146
+ let rising: readonly CpuRise[] = [];
107
147
  // This package targets lib es2022, which does not declare Promise.withResolvers.
108
148
  return new Promise((resolve) => {
109
149
  const finish = (outcome: HangDiagnostics['outcome']): void => {
@@ -120,6 +160,7 @@ export function captureHangDiagnostics(
120
160
  offsetFromExpiryMs,
121
161
  outcome,
122
162
  lockWaits: outcome === 'incomplete' ? [] : waits,
163
+ cpuRising: rising,
123
164
  text: `${lines.join('\n')}\nHang snapshot outcome=${outcome} capturedAtMs=${capturedAtMs} offsetFromExpiryMs=${offsetFromExpiryMs}\n`,
124
165
  });
125
166
  };
@@ -171,31 +212,41 @@ export function captureHangDiagnostics(
171
212
  const parsed: ObservedProcess[] = [];
172
213
  for (const row of result.stdout.split('\n').filter((line) => line.trim() !== '')) {
173
214
  const match =
174
- /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+\s+\S+\s+\d+\s+\d\d:\d\d:\d\d\s+\d{4})\s+(.*)$/.exec(
215
+ /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+\s+\S+\s+\d+\s+\d\d:\d\d:\d\d\s+\d{4})\s+(.*)$/.exec(
175
216
  row,
176
217
  );
177
218
  if (match === null) {
178
219
  lines.push(`${name}: could not decode ps row: ${row}`);
179
220
  return null;
180
221
  }
181
- const [, pid, ppid, pgid, stat, , , started, command] = match;
222
+ const [, pid, ppid, pgid, stat, , , cputime, started, command] = match;
182
223
  if (
183
224
  pid === undefined ||
184
225
  ppid === undefined ||
185
226
  pgid === undefined ||
186
227
  stat === undefined ||
228
+ cputime === undefined ||
187
229
  started === undefined ||
188
230
  command === undefined
189
231
  ) {
190
232
  lines.push(`${name}: incomplete ps row: ${row}`);
191
233
  return null;
192
234
  }
193
- parsed.push({ pid: Number(pid), ppid: Number(ppid), pgid: Number(pgid), stat, started, command, raw: row });
235
+ parsed.push({
236
+ pid: Number(pid),
237
+ ppid: Number(ppid),
238
+ pgid: Number(pgid),
239
+ stat,
240
+ cpu: parseCpuTime(cputime),
241
+ started,
242
+ command,
243
+ raw: row,
244
+ });
194
245
  }
195
246
  const roots = [request.pid, ...parsed.filter((entry) => entry.pgid === request.pgid).map((entry) => entry.pid)];
196
247
  const selected = new Set(withDescendants(parsed, roots).map((entry) => entry.pid));
197
248
  const tree = parsed.filter((entry) => selected.has(entry.pid));
198
- lines.push(`${name}: PID PPID PGID STAT %CPU ELAPSED STARTED COMMAND`);
249
+ lines.push(`${name}: PID PPID PGID STAT %CPU ELAPSED CPUTIME STARTED COMMAND`);
199
250
  lines.push(...tree.map((entry) => entry.raw));
200
251
  if (!tree.some((entry) => entry.pid === request.pid)) {
201
252
  lines.push(`${name}: root pid ${request.pid} exited or is not visible; capture raced or is incomplete.`);
@@ -267,6 +318,7 @@ export function captureHangDiagnostics(
267
318
  const before = await census('ps before');
268
319
  if (finished) return;
269
320
  if (before === null) return finish('incomplete');
321
+ const beforeAtMs = Date.now();
270
322
  let incomplete = false;
271
323
  let observed = before.length > 0;
272
324
  if (source.platform === 'darwin') {
@@ -327,9 +379,17 @@ export function captureHangDiagnostics(
327
379
  lines.push(`Open-file census not provided on ${source.platform}; observations below name processes only.`);
328
380
  lines.push(`Stack samples not provided on ${source.platform}; no blocking call is observed.`);
329
381
  }
382
+ const settleMs = Math.min(beforeAtMs + CPU_INTERVAL_MS, endsAtMs - CENSUS_LIMIT_MS) - Date.now();
383
+ if (settleMs > 0) {
384
+ active.add('CPU interval');
385
+ await source.sleep(settleMs, controller.signal);
386
+ active.delete('CPU interval');
387
+ if (finished) return;
388
+ }
330
389
  const after = await census('ps after');
331
390
  if (finished) return;
332
391
  if (after === null) return finish('incomplete');
392
+ const afterAtMs = Date.now();
333
393
  if (
334
394
  before.length !== after.length ||
335
395
  before.some(
@@ -348,6 +408,28 @@ export function captureHangDiagnostics(
348
408
  incomplete = true;
349
409
  lines.push('Process identities or membership changed during capture; open-file census raced.');
350
410
  }
411
+ const change = cpuChange(before, after);
412
+ const thresholdsMs = cpuRiseThresholdsMs(before, after);
413
+ rising = change.rising;
414
+ const intervalMs = afterAtMs - beforeAtMs;
415
+ if (change.unread.length > 0) {
416
+ lines.push(
417
+ `CPU time not read for pid ${change.unread.join(', ')}: ps printed none that parses, so they are not counted idle.`,
418
+ );
419
+ }
420
+ if (rising.length > 0) {
421
+ lines.push(
422
+ `CPU rose in the tree over about ${intervalMs}ms between the censuses (a rise is named from ${describeCpuThresholds(thresholdsMs)} of CPU):`,
423
+ ...rising.map((rise) => ` ${describeCpuRise(rise)}`),
424
+ `Classification: ${CPU_RISING_CLASS}`,
425
+ );
426
+ } else if (thresholdsMs.length === 0) {
427
+ lines.push('CPU: no process of the tree had a CPU time ps gave in both censuses, so none was compared.');
428
+ } else {
429
+ lines.push(
430
+ `CPU: no process of the tree gained ${describeCpuThresholds(thresholdsMs)} of CPU over about ${intervalMs}ms between the censuses.`,
431
+ );
432
+ }
351
433
  if (!incomplete) {
352
434
  for (const wait of waits) {
353
435
  lines.push(
@@ -6,10 +6,19 @@ import {
6
6
  grantedSizing,
7
7
  invocationSpan,
8
8
  type ReservedDemand,
9
+ runnerDemand,
9
10
  WANT_ENV,
10
11
  } from './runner-sizing.js';
11
12
  import { shellQuote } from './verdict.js';
12
13
 
14
+ /**
15
+ * mbx's pool priorities, as `MBX_SCHEDULER_PRIORITY` and a reservation name them: interactive work's `interactive`,
16
+ * which mbx ranks first and decays as its job ages, a gate's `normal`, every other job's `low` (cowshed's
17
+ * `PoolPriority`).
18
+ */
19
+ export const MBX_PRIORITIES = ['interactive', 'normal', 'low'] as const;
20
+ export type MbxPriority = (typeof MBX_PRIORITIES)[number];
21
+
13
22
  /**
14
23
  * mbx's machine-wide pool as a leaf runner meets it (specs/cowshed/05_gateway.md, "Host scheduling", "Parallelism inside
15
24
  * a job"). With the mbx capability the canonical environment names the one pool (`MBX_SCHEDULER=1` beside the shared
@@ -17,7 +26,7 @@ import { shellQuote } from './verdict.js';
17
26
  * serves a pool of its own beside it.
18
27
  */
19
28
  export interface MbxPool {
20
- readonly priority: 'normal' | 'low';
29
+ readonly priority: MbxPriority;
21
30
  }
22
31
 
23
32
  /**
@@ -32,7 +41,8 @@ export function mbxPool(environment: Readonly<NodeJS.ProcessEnv>): MbxPool | nul
32
41
  if (environment.MBX_SCHEDULER !== '1' || !environment.MBX_CACHE_DIR || environment.MBX_SCHED_DISABLE === '1') {
33
42
  return null;
34
43
  }
35
- return { priority: environment.MBX_SCHEDULER_PRIORITY === 'normal' ? 'normal' : 'low' };
44
+ const named = MBX_PRIORITIES.find((priority) => priority === environment.MBX_SCHEDULER_PRIORITY);
45
+ return { priority: named ?? 'low' };
36
46
  }
37
47
 
38
48
  /**
@@ -51,7 +61,7 @@ export type MbxLeaf =
51
61
  | { readonly kind: 'cargo' }
52
62
  | {
53
63
  readonly kind: 'reserve';
54
- readonly priority: 'normal' | 'low';
64
+ readonly priority: MbxPriority;
55
65
  readonly demand: ReservedDemand;
56
66
  };
57
67
 
@@ -94,6 +104,17 @@ export function mbxLeaf(pool: MbxPool, demand: Demand, command: string): MbxLeaf
94
104
  return reserve(demand);
95
105
  }
96
106
  }
107
+
108
+ /**
109
+ * Whether a run of `command` under mbx's pool is a Cargo leaf: the one leaf whose admission wait the run reads from
110
+ * mbx's activity report and its waiters' native lineage (mbx-queue.ts; specs/cowshed/11_shell.md, "Cargo test-runner
111
+ * boundaries"), so the only run that loads cowshed's process-lineage entry and its addon. The command alone decides
112
+ * it: the pool's priority, the core count and the environment size a leaf, never its kind.
113
+ */
114
+ export function runsAsCargoLeaf(command: string): boolean {
115
+ return mbxLeaf({ priority: 'low' }, runnerDemand(command, undefined, 1, {}), command)?.kind === 'cargo';
116
+ }
117
+
97
118
  /**
98
119
  * The value an in-place runner gives `--target-dir-remap`: smoo-nx-nextest-binaries' answer, quoted. The plugin infers
99
120
  * the native helper; a repository's own runner may name its script through bun, which prints the same path.
@@ -8,7 +8,7 @@ import type { PostTasksExecutionContext, PreTasksExecutionContext } from 'nx/src
8
8
  import typia, { type tags } from 'typia';
9
9
  import { ownedSocketDirectory } from '../../testing.js';
10
10
  import { CONTROL_MS } from './lease-wait.js';
11
- import { mbxPool } from './mbx-pool.js';
11
+ import { type MbxPriority, mbxPool } from './mbx-pool.js';
12
12
 
13
13
  /**
14
14
  * mbx's reservation endpoint of one Nx run (specs/cowshed/05_gateway.md, "Parallelism inside a job"). A reserved leaf
@@ -16,7 +16,9 @@ import { mbxPool } from './mbx-pool.js';
16
16
  * table recycling. With mbx's pool, the run's `preTasksExecution` starts `mbx reservations <socket>` once and publishes
17
17
  * the socket in `RESERVATIONS_ENV`, which every task inherits; each task's executor asks it for its leaf's cores over
18
18
  * one connection per process (`reserveCores`), and the run's `postTasksExecution` closes the endpoint's standard input,
19
- * which ends it. Admission is `mbx reserve`'s: the same pool, priority and order. A grant lasts until its leaf ends, or
19
+ * which ends it. Nx runs both hooks in the process that runs the tasks, never in a daemon (patches/README.md, "A run's
20
+ * task hooks run in its own process"), so the endpoint is a child of the run and ends with it, whatever becomes of the
21
+ * run's daemon. Admission is `mbx reserve`'s: the same pool, priority and order. A grant lasts until its leaf ends, or
20
22
  * until its connection closes, so an executor that dies gives back what it held.
21
23
  */
22
24
  export const RESERVATIONS_ENV = 'MBX_RESERVATIONS_SOCKET';
@@ -191,7 +193,7 @@ type Uint = number & tags.Type<'uint32'>;
191
193
 
192
194
  /** The lines a client writes (mbx's `docs/scheduling.md`, "Serve reservations to many commands"). */
193
195
  type ReservationRequest =
194
- | { type: 'reserve'; id: Uint; cpus: Uint & tags.Minimum<1>; priority: 'normal' | 'low' }
196
+ | { type: 'reserve'; id: Uint; cpus: Uint & tags.Minimum<1>; priority: MbxPriority }
195
197
  | { type: 'release'; id: Uint };
196
198
 
197
199
  /** The lines the endpoint answers; an added field is mbx's to add. */
@@ -232,7 +234,7 @@ class Connection {
232
234
  this.#socket.on('close', () => this.#close(`mbx reservation endpoint ${this.endpoint} closed the connection`));
233
235
  }
234
236
 
235
- reserve(cpus: number, priority: 'normal' | 'low'): Reservation {
237
+ reserve(cpus: number, priority: MbxPriority): Reservation {
236
238
  if (this.#closed !== null) {
237
239
  return { granted: Promise.resolve({ ok: false, error: this.#closed }), release() {} };
238
240
  }
@@ -317,7 +319,7 @@ const connections = new Map<string, Connection>();
317
319
  * Ask the endpoint at `socket` for at most `cpus` cores at `priority`, over this process's one connection to it,
318
320
  * opened by its first request and kept while another one may come.
319
321
  */
320
- export function reserveCores(socket: string, cpus: number, priority: 'normal' | 'low'): Reservation {
322
+ export function reserveCores(socket: string, cpus: number, priority: MbxPriority): Reservation {
321
323
  let connection = connections.get(socket);
322
324
  if (connection === undefined) {
323
325
  const opened: Connection = new Connection(socket, () => {
@@ -0,0 +1,129 @@
1
+ import { rm } from 'node:fs/promises';
2
+ import { createServer, type Server, type Socket } from 'node:net';
3
+ import { join } from 'node:path';
4
+ import { ownedSocketDirectory } from '../../testing.js';
5
+
6
+ /**
7
+ * A run's progress channel: a running test reports its own forward motion on it, and each record restarts the idle
8
+ * bound as a finished test does (specs/cowshed/08_testing.md, "Integration tests (real substrate)"). An agent harness
9
+ * that enforces its own no-progress bound over what its run does (a span written, a byte of output) reports each
10
+ * movement that check sees, so a long test that keeps moving is not ended because it is the last test still running.
11
+ * CPU is never a record: a compile inside a test reports nothing and is still ended at the bound, where the pre-signal
12
+ * snapshot names it (cpu-rise.ts).
13
+ *
14
+ * The channel is a Unix stream socket the executor serves for one run, in a fresh 0700 directory, and names in the
15
+ * command's environment as {@link PROGRESS_SOCKET_ENV}. Each line a client writes is one record and must be
16
+ * {@link PROGRESS_RECORD}; anything else is a broken reporter, said and failed, never read as progress. A client may
17
+ * write one record per connection or many. The server ends its side when the client ends its own, after every record
18
+ * the client wrote was read, so a client that waits for that end knows its records counted.
19
+ */
20
+ export const PROGRESS_SOCKET_ENV = 'BOUNDED_EXEC_PROGRESS_SOCKET';
21
+ /** The one record: something the test watches moved. */
22
+ export const PROGRESS_RECORD = 'moved';
23
+
24
+ const CHANNEL_DIRECTORY = 'bounded-progress-';
25
+ const CHANNEL_SOCKET = 'p.sock';
26
+ /** How much of a refused line its refusal quotes. */
27
+ const QUOTED_CHARS = 80;
28
+
29
+ export interface ProgressChannel {
30
+ /** The socket's path, as the command's environment names it. */
31
+ readonly socket: string;
32
+ /** Ends every connection and removes the socket's directory; the error that left something behind, or null. */
33
+ close(): Promise<string | null>;
34
+ }
35
+
36
+ export interface ProgressHandlers {
37
+ /** A record was read. */
38
+ readonly moved: () => void;
39
+ /** A client wrote something that is not a record; its connection is ended. */
40
+ readonly invalid: (detail: string) => void;
41
+ }
42
+
43
+ function describe(error: unknown): string {
44
+ return error instanceof Error ? error.message : String(error);
45
+ }
46
+
47
+ function quoted(line: string): string {
48
+ return JSON.stringify(line.length > QUOTED_CHARS ? `${line.slice(0, QUOTED_CHARS)}…` : line);
49
+ }
50
+
51
+ /** Serve one run's progress channel; a channel that cannot be served is the run's refusal, never a silent bound. */
52
+ export async function serveProgressChannel(
53
+ handlers: ProgressHandlers,
54
+ ): Promise<{ ok: true; channel: ProgressChannel } | { ok: false; error: string }> {
55
+ let directory: string;
56
+ try {
57
+ directory = await ownedSocketDirectory(CHANNEL_DIRECTORY, CHANNEL_SOCKET);
58
+ } catch (error) {
59
+ return { ok: false, error: `its directory could not be made: ${describe(error)}` };
60
+ }
61
+ const socketPath = join(directory, CHANNEL_SOCKET);
62
+ const clients = new Set<Socket>();
63
+ const serve = (client: Socket): void => {
64
+ clients.add(client);
65
+ client.on('close', () => clients.delete(client));
66
+ // A client that dies mid-record (its test was killed) loses that record with its connection; nothing else is read.
67
+ client.on('error', () => {});
68
+ client.setEncoding('utf8');
69
+ let partial = '';
70
+ const refuse = (detail: string): void => {
71
+ partial = '';
72
+ client.destroy();
73
+ handlers.invalid(detail);
74
+ };
75
+ client.on('data', (text: string) => {
76
+ const lines = (partial + text).split('\n');
77
+ partial = lines.pop() ?? '';
78
+ for (const line of lines) {
79
+ if (line !== PROGRESS_RECORD) {
80
+ refuse(`a progress record is the line ${PROGRESS_RECORD}, not ${quoted(line)}`);
81
+ return;
82
+ }
83
+ handlers.moved();
84
+ }
85
+ // No prefix longer than the record can still become one.
86
+ if (partial.length > PROGRESS_RECORD.length) {
87
+ refuse(`a progress record is the line ${PROGRESS_RECORD}, not ${quoted(partial)}`);
88
+ }
89
+ });
90
+ client.on('end', () => {
91
+ if (partial !== '') refuse(`a progress connection ended inside a record: ${quoted(partial)}`);
92
+ });
93
+ };
94
+ const server: Server = createServer(serve);
95
+ // eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: listen is this process's own bind of a path in its fresh directory; the kernel answers it with the listening callback or an error, and no peer takes part
96
+ const listened = await new Promise<Error | null>((settle) => {
97
+ server.once('error', settle);
98
+ server.listen(socketPath, () => {
99
+ server.off('error', settle);
100
+ settle(null);
101
+ });
102
+ });
103
+ const close = async (): Promise<string | null> => {
104
+ const failures: string[] = [];
105
+ for (const client of clients) client.destroy();
106
+ if (listened === null) {
107
+ // eslint-disable-next-line deadline/no-unbounded-wait -- WHY unbounded: every accepted connection was destroyed just above, so the server closes without waiting on a peer
108
+ const closed = await new Promise<Error | undefined>((settle) => server.close(settle));
109
+ if (closed !== undefined) failures.push(`its socket did not close: ${closed.message}`);
110
+ }
111
+ try {
112
+ await rm(directory, { recursive: true, force: true });
113
+ } catch (error) {
114
+ failures.push(`its directory was not removed: ${describe(error)}`);
115
+ }
116
+ return failures.length === 0 ? null : `the progress channel at ${socketPath}: ${failures.join('; ')}`;
117
+ };
118
+ if (listened !== null) {
119
+ const left = await close();
120
+ return {
121
+ ok: false,
122
+ error: `it could not listen at ${socketPath}: ${listened.message}${left === null ? '' : `; ${left}`}`,
123
+ };
124
+ }
125
+ server.on('error', (error) => {
126
+ handlers.invalid(`the progress channel at ${socketPath} failed: ${error.message}`);
127
+ });
128
+ return { ok: true, channel: { socket: socketPath, close } };
129
+ }
@@ -6,7 +6,8 @@ import { BUN_TEST, GO_TEST, GO_TEST_RUNNER, NEXTEST_RUN } from './verdict.js';
6
6
 
7
7
  /**
8
8
  * What a run's output says about its forward motion: whether a test finished, and which tests the runner said
9
- * are running and has not said finished. `idleTimeoutMs` re-arms only on a finished test.
9
+ * are running and has not said finished. In the output, `idleTimeoutMs` re-arms only on a finished test; a running test
10
+ * reports its own progress on the run's progress channel instead (progress-channel.ts).
10
11
  *
11
12
  * Output alone is not progress. nextest repeats a `SLOW` line for every test still running at each slow-timeout
12
13
  * period, and a hung test may print a heartbeat, so a bound on silence never fires on either. A finished test is