@ultimat3/cli 27.12.0 → 27.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CLAUDE.md CHANGED
@@ -45,7 +45,7 @@ Commands: `bun test packages/cli` (from the repo root — the test preload lives
45
45
  | `verify-checks.ts` / `verify-step.ts` / `verify-run.ts` | the step list, the outcome shape, the run order (`BESIDE_SERIAL_SUITES`) |
46
46
  | `verify-floor.ts` | `x.verify.json`: a floor step that ran zero tests is `X_VERIFY_SUITE_VANISHED` |
47
47
  | `coverage-floor.ts` / `coverage-lcov.ts` / `coverage-source.ts` / `verify-coverage-run.ts` | an APP's `unit` step holds `coverage` in `x.verify.json` over its whole source tree. The suite runs as fixed slices, one plain `bun test` each — never `--parallel --coverage`, whose merge moves with the core count. `coverage-lcov.ts` is the ONE lcov reader; `scripts/coverage-gate.ts` reads through it. A shard defers to `verify-merge.ts` (`coverageRiders`), which also holds the shards' `files` to a partition of the corpus they hashed and reads each part step through `verify-part-step.ts`; `doctor-coverage.ts` lists the excludes |
48
- | `verify-deadline.ts` / `verify-stalled.ts` / `verify-progress.ts` | a step past its deadline is `X_VERIFY_STEP_TIMEOUT`: its children carry `ULTIMATE_VERIFY_STEP` and are killed by that tag; the finding names the test file `bun test` had not finished (`at`, `meta.inFlight`) and its `fix:` runs that file; `--json` streams a line per finished step to stderr. A `fix:` names `ctx.command` — `bun run verify` at the framework root |
48
+ | `verify-deadline.ts` / `verify-stalled.ts` / `verify-progress.ts` | a step past its deadline is `X_VERIFY_STEP_TIMEOUT`: its children carry `ULTIMATE_VERIFY_STEP` and are killed by that tag; the finding names the test file `bun test` had not finished (`at`, `meta.inFlight`) and its `fix:` runs that file; any render streams step lines + a heartbeat to stderr. A `fix:` names `ctx.command` — `bun run verify` at the framework root |
49
49
  | `load-findings.ts` | module-load failures are reported once, by `manifest`; other steps point there |
50
50
  | `boundary-findings.ts` / `app-boundaries.ts` / `boundary-cuts.ts` | surface + layer rules; a finding's `fix:` is the concrete cut |
51
51
  | `app-transport.ts` / `browser-transport.ts` / `transport-calls.ts` / `import-closure.ts` / `server-barrels.ts` | `boundaries`, built in: one browser transport. `app-transport.ts` reuses the files `readAppSources` read (no second walk), resolves through the app's tsconfig `paths` and workspace manifests, reads the app's own `packages/*` lazily, and STOPS at `node_modules`. `browser-transport.ts` is the pure rule — `scripts/browser-transport.ts` calls the same function with this repo's seam FILES. Never a `guards/` file: an app must not be able to delete it |
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@ultimat3/cli",
3
- "version": "27.12.0",
3
+ "version": "27.13.0",
4
4
  "description": "The `x` binary: new, dev, build, verify, generate, db, mcp, doctor, deploy",
5
5
  "license": "MIT",
6
6
  "type": "module",
@@ -39,33 +39,33 @@
39
39
  },
40
40
  "dependencies": {
41
41
  "@babel/core": "^8.0.1",
42
- "@ultimat3/action": "27.12.0",
43
- "@ultimat3/admin": "27.12.0",
44
- "@ultimat3/ai": "27.12.0",
45
- "@ultimat3/auth": "27.12.0",
46
- "@ultimat3/cache": "27.12.0",
47
- "@ultimat3/core": "27.12.0",
48
- "@ultimat3/db": "27.12.0",
49
- "@ultimat3/entity": "27.12.0",
50
- "@ultimat3/flags": "27.12.0",
51
- "@ultimat3/http": "27.12.0",
52
- "@ultimat3/i18n": "27.12.0",
53
- "@ultimat3/jobs": "27.12.0",
54
- "@ultimat3/mail": "27.12.0",
55
- "@ultimat3/manifest": "27.12.0",
56
- "@ultimat3/mcp": "27.12.0",
57
- "@ultimat3/money": "27.12.0",
58
- "@ultimat3/notify": "27.12.0",
59
- "@ultimat3/policy": "27.12.0",
60
- "@ultimat3/pwa": "27.12.0",
61
- "@ultimat3/query": "27.12.0",
62
- "@ultimat3/realtime": "27.12.0",
63
- "@ultimat3/render": "27.12.0",
64
- "@ultimat3/schema": "27.12.0",
65
- "@ultimat3/seo": "27.12.0",
66
- "@ultimat3/storage": "27.12.0",
67
- "@ultimat3/testing": "27.12.0",
68
- "@ultimat3/time": "27.12.0",
42
+ "@ultimat3/action": "27.13.0",
43
+ "@ultimat3/admin": "27.13.0",
44
+ "@ultimat3/ai": "27.13.0",
45
+ "@ultimat3/auth": "27.13.0",
46
+ "@ultimat3/cache": "27.13.0",
47
+ "@ultimat3/core": "27.13.0",
48
+ "@ultimat3/db": "27.13.0",
49
+ "@ultimat3/entity": "27.13.0",
50
+ "@ultimat3/flags": "27.13.0",
51
+ "@ultimat3/http": "27.13.0",
52
+ "@ultimat3/i18n": "27.13.0",
53
+ "@ultimat3/jobs": "27.13.0",
54
+ "@ultimat3/mail": "27.13.0",
55
+ "@ultimat3/manifest": "27.13.0",
56
+ "@ultimat3/mcp": "27.13.0",
57
+ "@ultimat3/money": "27.13.0",
58
+ "@ultimat3/notify": "27.13.0",
59
+ "@ultimat3/policy": "27.13.0",
60
+ "@ultimat3/pwa": "27.13.0",
61
+ "@ultimat3/query": "27.13.0",
62
+ "@ultimat3/realtime": "27.13.0",
63
+ "@ultimat3/render": "27.13.0",
64
+ "@ultimat3/schema": "27.13.0",
65
+ "@ultimat3/seo": "27.13.0",
66
+ "@ultimat3/storage": "27.13.0",
67
+ "@ultimat3/testing": "27.13.0",
68
+ "@ultimat3/time": "27.13.0",
69
69
  "babel-preset-solid": "^1.9.15"
70
70
  }
71
71
  }
package/src/cmd-build.ts CHANGED
@@ -26,6 +26,7 @@ import {
26
26
  renderStaticReport,
27
27
  staticReportData,
28
28
  } from './static-report';
29
+ import { measurePace } from './verify-pace';
29
30
  import type { VerifyStepName } from './verify-step';
30
31
 
31
32
  /**
@@ -385,7 +386,12 @@ export const buildCommand: CliCommand = {
385
386
  const verifySteps = (await import('./cmd-verify')).VERIFY_STEPS.filter((step) =>
386
387
  PREFLIGHT_STEPS.includes(step.name),
387
388
  );
388
- const verifyResult = await runVerify(verifySteps, { root, runner: ctx.runner, env: ctx.env });
389
+ const verifyResult = await runVerify(verifySteps, {
390
+ root,
391
+ runner: ctx.runner,
392
+ env: ctx.env,
393
+ pace: measurePace(),
394
+ });
389
395
  if (!verifyResult.ok) {
390
396
  return preflightResult(verifyResult);
391
397
  }
package/src/cmd-verify.ts CHANGED
@@ -23,7 +23,8 @@ import { VERIFY_STEPS } from './verify-checks';
23
23
  import { readVerifyFloor } from './verify-floor';
24
24
  import type { VerifyPart } from './verify-merge';
25
25
  import { coverageRiders, mergeParts, parsePart, VerifyMergeInputError } from './verify-merge';
26
- import { stepStream } from './verify-progress';
26
+ import { measurePace } from './verify-pace';
27
+ import { withVerifyProgress } from './verify-progress';
27
28
  import { runVerify } from './verify-run';
28
29
  import type { ShardSpec, Timings } from './verify-shard';
29
30
  import { assertShardable, parseShard, readTimings, VerifyShardInvalidError } from './verify-shard';
@@ -48,18 +49,22 @@ export const verifyCommand: CliCommand = {
48
49
  const only = readOnlySteps(ctx.args);
49
50
  const shard = await readShard(ctx.args, only);
50
51
  const isolate = flagBool(ctx.args, 'isolate') ? true : undefined;
51
- const result = await runVerify(VERIFY_STEPS, {
52
- root,
53
- runner: ctx.runner,
54
- env: ctx.env,
55
- ...(workers === undefined ? {} : { workers }),
56
- ...(only === undefined ? {} : { only }),
57
- ...(shard === undefined ? {} : { shard }),
58
- ...(isolate === undefined ? {} : { isolate }),
59
- // `--json` prints one document when the run ends; a line per finished step on stderr is
60
- // what a cancelled CI job's log ends on (#589). stdout stays the single document.
61
- ...stepStream(ctx.args.json, writeErrorLine),
62
- });
52
+ // Either render prints its verdict only when the run ends; the progress on stderr — a line per
53
+ // finished step, a heartbeat while one runs — is what a cancelled job's log ends on (#589) and
54
+ // what keeps a silence-killing harness from killing a healthy gate. stdout stays the document.
55
+ const result = await withVerifyProgress(writeErrorLine, (progress) =>
56
+ runVerify(VERIFY_STEPS, {
57
+ root,
58
+ runner: ctx.runner,
59
+ env: ctx.env,
60
+ pace: measurePace(),
61
+ ...(workers === undefined ? {} : { workers }),
62
+ ...(only === undefined ? {} : { only }),
63
+ ...(shard === undefined ? {} : { shard }),
64
+ ...(isolate === undefined ? {} : { isolate }),
65
+ ...progress,
66
+ }),
67
+ );
63
68
  return shard === undefined ? result : withShardData(result, shard);
64
69
  },
65
70
  };
package/src/mcp-host.ts CHANGED
@@ -55,6 +55,7 @@ import { resolveServices } from './runtime-bindings';
55
55
  import type { RunningServices } from './runtime-services';
56
56
  import { startServices } from './runtime-services';
57
57
  import { testEnvOverrides } from './test-dotenv';
58
+ import { measurePace } from './verify-pace';
58
59
 
59
60
  export interface DevHostInput {
60
61
  readonly root: string;
@@ -317,7 +318,7 @@ function capabilities(
317
318
  // The one safe autofix this repo actually has. Anything more would be the gate rewriting
318
319
  // code it was asked to judge.
319
320
  if (fix) await runner(['bunx', 'biome', 'check', '--write', '.'], { cwd: root });
320
- const result = await runVerify(VERIFY_STEPS, { root, runner });
321
+ const result = await runVerify(VERIFY_STEPS, { root, runner, pace: measurePace() });
321
322
  return {
322
323
  ok: result.ok,
323
324
  steps: (result.steps ?? []).map((step): VerifyStep => {
package/src/messages.ts CHANGED
@@ -337,6 +337,9 @@ const CATALOG = {
337
337
  // no skipped test to read a reason off, so the files the step selected hold no test.
338
338
  'cli.verify.ranNothing': 'found no test to run in the file(s) it selected',
339
339
  'cli.verify.merged': 'merged {parts} part(s) — {summary}',
340
+ // The heartbeat `verify-progress.ts` writes while a step is still running: `{steps}` is
341
+ // `unit (12m 3s), live (4m 0s)`, each step with the time since it started.
342
+ 'cli.verify.stillRunning': 'still running: {steps}',
340
343
  // The `unit` step's own measurement, in `output`: what the suite covered of the app's whole
341
344
  // source tree, beside the floor it was held to. A shard says it measured a slice instead.
342
345
  'cli.verify.coverage':
@@ -24,6 +24,8 @@ import { countsOf } from './test-counts';
24
24
  import { testEnvOverrides } from './test-dotenv';
25
25
  import { failureOf } from './test-shards';
26
26
  import { SLOT_HELD_ENV } from './test-slots';
27
+ import type { MachinePace } from './verify-pace';
28
+ import { defaultTimeoutArgs, paceEnv } from './verify-pace';
27
29
  import type { StepOutcome } from './verify-step';
28
30
 
29
31
  /**
@@ -83,6 +85,8 @@ export interface CoveredRunOptions {
83
85
  readonly lease?: (want: number) => Promise<{ readonly count: number; release(): void }>;
84
86
  /** Test files per process. Absent is `COVERAGE_SLICE_FILES`; a test cuts smaller. */
85
87
  readonly sliceFiles?: number;
88
+ /** The machine's pace (`verify-pace.ts`): each test's default and declared deadline stretch by it. */
89
+ readonly pace?: MachinePace;
86
90
  }
87
91
 
88
92
  export interface CoveredRun extends StepOutcome {
@@ -95,10 +99,12 @@ export const coveredArgs = (
95
99
  files: readonly string[],
96
100
  dir: string,
97
101
  isolate: boolean,
102
+ pace?: MachinePace,
98
103
  ): readonly string[] => [
99
104
  'bun',
100
105
  'test',
101
106
  ...(isolate ? ['--isolate'] : []),
107
+ ...defaultTimeoutArgs(pace),
102
108
  '--coverage',
103
109
  '--coverage-reporter=lcov',
104
110
  `--coverage-dir=${dir}`,
@@ -116,6 +122,7 @@ export async function runCovered(options: CoveredRunOptions): Promise<CoveredRun
116
122
  ...testEnvOverrides(options.root, options.env ?? Bun.env),
117
123
  ...(isolate ? { [ISOLATED_ENV]: '1' } : {}),
118
124
  ...(lease === undefined ? {} : { [SLOT_HELD_ENV]: '1' }),
125
+ ...paceEnv(options.pace),
119
126
  };
120
127
  const results = new Array<ExecResult>(slices.length);
121
128
  let next = 0;
@@ -125,7 +132,12 @@ export async function runCovered(options: CoveredRunOptions): Promise<CoveredRun
125
132
  const index = next;
126
133
  next += 1;
127
134
  results[index] = await options.runner(
128
- coveredArgs(slices[index] as readonly string[], join(base, String(index)), isolate),
135
+ coveredArgs(
136
+ slices[index] as readonly string[],
137
+ join(base, String(index)),
138
+ isolate,
139
+ options.pace,
140
+ ),
129
141
  { cwd: options.root, env: { ...overrides, ULTIMATE_TEST_WORKER: String(slot) } },
130
142
  );
131
143
  }
@@ -10,12 +10,16 @@ import type { VerifyStepName } from './verify-step';
10
10
  /** The `x.verify.json` key: `{ "stepTimeoutMs": { "unit": 900000 } }`. */
11
11
  export const STEP_TIMEOUT_FIELD = 'stepTimeoutMs';
12
12
 
13
- /** A check step — the compiler, biome, an in-process scan. Five minutes. */
13
+ /**
14
+ * A check step — the compiler, biome, an in-process scan. Five minutes at a 4-core CI runner's
15
+ * pace; a slower machine runs it under this times its own (`stepDeadline`, `verify-pace.ts`).
16
+ */
14
17
  export const CHECK_STEP_TIMEOUT_MS = 300_000;
15
18
 
16
19
  /**
17
20
  * A test suite. Eight minutes: under the ten a CI job is commonly given, so the step names itself
18
21
  * before the runner cancels the job — the framework's own `unit` step is ~3 on a 4-core runner.
22
+ * Stretched on a slower machine the same way as the check number.
19
23
  */
20
24
  export const SUITE_STEP_TIMEOUT_MS = 480_000;
21
25
 
@@ -26,16 +30,27 @@ export const defaultStepTimeoutMs = (step: string): number =>
26
30
 
27
31
  export type StepTimeouts = Readonly<Partial<Record<VerifyStepName, number>>>;
28
32
 
29
- /** The caller's override, then the floor file's, then the step class's declared number. */
30
- export const stepTimeoutMs = (
33
+ /**
34
+ * The deadline a step runs under. `pace` is set when it is the step class's declared number
35
+ * stretched to this machine (`verify-pace.ts`), so the finding past it can say so.
36
+ */
37
+ export type StepDeadline = { readonly ms: number; readonly pace?: number };
38
+
39
+ /**
40
+ * The caller's override, then the floor file's — each a number someone wrote down for this repo,
41
+ * taken as written — then the step class's declared number at this machine's pace `factor`.
42
+ */
43
+ export const stepDeadline = (
31
44
  step: VerifyStepName,
32
- ...overrides: readonly (StepTimeouts | undefined)[]
33
- ): number => {
45
+ overrides: readonly (StepTimeouts | undefined)[],
46
+ factor: number,
47
+ ): StepDeadline => {
34
48
  for (const table of overrides) {
35
49
  const ms = table !== undefined && Object.hasOwn(table, step) ? table[step] : undefined;
36
- if (ms !== undefined) return ms;
50
+ if (ms !== undefined) return { ms };
37
51
  }
38
- return defaultStepTimeoutMs(step);
52
+ const declared = defaultStepTimeoutMs(step);
53
+ return factor === 1 ? { ms: declared } : { ms: Math.round(declared * factor), pace: factor };
39
54
  };
40
55
 
41
56
  /**
@@ -0,0 +1,120 @@
1
+ // How much slower this machine runs the gate than the one its declared numbers were measured on.
2
+ // Every deadline the gate declares — a step's (`verify-deadline.ts`), Bun's 5 s per test, a test's
3
+ // own — was set against a 4-core CI runner, and on a 2-core 1.8 GHz box the same honest work runs
4
+ // five to seven times longer: the gate failed there on the deadlines, never on the code. Measured
5
+ // once per gate, before any step runs, and applied to every one of those numbers.
6
+
7
+ import { TEST_PACE_ENV } from '@ultimat3/testing';
8
+ import { availableCpus } from './test-workers';
9
+
10
+ /**
11
+ * The width the declared deadlines were measured at: a GitHub-hosted `ubuntu-latest` runner on a
12
+ * public repository has 4 cores, and the `unit` deadline's "~3 minutes" is on one.
13
+ */
14
+ export const REFERENCE_CPUS = 4;
15
+
16
+ /**
17
+ * What `probeWork` takes on that runner, the median of `PROBE_RUNS`: the SLOWEST of the five gate
18
+ * parts' readings, so the runner's own spread reads as pace 1 and its deadlines stay the declared
19
+ * ones. MEASURED on GitHub's `ubuntu-latest`, `As of 2026-10-19`: 31, 32, 32, 34, 35 ms (`ci` run
20
+ * 39208102081, each part's `data.pace.probeMs`). An idle 3.1 GHz Haswell core reads 65-90 ms.
21
+ */
22
+ export const REFERENCE_PROBE_MS = 35;
23
+
24
+ /** Rows the probe allocates, round-trips through JSON and sorts: ~25 ms a run on the reference. */
25
+ export const PROBE_ROWS = 30_000;
26
+
27
+ /** Timed runs after one untimed warm-up, so the JIT is not what gets measured. */
28
+ export const PROBE_RUNS = 5;
29
+
30
+ /** The machine's pace, and the two readings it was computed from — `--json` carries all three. */
31
+ export type MachinePace = {
32
+ /** Every declared deadline is multiplied by this. Never below 1: a fast machine keeps the number. */
33
+ readonly factor: number;
34
+ readonly cpus: number;
35
+ /** The probe's median, in milliseconds. */
36
+ readonly probeMs: number;
37
+ };
38
+
39
+ /** Pace 1: the numbers as declared. What a caller that measured nothing runs at. */
40
+ export const DECLARED_PACE: MachinePace = {
41
+ factor: 1,
42
+ cpus: REFERENCE_CPUS,
43
+ probeMs: REFERENCE_PROBE_MS,
44
+ };
45
+
46
+ /**
47
+ * Fewer cores than the reference is a proportionally longer parallel step, and a slower core is a
48
+ * proportionally longer everything: the two multiply. Each is held at 1 from below, so a wide or
49
+ * fast machine is never handed a TIGHTER deadline than the one declared. Two decimals: the number
50
+ * is printed, and `ULTIMATE_TEST_PACE` carries it to every test run.
51
+ */
52
+ export function paceOf(cpus: number, probeMs: number): MachinePace {
53
+ const width = Math.max(1, REFERENCE_CPUS / Math.max(1, cpus));
54
+ const speed = Math.max(1, probeMs / REFERENCE_PROBE_MS);
55
+ return { factor: Math.round(width * speed * 100) / 100, cpus, probeMs };
56
+ }
57
+
58
+ /** A declared deadline at this pace. */
59
+ export const pacedMs = (ms: number, pace: MachinePace): number => Math.round(ms * pace.factor);
60
+
61
+ /**
62
+ * Fixed single-threaded work of the kind the gate does — allocate, serialize, parse, sort, scan
63
+ * strings — so a slower core, and a core shared with whatever else the box runs, both read slower.
64
+ * The result is folded into the return value so the work cannot be optimized away.
65
+ */
66
+ export function probeWork(): number {
67
+ const rows = Array.from({ length: PROBE_ROWS }, (_, i) => ({
68
+ id: `row-${String((i * 7919) % PROBE_ROWS)}`,
69
+ n: i,
70
+ tags: [i % 3, i % 5],
71
+ }));
72
+ const back = JSON.parse(JSON.stringify(rows)) as typeof rows;
73
+ back.sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
74
+ return back
75
+ .map((row) => row.id)
76
+ .join('\n')
77
+ .split('\n')
78
+ .filter((line) => line.endsWith('7')).length;
79
+ }
80
+
81
+ const median = (values: readonly number[]): number => {
82
+ const sorted = [...values].sort((a, b) => a - b);
83
+ return sorted[Math.floor(sorted.length / 2)] ?? 0;
84
+ };
85
+
86
+ /** The probe's median over `PROBE_RUNS`, in whole milliseconds. */
87
+ export function probeMs(work: () => number = probeWork): number {
88
+ work();
89
+ const runs = Array.from({ length: PROBE_RUNS }, () => {
90
+ const started = performance.now();
91
+ work();
92
+ return performance.now() - started;
93
+ });
94
+ return Math.round(median(runs));
95
+ }
96
+
97
+ /** This machine, now: what an entry point (`x verify`, `x build`, the MCP host) hands the gate. */
98
+ export const measurePace = (): MachinePace => paceOf(availableCpus(), probeMs());
99
+
100
+ /** Bun's own per-test default: what every test and hook without a number of its own runs under. */
101
+ export const BUN_TEST_TIMEOUT_MS = 5_000;
102
+
103
+ /**
104
+ * `--timeout` for a test run at this pace, so a test with no number of its own — and every
105
+ * `beforeEach` booting an embedded Postgres — gets Bun's 5 s stretched like everything else. None
106
+ * at pace 1 or unmeasured: Bun's default already is the number, and the argv stays the one a
107
+ * developer types.
108
+ */
109
+ export const defaultTimeoutArgs = (pace: MachinePace | undefined): readonly string[] =>
110
+ pace === undefined || pace.factor === 1
111
+ ? []
112
+ : [`--timeout=${String(pacedMs(BUN_TEST_TIMEOUT_MS, pace))}`];
113
+
114
+ /**
115
+ * What a test run started at this pace is told, so its own declared deadlines stretch with it
116
+ * (`@ultimat3/testing`'s `paced`). Set at 1 too, whenever a pace was measured: a gate nested in a
117
+ * test must not inherit the outer run's pace as its own. Nothing when none was measured.
118
+ */
119
+ export const paceEnv = (pace: MachinePace | undefined): Readonly<Record<string, string>> =>
120
+ pace === undefined ? {} : { [TEST_PACE_ENV]: String(pace.factor) };
@@ -1,8 +1,31 @@
1
- // One finished gate step as one line of JSON, for stderr. `x verify --json` prints its document
2
- // when the run ENDS, so a job cancelled mid-run used to leave a log with nothing in it (#589):
3
- // the last of these lines is the last step that finished, and the hung one is whichever is next.
1
+ // What the gate writes to stderr WHILE it runs, in every render mode: a line per finished step and
2
+ // a heartbeat naming the steps still running. Both entries print their verdict only at the end, so
3
+ // a run killed mid-way left nothing (#589), and a harness that kills on SILENCE killed a healthy one.
4
4
 
5
+ import type { Clock } from '@ultimat3/core';
6
+ import { systemClock } from '@ultimat3/core';
7
+ import { msg } from './messages';
5
8
  import type { StepResult } from './output';
9
+ import type { VerifyStepName } from './verify-step';
10
+
11
+ /**
12
+ * How often the heartbeat names the steps still running. Bounded by the silence budget of the
13
+ * harness that runs the gate, not by cost: an agent runner or CI job that kills a process after
14
+ * 30 minutes with no new output byte killed green gates that finished seconds past it, and one
15
+ * step on a slow 2-core box can outlast that window on its own, so a line per finished step is
16
+ * not enough. A minute is far inside any silence window a harness sets.
17
+ */
18
+ export const HEARTBEAT_MS = 60_000;
19
+
20
+ /** A repeating timer that returns its own stop — injected so a test advances time, never sleeps. */
21
+ export type Every = (tick: () => void, intervalMs: number) => () => void;
22
+
23
+ /** Unref'd: a heartbeat must never be the thing that keeps a finished gate's process alive. */
24
+ const unrefInterval: Every = (tick, intervalMs) => {
25
+ const timer = setInterval(tick, intervalMs);
26
+ timer.unref();
27
+ return () => clearInterval(timer);
28
+ };
6
29
 
7
30
  /** `{"step":"lint","ok":true,"ms":21987}` — plus `"skipped":true` for a step that did not apply. */
8
31
  export const stepLine = (step: StepResult): string =>
@@ -13,13 +36,63 @@ export const stepLine = (step: StepResult): string =>
13
36
  ...(step.skipped === true ? { skipped: true } : {}),
14
37
  });
15
38
 
39
+ /** `12m 3s`, `45s`: a person reading a CI log, not a parser — the step lines carry exact ms. */
40
+ export const elapsedLabel = (ms: number): string => {
41
+ const seconds = Math.floor(ms / 1000);
42
+ const minutes = Math.floor(seconds / 60);
43
+ return minutes === 0 ? `${seconds}s` : `${minutes}m ${seconds % 60}s`;
44
+ };
45
+
46
+ /** `still running: unit (12m 3s), live (4m 0s)` — each step with the time since IT started. */
47
+ export const heartbeatLine = (running: ReadonlyMap<string, number>, now: number): string =>
48
+ msg('cli.verify.stillRunning', {
49
+ steps: [...running].map(([name, began]) => `${name} (${elapsedLabel(now - began)})`).join(', '),
50
+ });
51
+
52
+ /** The two `VerifyContext` listeners a progress run hands the gate. */
53
+ export interface VerifyProgressHooks {
54
+ readonly onStepStart: (name: VerifyStepName) => void;
55
+ readonly onStep: (step: StepResult) => void;
56
+ }
57
+
58
+ export interface VerifyProgressOptions {
59
+ readonly clock?: Clock;
60
+ readonly every?: Every;
61
+ }
62
+
16
63
  /**
17
- * The `onStep` a `--json` run streams through, or nothing: the human render needs no second
18
- * channel. One function for both entries (`x verify`, and `bun run verify` at the framework
19
- * root), so the two cannot print different lines.
64
+ * Run the gate with progress on `write` (stderr), the heartbeat stopped however `run` ends. One
65
+ * function for both entries (`x verify`, and `bun run verify` at the framework root), so the two
66
+ * cannot print different lines; stdout is untouched, so `--json` stays one document there.
20
67
  */
21
- export const stepStream = (
22
- json: boolean,
68
+ export async function withVerifyProgress<T>(
23
69
  write: (line: string) => void,
24
- ): { readonly onStep?: (step: StepResult) => void } =>
25
- json ? { onStep: (step) => write(stepLine(step)) } : {};
70
+ run: (hooks: VerifyProgressHooks) => Promise<T>,
71
+ options: VerifyProgressOptions = {},
72
+ ): Promise<T> {
73
+ const clock = options.clock ?? systemClock;
74
+ const running = new Map<string, number>();
75
+ const stop = (options.every ?? unrefInterval)(() => {
76
+ if (running.size === 0) return;
77
+ // Inside a timer callback a throw is an uncaught exception: stderr closed under the gate must
78
+ // not take the run down with it. The verdict still reaches stdout.
79
+ try {
80
+ write(heartbeatLine(running, clock.monotonic()));
81
+ } catch {
82
+ // Nowhere left to say it.
83
+ }
84
+ }, HEARTBEAT_MS);
85
+ try {
86
+ return await run({
87
+ onStepStart: (name) => {
88
+ running.set(name, clock.monotonic());
89
+ },
90
+ onStep: (step) => {
91
+ running.delete(step.name);
92
+ write(stepLine(step));
93
+ },
94
+ });
95
+ } finally {
96
+ stop();
97
+ }
98
+ }
package/src/verify-run.ts CHANGED
@@ -7,8 +7,8 @@ import type { CoverageMap } from './coverage-lcov';
7
7
  import { encodeCoverage } from './coverage-lcov';
8
8
  import { msg } from './messages';
9
9
  import type { CommandResult, Finding, JsonValue, StepResult } from './output';
10
- import type { StepGuard } from './verify-deadline';
11
- import { guardStep, raceDeadline, stepTimeoutMs } from './verify-deadline';
10
+ import type { StepDeadline, StepGuard } from './verify-deadline';
11
+ import { guardStep, raceDeadline, stepDeadline } from './verify-deadline';
12
12
  import {
13
13
  floorRequires,
14
14
  readVerifyFloor,
@@ -49,6 +49,12 @@ export async function runVerify(
49
49
  // other's children.
50
50
  const runId = crypto.randomUUID().slice(0, 8);
51
51
  const run = async (step: VerifyStep, stepCtx: VerifyContext): Promise<void> => {
52
+ // A progress listener, like `onStep` below: one that throws must not take the gate down.
53
+ try {
54
+ ctx.onStepStart?.(step.name);
55
+ } catch {
56
+ // The step still runs; its line still reaches `onStep` and the document.
57
+ }
52
58
  const ran = await runStep(step, stepCtx, floor, runId);
53
59
  byName.set(step.name, ran.result);
54
60
  if (ran.coverage !== undefined) {
@@ -131,6 +137,9 @@ export async function runVerify(
131
137
  // `only` is always the LIST, in declared order — one name is a list of one.
132
138
  ...(only === undefined ? {} : { notAGateRun: true, only: [...only] }),
133
139
  ...(coverage.size === 0 ? {} : { coverage: Object.fromEntries(coverage) }),
140
+ // The machine the deadlines were stretched for, when the entry point measured one: a step
141
+ // that timed out at ×1 and one that timed out at ×6 are different findings.
142
+ ...(ctx.pace === undefined ? {} : { pace: { ...ctx.pace } }),
134
143
  },
135
144
  // The step's own status: one step, so `failedSteps` is that step and nothing else.
136
145
  exitCode: failedSteps.length === 0 ? 0 : 1,
@@ -239,7 +248,11 @@ async function runStep(
239
248
  // The deadline: a step that hangs fails BY NAME and its processes are killed, instead of eating
240
249
  // the CI job's whole timeout and leaving a log that says nothing (#589). The step runs on a
241
250
  // guarded runner, so everything it starts carries its tag — that is what the kill finds.
242
- const limit = stepTimeoutMs(step.name, ctx.stepTimeoutMs, floor?.stepTimeoutMs);
251
+ const deadline = stepDeadline(
252
+ step.name,
253
+ [ctx.stepTimeoutMs, floor?.stepTimeoutMs],
254
+ ctx.pace?.factor ?? 1,
255
+ );
243
256
  const guard = guardStep(ctx.runner, `${step.name}@${runId}`);
244
257
  const raced = await raceDeadline(
245
258
  step.run({ ...ctx, runner: guard.runner }).catch(
@@ -248,10 +261,10 @@ async function runStep(
248
261
  findings: [findingOf(error, step.name, command)],
249
262
  }),
250
263
  ),
251
- limit,
264
+ deadline.ms,
252
265
  );
253
266
  const outcome: StepOutcome = raced.timedOut
254
- ? await expired(step.name, limit, guard, command)
267
+ ? await expired(step.name, deadline, guard, command)
255
268
  : raced.value;
256
269
  // A suite that executed nothing did not run, whatever its exit code says: `bun test` exits 0
257
270
  // over an all-skipped file, so the counts are the only channel that can tell the two apart.
@@ -293,7 +306,7 @@ async function runStep(
293
306
  /** The timed-out step's outcome: the finding naming what was in flight, over what it last printed. */
294
307
  async function expired(
295
308
  step: VerifyStepName,
296
- limit: number,
309
+ deadline: StepDeadline,
297
310
  guard: StepGuard,
298
311
  command: string,
299
312
  ): Promise<StepOutcome> {
@@ -301,7 +314,7 @@ async function expired(
301
314
  const output = stalledOutput(expiry);
302
315
  return {
303
316
  ok: false,
304
- findings: [stepTimeoutFinding(step, limit, expiry, command)],
317
+ findings: [stepTimeoutFinding(step, deadline, expiry, command)],
305
318
  ...(output === undefined ? {} : { output }),
306
319
  };
307
320
  }
@@ -5,7 +5,7 @@
5
5
  import { ERROR_DOCS_URL } from '@ultimat3/core';
6
6
  import type { Finding } from './output';
7
7
  import { quoteArg } from './shell-quote';
8
- import type { InFlightRun, StepExpiry, StepProgress } from './verify-deadline';
8
+ import type { InFlightRun, StepDeadline, StepExpiry, StepProgress } from './verify-deadline';
9
9
  import { STEP_TIMEOUT_FIELD } from './verify-deadline';
10
10
  import type { VerifyStepName } from './verify-step';
11
11
 
@@ -30,6 +30,8 @@ export type StalledRun = {
30
30
  export type StepTimeoutMeta = {
31
31
  readonly step: string;
32
32
  readonly deadlineMs: number;
33
+ /** Set when `deadlineMs` is the declared number stretched to this machine (`verify-pace.ts`). */
34
+ readonly pace?: number;
33
35
  readonly killed: readonly { readonly pid: number; readonly command: string }[];
34
36
  readonly inFlight: readonly StalledRun[];
35
37
  /** The runs that had finished before the deadline — the step's progress. */
@@ -147,6 +149,12 @@ const inFlightClause = (runs: readonly StalledRun[], expiry: StepExpiry): string
147
149
  : `; its running "${runs[0].command}" named no file still in flight, so the run itself did not exit${waiting.runningMs === undefined ? '' : ` after ${seconds(waiting.runningMs)} s`}`;
148
150
  };
149
151
 
152
+ /** Said only when the deadline was stretched: a reader comparing it with the declared one asks. */
153
+ const pacedClause = (deadline: StepDeadline): string =>
154
+ deadline.pace === undefined
155
+ ? ''
156
+ : ` (the declared ${String(Math.round(deadline.ms / deadline.pace))} ms at this machine's pace, ×${String(deadline.pace)})`;
157
+
150
158
  /**
151
159
  * A step that ran past its deadline, on that step's own line. `command` is the gate as it was
152
160
  * invoked here (`x verify`, or `bun run verify` at the framework root), so the fix runs where the
@@ -155,10 +163,11 @@ const inFlightClause = (runs: readonly StalledRun[], expiry: StepExpiry): string
155
163
  */
156
164
  export const stepTimeoutFinding = (
157
165
  step: VerifyStepName,
158
- ms: number,
166
+ deadline: StepDeadline,
159
167
  expiry: StepExpiry,
160
168
  command: string,
161
169
  ): Finding => {
170
+ const ms = deadline.ms;
162
171
  const runs = expiry.inFlight.flatMap((run) => stalledRun(run) ?? []);
163
172
  const stuck = runs.flatMap((run) => run.stuck);
164
173
  const rerun = `${command} --only ${step} --json`;
@@ -169,6 +178,7 @@ export const stepTimeoutFinding = (
169
178
  const meta: StepTimeoutMeta = {
170
179
  step,
171
180
  deadlineMs: ms,
181
+ ...(deadline.pace === undefined ? {} : { pace: deadline.pace }),
172
182
  killed: expiry.killed.map((one) => ({ pid: one.pid, command: one.command })),
173
183
  inFlight: runs,
174
184
  finished: expiry.finished ?? { runs: 0, slowestMs: 0 },
@@ -176,7 +186,7 @@ export const stepTimeoutFinding = (
176
186
  const wider = `${command} --only ${step} --workers <n> --json`;
177
187
  return {
178
188
  code: 'X_VERIFY_STEP_TIMEOUT',
179
- cause: `step "${step}" did not finish within its ${String(ms)} ms deadline, so it was stopped and ${String(expiry.killed.length)} process(es) it had started were killed${inFlightClause(runs, expiry)}`,
189
+ cause: `step "${step}" did not finish within its ${String(ms)} ms deadline${pacedClause(deadline)}, so it was stopped and ${String(expiry.killed.length)} process(es) it had started were killed${inFlightClause(runs, expiry)}`,
180
190
  fix: outgrewDeadline(runs, expiry)
181
191
  ? `${wider} # it outgrew its deadline at this width, so run it wider; or raise the deadline in x.verify.json — add "${STEP_TIMEOUT_FIELD}": { "${step}": <milliseconds> } above the ${String(ms)} it was given`
182
192
  : stuck.length > 0
@@ -9,6 +9,7 @@ import { execOutput } from './exec';
9
9
  import type { Finding, ShardFacts, StepResult } from './output';
10
10
  import type { TestCounts } from './test-counts';
11
11
  import type { StepTimeouts } from './verify-deadline';
12
+ import type { MachinePace } from './verify-pace';
12
13
 
13
14
  /**
14
15
  * Every step of the gate, in cost order — cheapest and most informative first, and never a check
@@ -114,12 +115,25 @@ export interface VerifyContext {
114
115
  * with `X_VERIFY_STEP_TIMEOUT` and every process it started is killed.
115
116
  */
116
117
  readonly stepTimeoutMs?: StepTimeouts;
118
+ /**
119
+ * How much slower this machine runs than the 4-core runner the declared deadlines were measured
120
+ * on (`verify-pace.ts`): each step class's deadline, Bun's per-test default and a test's own
121
+ * declared timeout (`ULTIMATE_TEST_PACE`) are multiplied by it. Measured by the entry point —
122
+ * `x verify`, `x build`, the MCP host, the framework root's `scripts/verify.ts` — once, before
123
+ * any step. Absent is the numbers as declared.
124
+ */
125
+ readonly pace?: MachinePace;
117
126
  /**
118
127
  * Called as each step finishes, in the order they finish. `--json` prints nothing until the run
119
128
  * ends, so a job cancelled mid-run left no trace of where it hung (#589): the command streams
120
129
  * one line per step to stderr through this, and stdout stays the single document.
121
130
  */
122
131
  readonly onStep?: (step: StepResult) => void;
132
+ /**
133
+ * Called as each step starts. The heartbeat (`verify-progress.ts`) names the steps between this
134
+ * and `onStep`: one step can run longer than a harness's silence budget on its own.
135
+ */
136
+ readonly onStepStart?: (name: VerifyStepName) => void;
123
137
  /**
124
138
  * The gate as it was invoked HERE, for a `fix:` that has to run where it was raised: `x verify`
125
139
  * when absent, `bun run verify` from the framework root's own entry — where `x verify` answers
@@ -14,6 +14,8 @@ import { tidyBeforeTestRun } from './test-housekeeping';
14
14
  import type { TestFile } from './test-select';
15
15
  import { failureOf, TEST_TIMINGS_FILE, testArgs } from './test-shards';
16
16
  import { BATCH_FILES_PER_WORKER, SHARED_BATCH_FILES_PER_WORKER } from './test-workers';
17
+ import type { MachinePace } from './verify-pace';
18
+ import { defaultTimeoutArgs, paceEnv } from './verify-pace';
17
19
  import type { StepOutcome } from './verify-step';
18
20
  // Type-only, so nothing here evaluates verify-tests.ts and the two files cannot form a cycle.
19
21
  import type { TestType } from './verify-tests';
@@ -34,6 +36,8 @@ export interface ParallelRunOptions {
34
36
  readonly widthReason?: string;
35
37
  /** The machine-pool lease (`test-slots.ts`), for a default-width run. */
36
38
  readonly lease?: (want: number) => Promise<{ readonly count: number; release(): void }>;
39
+ /** The machine's pace (`verify-pace.ts`): each test's default and declared deadline stretch by it. */
40
+ readonly pace?: MachinePace;
37
41
  }
38
42
 
39
43
  /**
@@ -56,6 +60,7 @@ export async function runParallel(options: ParallelRunOptions): Promise<StepOutc
56
60
  const envOverrides = {
57
61
  ...testEnvOverrides(options.root, options.env ?? Bun.env),
58
62
  ...(options.isolate === true ? { [ISOLATED_ENV]: '1' } : {}),
63
+ ...paceEnv(options.pace),
59
64
  };
60
65
  // Batched exactly as `x test` batches the same selection at the same width, so the gate's
61
66
  // `fix:` line — `x test <type> --workers N` — reruns the very processes that failed.
@@ -77,6 +82,7 @@ export async function runParallel(options: ParallelRunOptions): Promise<StepOutc
77
82
  workers: width,
78
83
  isolate: options.isolate === true,
79
84
  timings: join(options.root, TEST_TIMINGS_FILE),
85
+ passthrough: defaultTimeoutArgs(options.pace),
80
86
  }),
81
87
  options: {
82
88
  cwd: options.root,
@@ -27,6 +27,8 @@ import { availableCpus, SERIAL_TYPES, totalMemory, workerPlan } from './test-wor
27
27
  import { runCovered } from './verify-coverage-run';
28
28
  import { withE2eApp } from './verify-e2e';
29
29
  import { readVerifyFloor } from './verify-floor';
30
+ import type { MachinePace } from './verify-pace';
31
+ import { DECLARED_PACE, defaultTimeoutArgs, pacedMs, paceEnv } from './verify-pace';
30
32
  import { shardFiles } from './verify-shard';
31
33
  import type { StepOutcome, VerifyContext, VerifyStep } from './verify-step';
32
34
  import { fromExec, fromFindings } from './verify-step';
@@ -153,18 +155,24 @@ export const typeFiltersOf = (type: Exclude<TestType, 'unit'>): readonly string[
153
155
  */
154
156
  export const E2E_TEST_TIMEOUT_MS = 60_000;
155
157
 
156
- /** Unit is everything the typed suites do not claim, so no test can fall between two steps. */
157
- export const testStepCommand = (type: TestType): readonly string[] =>
158
+ /**
159
+ * Unit is everything the typed suites do not claim, so no test can fall between two steps. Each
160
+ * per-test default is read at the machine's `pace` (`verify-pace.ts`).
161
+ */
162
+ export const testStepCommand = (type: TestType, pace?: MachinePace): readonly string[] =>
158
163
  type === 'unit'
159
164
  ? [
160
165
  'bun',
161
166
  'test',
167
+ ...defaultTimeoutArgs(pace),
162
168
  ...ignoreFlags([...NEVER_A_TEST, '**/e2e/**', `**/*.${TYPED_SUFFIXES}.test.*`]),
163
169
  ]
164
170
  : [
165
171
  'bun',
166
172
  'test',
167
- ...(type === 'e2e' ? [`--timeout=${String(E2E_TEST_TIMEOUT_MS)}`] : []),
173
+ ...(type === 'e2e'
174
+ ? [`--timeout=${String(pacedMs(E2E_TEST_TIMEOUT_MS, pace ?? DECLARED_PACE))}`]
175
+ : defaultTimeoutArgs(pace)),
168
176
  ...ignoreFlags([...NEVER_A_TEST, ...disownedBy(type)]),
169
177
  ...typeFiltersOf(type),
170
178
  ];
@@ -193,8 +201,8 @@ const filesFor = async (root: string, type: TestType): Promise<readonly TestFile
193
201
  export const resetTestDiscovery = (): void => discovered.clear();
194
202
 
195
203
  const runSerial = async (ctx: VerifyContext, type: TestType): Promise<StepOutcome> => {
196
- const command = testStepCommand(type);
197
- const envOverrides = testEnvOverrides(ctx.root, ctx.env ?? Bun.env);
204
+ const command = testStepCommand(type, ctx.pace);
205
+ const envOverrides = { ...testEnvOverrides(ctx.root, ctx.env ?? Bun.env), ...paceEnv(ctx.pace) };
198
206
  const exec = (e2e: { command: readonly string[]; env: typeof envOverrides }) =>
199
207
  ctx.runner(e2e.command, {
200
208
  cwd: ctx.root,
@@ -259,6 +267,7 @@ const runType = async (ctx: VerifyContext, type: TestType): Promise<StepOutcome>
259
267
  widthReason,
260
268
  shard,
261
269
  lease,
270
+ pace: ctx.pace,
262
271
  });
263
272
  }
264
273
  const outcome = await runParallel({
@@ -271,6 +280,7 @@ const runType = async (ctx: VerifyContext, type: TestType): Promise<StepOutcome>
271
280
  widthReason,
272
281
  ...(lease === undefined ? {} : { lease }),
273
282
  ...(ctx.env === undefined ? {} : { env: ctx.env }),
283
+ ...(ctx.pace === undefined ? {} : { pace: ctx.pace }),
274
284
  });
275
285
  return shard === undefined ? outcome : { ...outcome, shard };
276
286
  };
@@ -294,6 +304,7 @@ const coveredUnit = async (
294
304
  readonly widthReason: string;
295
305
  readonly shard: ReturnType<typeof shardFiles> | undefined;
296
306
  readonly lease: ReturnType<typeof machineLease> | undefined;
307
+ readonly pace: MachinePace | undefined;
297
308
  },
298
309
  ): Promise<StepOutcome> => {
299
310
  const { coverage, ...outcome } = await runCovered({
@@ -305,6 +316,7 @@ const coveredUnit = async (
305
316
  widthReason: run.widthReason,
306
317
  ...(run.lease === undefined ? {} : { lease: run.lease }),
307
318
  ...(ctx.env === undefined ? {} : { env: ctx.env }),
319
+ ...(run.pace === undefined ? {} : { pace: run.pace }),
308
320
  });
309
321
  if (!outcome.ok) return run.shard === undefined ? outcome : { ...outcome, shard: run.shard };
310
322
  if (run.shard !== undefined) {