@ultimat3/cli 1.0.0

package/src/test-shards.ts

@@ -0,0 +1,188 @@
+// Spending the selected test files: the LPT plan that balances them across worker processes, the
+// argv each child gets, and the command that reproduces one shard exactly. Split out of
+// cmd-test.ts because a printed reproduction is only true if it carries every input to the split —
+// that rule is this file's, and argv parsing is that one's.
+
+import { docsFor } from './errors';
+import type { Runner } from './exec';
+import { execOutput } from './exec';
+import { msg } from './messages';
+import type { CommandResult, Finding, JsonValue, StepResult } from './output';
+import type { TestFile } from './test-select';
+import { bySizeThenPath } from './test-select';
+import type { TestType } from './verify-tests';
+
+export interface Shard {
+  readonly index: number;
+  readonly files: readonly string[];
+  readonly bytes: number;
+}
+
+/**
+ * Largest-first greedy bin packing (LPT). Deterministic — the total order is (size desc, path asc),
+ * so the filesystem's scan order never reaches the assignment and a CI failure on worker 3 is the
+ * same worker 3 locally. Balanced — every file lands in the currently emptiest bin, which bounds a
+ * bin at average + largest file; round-robin or hashing can pile every slow file onto one worker.
+ * The inner scan is O(files × workers), and workers is a core count, so a heap would only add
+ * allocation.
+ */
+export function planShards(files: readonly TestFile[], workers: number): readonly Shard[] {
+  const count = Math.max(1, Math.min(Math.trunc(workers), files.length));
+  const loads = new Array<number>(count).fill(0);
+  const buckets: string[][] = Array.from({ length: count }, () => []);
+  for (const file of [...files].sort(bySizeThenPath)) {
+    let target = 0;
+    for (let i = 1; i < count; i += 1) if ((loads[i] ?? 0) < (loads[target] ?? 0)) target = i;
+    buckets[target]?.push(file.path);
+    loads[target] = (loads[target] ?? 0) + file.bytes;
+  }
+  return buckets.map((paths, index) => ({
+    index,
+    files: [...paths].sort(),
+    bytes: loads[index] ?? 0,
+  }));
+}
+
+/** Explicit file list, so the child never re-globs and can never pick up another shard's files. */
+export const shardArgs = (shard: Shard): readonly string[] => ['bun', 'test', ...shard.files];
+
+const SHELL_SAFE = /^[\w@%+=:,./-]+$/;
+
+/**
+ * POSIX single-quoting for the values a caller supplies. A `--filter` holding a space, a `$` or a
+ * `;` pastes back as two arguments or as a second command, so an unquoted line would run something
+ * other than the run it claims to reproduce. `'\''` is the only escape a single-quoted string has.
+ */
+export const quoteArg = (value: string): string =>
+  SHELL_SAFE.test(value) ? value : `'${value.split("'").join("'\\''")}'`;
+
+export interface ReproduceOptions {
+  /** The *effective* worker count: `planShards` clamps to the file count, and the split follows. */
+  readonly workers: number;
+  readonly filter?: string;
+  readonly type?: TestType;
+  /** Files `--sample` kept, so the rerun samples the same corpus instead of the whole type. */
+  readonly sample?: number;
+}
+
+/**
+ * Every input to the split, printed back. The type and `--filter` decide which files exist to
+ * shard, `--sample` decides how many of them survive, `--workers` decides the bins — drop any one
+ * and the command still runs, over a different file set, which reproduces nothing.
+ */
+export function reproduceFor(shard: Shard, options: ReproduceOptions): string {
+  return [
+    'x test',
+    ...(options.type === undefined ? [] : [quoteArg(options.type)]),
+    ...(options.filter === undefined ? [] : ['--filter', quoteArg(options.filter)]),
+    ...(options.sample === undefined ? [] : ['--sample', String(options.sample)]),
+    '--workers',
+    String(options.workers),
+    '--worker',
+    String(shard.index),
+  ].join(' ');
+}
+
+export interface RunShardsOptions {
+  readonly root: string;
+  readonly runner: Runner;
+  readonly files: readonly TestFile[];
+  readonly workers: number;
+  /** Run exactly one shard of the same split, not a one-worker run of everything. */
+  readonly only?: number;
+  readonly filter?: string;
+  readonly type?: TestType;
+  /**
+   * Set when `--sample` narrowed `files`: `kept` is what survived, `total` what discovery found.
+   * `kept` is carried rather than counted from the shards that ran, because `--worker N` runs one
+   * shard of the sample and would otherwise report that shard's size as the corpus.
+   */
+  readonly sample?: { readonly kept: number; readonly total: number };
+}
+
+/** The reproduction's inputs, resolved once: `workers` is the split's real width, not the ask. */
+const planOf = (options: RunShardsOptions, workers: number): ReproduceOptions => ({
+  workers,
+  ...(options.filter === undefined ? {} : { filter: options.filter }),
+  ...(options.type === undefined ? {} : { type: options.type }),
+  ...(options.sample === undefined ? {} : { sample: options.sample.kept }),
+});
+
+const failureOf = (shard: Shard, code: number, plan: ReproduceOptions): Finding => ({
+  code: 'X_TEST_SHARD_FAILED',
+  cause: `shard ${shard.index} of ${plan.workers} exited ${code} (${shard.files.length} file(s))`,
+  fix: reproduceFor(shard, plan),
+  docs: docsFor('X_TEST_SHARD_FAILED'),
+});
+
+export async function runShards(options: RunShardsOptions): Promise<CommandResult> {
+  const shards = planShards(options.files, options.workers);
+  const only = options.only;
+  const chosen = only === undefined ? shards : shards.filter((shard) => shard.index === only);
+  const started = performance.now();
+  // All shards at once: wall-clock is the whole point, and each one owns a separate database.
+  const runs = await Promise.all(
+    chosen.map(async (shard) => ({
+      shard,
+      result: await options.runner(shardArgs(shard), {
+        cwd: options.root,
+        env: { ULTIMATE_TEST_WORKER: String(shard.index) },
+      }),
+    })),
+  );
+  const durationMs = Math.round(performance.now() - started);
+  const plan = planOf(options, shards.length);
+  const steps: readonly StepResult[] = runs.map(({ shard, result }) => ({
+    name: `shard ${shard.index} · ${shard.files.length} files`,
+    ok: result.ok,
+    durationMs: result.durationMs,
+    findings: result.ok ? [] : [failureOf(shard, result.code, plan)],
+    output: execOutput(result),
+  }));
+  const failed = runs.filter((run) => !run.result.ok).map((run) => run.shard.index);
+  const fileCount = chosen.reduce((total, shard) => total + shard.files.length, 0);
+  const type = options.type;
+  const typeParam = type === undefined ? {} : { type };
+  const sample = options.sample;
+  const data: JsonValue = {
+    ...typeParam,
+    workers: shards.length,
+    files: fileCount,
+    durationMs,
+    ...(options.filter === undefined ? {} : { filter: options.filter }),
+    ...(sample === undefined ? {} : { sample: { kept: sample.kept, total: sample.total } }),
+    shards: runs.map(({ shard, result }) => ({
+      index: shard.index,
+      files: shard.files.length,
+      bytes: shard.bytes,
+      ok: result.ok,
+      exitCode: result.code,
+      durationMs: result.durationMs,
+      reproduce: reproduceFor(shard, plan),
+    })),
+    failed,
+  };
+  return {
+    ok: failed.length === 0,
+    command: 'test',
+    summary:
+      failed.length === 0
+        ? msg(type === undefined ? 'cli.test.pass' : 'cli.test.type.pass', {
+            ...typeParam,
+            files: fileCount,
+            workers: chosen.length,
+            ms: durationMs,
+          })
+        : msg(type === undefined ? 'cli.test.fail' : 'cli.test.type.fail', {
+            ...typeParam,
+            failed: failed.length,
+            workers: chosen.length,
+          }),
+    steps,
+    ...(sample === undefined
+      ? {}
+      : { lines: [msg('cli.test.sampled', { ...sample, type: type ?? 'all' })] }),
+    data,
+    exitCode: failed.length === 0 ? 0 : 1,
+  };
+}

package/src/thrown-by.ts

@@ -0,0 +1,24 @@
+// One `thrownBy` for every test in this package. What a caller acts on is the thrown *shape* —
+// the code, the cause and the fix — never the class, so a test that asserts on the class would
+// pass while the fix line rots. Shared rather than copied: a second copy asserts less, silently.
+
+import { expect } from 'bun:test';
+
+/** The three fields every `UltimateError` carries, as a test reads them off the thrown value. */
+export interface ThrownShape {
+  readonly code?: string;
+  readonly cause?: string;
+  readonly fix?: string;
+}
+
+export function thrownBy(call: () => unknown): ThrownShape {
+  try {
+    call();
+  } catch (error) {
+    return error as ThrownShape;
+  }
+  // Not `throw new Error(...)`: a bare throw carries no code and no fix, and this is the one
+  // failure a test author most needs named. `expect.unreachable` fails through the runner, which
+  // prints the assertion the caller wrote instead of a stack from inside this helper.
+  return expect.unreachable('expected a throw');
+}

package/src/ts-scan.ts

@@ -0,0 +1,217 @@
+// Reading two things out of TypeScript source without a parser: the strings a `fix:` can evaluate
+// to, and the `X_*` codes a package declares. Deliberately not `tsc` — a regex over a masked file
+// is the whole job. Masking is the load-bearing part: the contract's own 3-line rendering appears
+// verbatim in doc blocks and template literals, and a scanner that reads it as code invents work.
+
+export interface SourceSite {
+  /** Repo-relative file the site was read from. */
+  readonly at: string;
+  readonly line: number;
+}
+
+export interface FixSite extends SourceSite {
+  /** The literal exactly as written, `${…}` included. */
+  readonly fix: string;
+}
+
+export interface CodeSite extends SourceSite {
+  readonly code: string;
+}
+
+const QUOTES = new Set(["'", '"', '`']);
+const OPENERS = new Set(['(', '[', '{']);
+const CLOSERS = new Set([')', ']', '}']);
+const WORD = /[\w$]/;
+
+/** After one of these words a `/` opens a regex; after any other identifier it divides. */
+const REGEX_AFTER_WORDS = new Set(
+  'await case delete do else in instanceof new of return throw typeof void yield'.split(' '),
+);
+
+/** Index just past the closing quote of the literal opening at `from`, or the end of the text. */
+function endOfLiteral(text: string, from: number): number {
+  const quote = text[from] as string;
+  for (let i = from + 1; i < text.length; i += 1) {
+    if (text[i] === '\\') i += 1;
+    else if (text[i] === quote) return i + 1;
+  }
+  return text.length;
+}
+
+/**
+ * Whether the `/` at `at` opens a regex rather than divides — the call no scanner without a parser
+ * avoids. A regex cannot follow what ends an expression: an identifier that is not one of the words
+ * above, a number, `)`, `]`, a string's closing quote. Every other position is an operator's and
+ * opens one; `</` and `/>` are JSX delimiters. Read from the masked prefix, so a comment is space.
+ */
+function opensRegex(out: readonly string[], at: number): boolean {
+  if (out[at + 1] === '>') return false;
+  let i = at - 1;
+  while (i >= 0 && /\s/.test(out[i] as string)) i -= 1;
+  if (i < 0) return true;
+  const ch = out[i] as string;
+  if (ch === '<' || ch === ')' || ch === ']' || QUOTES.has(ch)) return false;
+  if (!WORD.test(ch)) return true;
+  let start = i;
+  while (start >= 0 && WORD.test(out[start] as string)) start -= 1;
+  return REGEX_AFTER_WORDS.has(out.slice(start + 1, i + 1).join(''));
+}
+
+/**
+ * Index just past the closing `/` of the regex opening at `from`, or `from + 1` when it does not
+ * close on its own line — a literal may not span one, so an unterminated candidate was a division
+ * or a JSX delimiter after all. A `/` inside a `[…]` class does not close the literal.
+ */
+function endOfRegex(text: string, from: number): number {
+  let inClass = false;
+  let escaped = false;
+  for (let i = from + 1; i < text.length; i += 1) {
+    const ch = text[i];
+    if (ch === '\n') break;
+    if (escaped) escaped = false;
+    else if (ch === '\\') escaped = true;
+    else if (inClass) inClass = ch !== ']';
+    else if (ch === '[') inClass = true;
+    else if (ch === '/') return i + 1;
+  }
+  return from + 1;
+}
+
+/**
+ * Comments — and optionally string contents — replaced by spaces, newlines kept so line numbers
+ * survive and quote delimiters kept so the caller can still find where a literal starts and ends.
+ * A regex body is masked the same way: `/(['"`])/` holds three quotes that delimit nothing, and
+ * reading one as an opening quote desyncs every literal after it.
+ */
+function blankRegions(text: string, strings: boolean): string {
+  const out = [...text];
+  const blank = (from: number, to: number): void => {
+    for (let n = from; n < to; n += 1) if (out[n] !== '\n') out[n] = ' ';
+  };
+  let i = 0;
+  while (i < text.length) {
+    const ch = text[i] as string;
+    if (ch === '/' && (text[i + 1] === '/' || text[i + 1] === '*')) {
+      const line = text[i + 1] === '/';
+      const end = line ? text.indexOf('\n', i) : text.indexOf('*/', i + 2);
+      const stop = end === -1 ? text.length : line ? end : end + 2;
+      blank(i, stop);
+      i = stop;
+      continue;
+    }
+    // Not code: a regex body or a literal. `end === i + 1` blanks nothing and steps one char.
+    const end =
+      ch === '/' && opensRegex(out, i)
+        ? endOfRegex(text, i)
+        : QUOTES.has(ch)
+          ? endOfLiteral(text, i)
+          : i + 1;
+    if (strings) blank(i + 1, end - 1);
+    i = end;
+  }
+  return out.join('');
+}
+
+/** Comments gone, string literals intact — what a scan for declared codes reads. */
+export const stripComments = (text: string): string => blankRegions(text, false);
+
+/** Comments and string contents gone, delimiters kept — what a scan for code structure reads. */
+export const maskLiterals = (text: string): string => blankRegions(text, true);
+
+const lineOf = (text: string, index: number): number => {
+  let line = 1;
+  for (let i = 0; i < index; i += 1) if (text[i] === '\n') line += 1;
+  return line;
+};
+
+/**
+ * Every string literal in the value expression starting at `from`, at the expression's own bracket
+ * depth. Spanning the expression is what makes `cond ? 'a' : 'b'` and `table[k] ?? 'c'` checkable
+ * instead of silently skipped; the depth rule is what keeps `command.join(' ')`'s separator and
+ * `table['key']`'s key out — an argument is not a fix.
+ */
+function valueLiterals(masked: string, source: string, from: number): readonly FixSite[] {
+  const found: { value: string; index: number }[] = [];
+  let depth = 0;
+  for (let i = from; i < masked.length; i += 1) {
+    const ch = masked[i] as string;
+    if (QUOTES.has(ch)) {
+      const end = endOfLiteral(masked, i);
+      if (depth === 0) found.push({ value: source.slice(i + 1, end - 1), index: i });
+      i = end - 1;
+    } else if (OPENERS.has(ch)) depth += 1;
+    else if (CLOSERS.has(ch)) {
+      if (depth === 0) break;
+      depth -= 1;
+    } else if (depth === 0 && (ch === ',' || ch === ';')) break;
+  }
+  return found.map((literal) => ({
+    at: '',
+    line: lineOf(masked, literal.index),
+    fix: literal.value,
+  }));
+}
+
+/** The lookbehind rejects member access: `cond ? e.fix : ''` is a ternary, not a declaration. */
+const FIX_KEY = /(?<![.\w$])fix\s*:\s*/g;
+
+/**
+ * Every string a `fix:` can evaluate to. Searched over the masked source, so a `fix:` written
+ * inside a doc comment or interpolated into a message is not mistaken for a declaration. A `fix`
+ * computed at runtime — a bare identifier, a parameter, a table lookup with no literal fallback —
+ * has nothing to read and is beyond a static scan; the gate says so rather than guessing.
+ */
+export function scanFixes(source: string, at: string): readonly FixSite[] {
+  const masked = maskLiterals(source);
+  const sites: FixSite[] = [];
+  for (const key of masked.matchAll(FIX_KEY)) {
+    const start = key.index + key[0].length;
+    for (const literal of valueLiterals(masked, source, start)) sites.push({ ...literal, at });
+  }
+  return sites;
+}
+
+/** `packages/<pkg>/src/errors.ts` and core's `error-codes.ts`: one file per package, by rule. */
+export const isCodeRegistry = (path: string): boolean =>
+  /(?:^|\/)(?:errors|error-codes)\.ts$/.test(path);
+
+const CODE_AT_KEY = /\bcode\s*[:=]\s*(['"`])(X_[A-Z0-9_]+)\1/g;
+const CODE_LITERAL = /(['"`])(X_[A-Z0-9_]+)\1/g;
+const CODE_KEY = /^[\t ]*(X_[A-Z0-9_]+)\s*:/gm;
+
+/**
+ * Codes this file declares: every `code:` / `code =` throw site, plus — in a package's own code
+ * registry — every entry of its code list or title table, whichever shape it uses. A registry is
+ * the only place a bare `X_*` literal is a declaration; anywhere else it is a reference (an env
+ * var named `X_BUILD_ID`, an HTTP status map keyed by code) and collecting it would invent a code.
+ */
+export function scanCodes(source: string, at: string): readonly CodeSite[] {
+  const text = stripComments(source);
+  const sites = new Map<string, CodeSite>();
+  const add = (code: string, index: number): void => {
+    if (!sites.has(code)) sites.set(code, { at, line: lineOf(text, index), code });
+  };
+  for (const match of text.matchAll(CODE_AT_KEY)) add(match[2] as string, match.index);
+  if (isCodeRegistry(at)) {
+    for (const match of text.matchAll(CODE_LITERAL)) add(match[2] as string, match.index);
+    for (const match of text.matchAll(CODE_KEY)) add(match[1] as string, match.index);
+  }
+  return [...sites.values()];
+}
+
+const BORROWED_LIST = /BORROWED_ERROR_CODES[^=]*=[^[]*\[([^\]]*)\]/g;
+
+/**
+ * The codes a registry names and says are not its own. Every borrower declares them in one place
+ * and one shape — `export const CLI_BORROWED_ERROR_CODES = ['X_NOT_IMPLEMENTED'] as const` — so
+ * "who owns this code?" is answerable from source rather than guessed at. Without it the answer
+ * for a code eleven packages throw and one titles is whichever file sorts first, which is how
+ * `X_NOT_IMPLEMENTED` came to be attributed to `storage` instead of `core`.
+ */
+export function scanBorrowedCodes(source: string): ReadonlySet<string> {
+  const borrowed = new Set<string>();
+  for (const list of stripComments(source).matchAll(BORROWED_LIST)) {
+    for (const code of (list[1] ?? '').matchAll(CODE_LITERAL)) borrowed.add(code[2] as string);
+  }
+  return borrowed;
+}

package/src/verify-step.ts

@@ -0,0 +1,83 @@
+// The shape of one gate step: its name, what it checks, whether it applies here, and how a host
+// repo feeds it findings it could not produce on its own. Split from the step list so a step
+// implementation can live beside the code it checks without importing the list.
+
+import type { ExecResult, Runner } from './exec';
+import { execOutput } from './exec';
+import type { Finding } from './output';
+
+/**
+ * Every step of the gate, in cost order — cheapest and most informative first, and never a check
+ * whose result would be meaningless because an earlier one failed. This list is the definition of
+ * shippable: the framework repo and a generated app run exactly it, whole, or not at all.
+ */
+export const VERIFY_STEP_NAMES = [
+  'typecheck',
+  'lint',
+  'boundaries',
+  'filesize',
+  'package-shape',
+  'errors',
+  'unit',
+  'contract',
+  'live',
+  'job',
+  'e2e',
+  'eval',
+  'drift',
+  'contract-diff',
+  'budgets',
+  'manifest',
+  'roadmap',
+] as const;
+
+export type VerifyStepName = (typeof VERIFY_STEP_NAMES)[number];
+
+/**
+ * A rule the host repo enforces inside an existing step — the framework monorepo's package tier
+ * table under `boundaries`, its generated manifest under `manifest`. A host adds findings to a
+ * step; it can never add, remove, reorder or skip one.
+ */
+export type HostCheck = (root: string) => Promise<readonly Finding[]>;
+
+export interface VerifyContext {
+  readonly root: string;
+  readonly runner: Runner;
+  readonly hostChecks?: Partial<Record<VerifyStepName, HostCheck>>;
+}
+
+export interface StepOutcome {
+  readonly ok: boolean;
+  readonly findings: readonly Finding[];
+  readonly output?: string;
+}
+
+export interface VerifyStep {
+  readonly name: VerifyStepName;
+  readonly summary: string;
+  /** Returns false to record the step as skipped rather than passed. */
+  applies?(ctx: VerifyContext): Promise<boolean>;
+  run(ctx: VerifyContext): Promise<StepOutcome>;
+}
+
+export const passed: StepOutcome = { ok: true, findings: [] };
+
+export function fromExec(result: ExecResult, finding: Omit<Finding, 'docs'>): StepOutcome {
+  if (result.ok) return { ok: true, findings: [], output: execOutput(result) };
+  return {
+    ok: false,
+    findings: [{ ...finding, docs: `https://ultimate.dev/errors/${finding.code}` }],
+    output: execOutput(result),
+  };
+}
+
+export const fromFindings = (findings: readonly Finding[]): StepOutcome => ({
+  ok: findings.length === 0,
+  findings,
+});
+
+/** What the host repo contributes to this step, or nothing. */
+export const hostFindings = async (
+  ctx: VerifyContext,
+  step: VerifyStepName,
+): Promise<readonly Finding[]> => (await ctx.hostChecks?.[step]?.(ctx.root)) ?? [];

package/src/verify-tests.ts

@@ -0,0 +1,166 @@
+// One `bun test` invocation per test type, so every type reports on its own line of the gate. A
+// test's type is its filename suffix — `*.contract.test.ts`, `*.live.test.ts`, `*.job.test.ts`,
+// `*.e2e.test.ts` (or any file under an `e2e/` directory), `*.eval.test.ts`. Everything else is a
+// unit test, which is why the unit step is the only one that selects by exclusion.
+//
+// `eval` carries one rule beyond its suite — every prompt must have an eval — so it is the only
+// step here that can fail with no test file of its own.
+
+// Bun ships no `Bun.*` equivalent for either: `existsSync` answers whether this root is an app,
+// and `join` builds the host-separator path to its config file.
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { checkEvalBaselines, checkEvalCoverage, checkEvalRecording } from './app-evals';
+import { APP_CONFIG_FILE } from './app-root';
+import type { StepOutcome, VerifyContext, VerifyStep } from './verify-step';
+import { fromExec, fromFindings } from './verify-step';
+
+export const TEST_TYPES = ['unit', 'contract', 'live', 'job', 'e2e', 'eval'] as const;
+
+export type TestType = (typeof TEST_TYPES)[number];
+
+interface TestSuite {
+  readonly summary: string;
+  /** Substring `bun test` matches against each file path. */
+  readonly filter: string;
+  /** Globs that decide whether this type exists here at all. */
+  readonly globs: readonly string[];
+}
+
+const TYPED_SUFFIXES = '{contract,live,job,e2e,eval}';
+
+const SUITES: Readonly<Record<Exclude<TestType, 'unit'>, TestSuite>> = {
+  contract: {
+    summary: 'action/query schemas, policy denials, emitted OpenAPI and MCP shapes',
+    filter: '.contract.test.',
+    globs: ['**/*.contract.test.{ts,tsx}'],
+  },
+  live: {
+    summary: 'live-query snapshots, incremental patches, reconnect deltas',
+    filter: '.live.test.',
+    globs: ['**/*.live.test.{ts,tsx}'],
+  },
+  job: {
+    summary: 'step replay, idempotency dedupe, retry/backoff, outbox atomicity',
+    filter: '.job.test.',
+    globs: ['**/*.job.test.{ts,tsx}'],
+  },
+  e2e: {
+    summary: 'the built output, incl. offline and SW update',
+    filter: 'e2e',
+    globs: ['**/*.e2e.test.{ts,tsx}', '**/e2e/**/*.test.{ts,tsx}'],
+  },
+  eval: {
+    summary: 'LLM output scored against thresholds',
+    filter: '.eval.test.',
+    globs: ['**/*.eval.test.{ts,tsx}'],
+  },
+};
+
+/**
+ * Build output, and nested projects that carry their own `x verify`. `examples/**` is the second
+ * kind: the reference app is gated by its own run of this same step list, so collecting it here
+ * would report one app failure on two different gates. The patterns are relative to the run's
+ * root, so this excludes nothing when the app itself is the root.
+ */
+const NEVER_A_TEST = ['**/dist/**', '**/build/**', '**/examples/**'];
+
+const ignoreFlags = (patterns: readonly string[]): readonly string[] =>
+  patterns.map((pattern) => `--path-ignore-patterns=${pattern}`);
+
+/**
+ * The substring that decides a file's type — the same one the step's `bun test` runs with, so
+ * `x test <type>` and the gate's `<type>` step can never disagree about what a contract test is.
+ */
+export const typeFilterOf = (type: Exclude<TestType, 'unit'>): string => SUITES[type].filter;
+
+/** Unit is everything the typed suites do not claim, so no test can fall between two steps. */
+export const testStepCommand = (type: TestType): readonly string[] =>
+  type === 'unit'
+    ? [
+        'bun',
+        'test',
+        ...ignoreFlags([...NEVER_A_TEST, '**/e2e/**', `**/*.${TYPED_SUFFIXES}.test.*`]),
+      ]
+    : ['bun', 'test', ...ignoreFlags(NEVER_A_TEST), SUITES[type].filter];
+
+/**
+ * Whether a step applies has to be decided by the same rule that decides what it runs. When the
+ * two drifted, a suite that lived only under an ignored path made its step apply and then fail
+ * with "no test files matched" — a red gate reporting a suite that, by its own rule, is not here.
+ */
+const NEVER_A_TEST_GLOBS = NEVER_A_TEST.map((pattern) => new Bun.Glob(pattern));
+
+const ignoredPath = (path: string): boolean =>
+  path.includes('node_modules') || NEVER_A_TEST_GLOBS.some((glob) => glob.match(path));
+
+const exists = async (root: string, globs: readonly string[]): Promise<boolean> => {
+  for (const pattern of globs) {
+    for await (const path of new Bun.Glob(pattern).scan({ cwd: root, absolute: false })) {
+      if (!ignoredPath(path)) return true;
+    }
+  }
+  return false;
+};
+
+const runType = async (ctx: VerifyContext, type: TestType): Promise<StepOutcome> => {
+  const command = testStepCommand(type);
+  const result = await ctx.runner(command, { cwd: ctx.root });
+  return fromExec(result, {
+    code: 'X_TEST_FAILED',
+    cause: `one or more ${type} tests failed`,
+    fix: command.join(' '),
+  });
+};
+
+const isApp = (root: string): boolean => existsSync(join(root, APP_CONFIG_FILE));
+
+/**
+ * The eval step carries rules beyond "the suite is green": every prompt must have an eval, every
+ * eval must have a baseline to gate against, and the run must not be recording. So it is the one
+ * test step that applies with no suite of its own — an app whose only prompt has no eval file
+ * would otherwise skip this step and report a green gate over untested code.
+ */
+const evalStep: VerifyStep = {
+  name: 'eval',
+  summary: SUITES.eval.summary,
+  applies: async (ctx) => isApp(ctx.root) || (await exists(ctx.root, SUITES.eval.globs)),
+  async run(ctx) {
+    // First, and instead of the suite: under recording every eval writes the numbers it just
+    // measured and passes, so running it here would rewrite the committed baselines during the
+    // gate — damage a red step does not undo.
+    const recording = checkEvalRecording();
+    if (recording.length > 0) return fromFindings(recording);
+    const declarations = isApp(ctx.root)
+      ? [...(await checkEvalCoverage(ctx.root)), ...(await checkEvalBaselines(ctx.root))]
+      : [];
+    if (!(await exists(ctx.root, SUITES.eval.globs))) return fromFindings(declarations);
+    const suite = await runType(ctx, 'eval');
+    return {
+      ok: suite.ok && declarations.length === 0,
+      findings: [...declarations, ...suite.findings],
+      ...(suite.output === undefined ? {} : { output: suite.output }),
+    };
+  },
+};
+
+const stepFor = (type: TestType): VerifyStep => {
+  if (type === 'unit') {
+    return {
+      name: 'unit',
+      summary: 'pure logic — no database, no network',
+      run: (ctx) => runType(ctx, 'unit'),
+    };
+  }
+  if (type === 'eval') return evalStep;
+  const suite = SUITES[type];
+  return {
+    name: type,
+    summary: suite.summary,
+    applies: (ctx) => exists(ctx.root, suite.globs),
+    run: (ctx) => runType(ctx, type),
+  };
+};
+
+/** In cost order: unit needs nothing, e2e needs a build. */
+export const TEST_STEPS: readonly VerifyStep[] = TEST_TYPES.map(stepFor);