@ultimat3/cli 19.2.0 → 19.3.2

package/src/test-shards.ts

@@ -40,6 +40,7 @@ import { execOutput } from './exec';
 import { msg } from './messages';
 import type { CommandResult, Finding, JsonValue, StepResult } from './output';
 import { quoteArg } from './shell-quote';
+import { testPasses } from './test-passes';
 import type { TestFile } from './test-select';
 import type { TestType } from './verify-tests';
 
@@ -65,15 +66,26 @@ export function testArgs(input: {
   readonly workers: number;
   /** 0-based, matching `--worker`. Absent runs the whole selection across `workers` processes. */
   readonly shard?: number;
+  /**
+   * Everything after a bare `--`, handed to `bun test` verbatim and BEFORE the file list, which is
+   * where bun reads its flags. `ParsedArgs.passthrough` had no reader anywhere until 2026-09, so
+   * `x test unit -- --coverage --bail` parsed both flags, carried them through the command and
+   * dropped them on the floor — a run that reported exactly what a coverage run reports, with no
+   * coverage measured. `CommandSpec.passthrough` is what keeps the other commands from doing the
+   * same in silence: they refuse the `--` instead.
+   */
+  readonly passthrough?: readonly string[];
 }): readonly string[] {
   const files = [...input.files].sort();
+  const extra = input.passthrough ?? [];
   return input.shard === undefined
-    ? ['bun', 'test', `--parallel=${String(input.workers)}`, ...files]
+    ? ['bun', 'test', `--parallel=${String(input.workers)}`, ...extra, ...files]
     : [
         'bun',
         'test',
         '--isolate',
         `--shard=${String(input.shard + 1)}/${String(input.workers)}`,
+        ...extra,
         ...files,
       ];
 }
@@ -102,6 +114,8 @@ export interface ReproduceOptions {
   readonly affected?: AffectedSelection;
   /** 0-based, when reproducing ONE shard. Absent reproduces the whole selection. */
   readonly shard?: number;
+  /** What the caller put after `--`. It reaches `bun test`, so a rerun without it runs differently. */
+  readonly passthrough?: readonly string[];
 }
 
 /**
@@ -124,6 +138,11 @@ export function reproduceFor(options: ReproduceOptions): string {
     '--workers',
     String(options.workers),
     ...(options.shard === undefined ? [] : ['--worker', String(options.shard)]),
+    // Last, and after a `--` of its own, because that is where the caller typed it and where the
+    // parser will find it again. Quoted for `shell-quote.ts`'s reason: a reproduce line is pasted.
+    ...(options.passthrough === undefined || options.passthrough.length === 0
+      ? []
+      : ['--', ...options.passthrough.map(quoteArg)]),
   ].join(' ');
 }
 
@@ -144,16 +163,28 @@ export interface RunShardsOptions {
   readonly sample?: { readonly kept: number; readonly total: number };
   /** Passed straight to `reproduceFor`: see `ReproduceOptions.affected`. */
   readonly affected?: AffectedSelection;
+  /** Everything after the caller's `--`, forwarded to every pass and printed in the reproduce. */
+  readonly passthrough?: readonly string[];
 }
 
-/** The reproduction's inputs, resolved once: `workers` is the run's real width, not the ask. */
-const planOf = (options: RunShardsOptions, workers: number): ReproduceOptions => ({
-  workers,
+/**
+ * The reproduction's inputs for ONE pass: `workers` is that pass's real width, not the ask, and
+ * `type` is the pass's own when the split gave it one — `x test live --workers 1` reruns exactly
+ * the files that failed, where the whole invocation's flags would rerun the corpus around them.
+ */
+const planOf = (
+  options: RunShardsOptions,
+  pass: { readonly workers: number; readonly type?: TestType },
+): ReproduceOptions => ({
+  workers: pass.workers,
   ...(options.filter === undefined ? {} : { filter: options.filter }),
-  ...(options.type === undefined ? {} : { type: options.type }),
+  ...(pass.type === undefined ? {} : { type: pass.type }),
   ...(options.sample === undefined ? {} : { sample: options.sample.kept }),
   ...(options.affected === undefined ? {} : { affected: options.affected }),
   ...(options.only === undefined ? {} : { shard: options.only }),
+  ...(options.passthrough === undefined || options.passthrough.length === 0
+    ? {}
+    : { passthrough: options.passthrough }),
 });
 
 /**
@@ -173,68 +204,111 @@ export const failureOf = (code: number, files: number, plan: ReproduceOptions):
 });
 
 /**
- * ONE `bun test`, not one per worker. Bun owns the pool and hands each free worker the next file,
- * so nothing here decides which file runs where — see this file's header for what that measured.
+ * ONE `bun test` PER PASS, and one pass unless the selection mixes serial files with the rest —
+ * `test-passes.ts` decides that, and this spends it. Bun owns the pool inside a pass and hands
+ * each free worker the next file, so nothing here decides which file runs where; see this file's
+ * header for what that measured.
+ *
+ * Sequential, never `Promise.all`: the whole point of a serial pass is that nothing runs beside
+ * it. And every pass runs even after one fails — the caller asked for a suite, and a report that
+ * stops at the first red step hides the rest of the answer.
  *
  * `ULTIMATE_TEST_WORKER` is still set for a `--worker` rerun and only then: that run is one
  * process, so naming its database is this file's to do. A `--parallel` run has N of them and Bun
  * numbers each with `BUN_TEST_WORKER_ID`, which `@ultimat3/testing`'s `workerId` already reads.
  */
 export async function runShards(options: RunShardsOptions): Promise<CommandResult> {
-  const files = options.files.map((file) => file.path);
-  const workers = Math.max(1, Math.min(Math.trunc(options.workers), files.length || 1));
   const only = options.only;
+  const passes = testPasses({
+    files: options.files,
+    workers: options.workers,
+    ...(options.type === undefined ? {} : { type: options.type }),
+    ...(only === undefined ? {} : { shard: only }),
+  });
   const started = performance.now();
-  const result = await options.runner(
-    testArgs({ files, workers, ...(only === undefined ? {} : { shard: only }) }),
-    {
-      cwd: options.root,
-      ...(only === undefined ? {} : { env: { ULTIMATE_TEST_WORKER: String(only) } }),
-    },
-  );
-  const durationMs = Math.round(performance.now() - started);
-  const plan = planOf({ ...options, workers }, workers);
-  const type = options.type;
-  const typeParam = type === undefined ? {} : { type };
-  const sample = options.sample;
-  const label = only === undefined ? `${workers} worker(s)` : `shard ${only} of ${workers}`;
-  const steps: readonly StepResult[] = [
-    {
-      name: `${label} · ${files.length} files`,
+  const steps: StepResult[] = [];
+  const spent: JsonValue[] = [];
+  let ok = true;
+  let exitCode = 0;
+  for (const pass of passes) {
+    const files = pass.files.map((file) => file.path);
+    const result = await options.runner(
+      testArgs({
+        files,
+        workers: pass.workers,
+        ...(only === undefined ? {} : { shard: only }),
+        ...(options.passthrough === undefined ? {} : { passthrough: options.passthrough }),
+      }),
+      {
+        cwd: options.root,
+        ...(only === undefined ? {} : { env: { ULTIMATE_TEST_WORKER: String(only) } }),
+      },
+    );
+    const plan = planOf(options, pass);
+    const label =
+      only === undefined ? `${pass.workers} worker(s)` : `shard ${only} of ${pass.workers}`;
+    steps.push({
+      name: `${pass.type === undefined ? label : `${pass.type} · ${label}`} · ${files.length} files`,
       ok: result.ok,
       durationMs: result.durationMs,
       // `output.ts` documents this field as absent for a NON-test step, so omitting it here made
       // `renderJson` describe the test step as one — recoverable only by parsing `name`.
-      workers,
+      workers: pass.workers,
       findings: result.ok ? [] : [failureOf(result.code, files.length, plan)],
       output: execOutput(result),
-    },
-  ];
+    });
+    spent.push({
+      ...(pass.type === undefined ? {} : { type: pass.type }),
+      files: files.length,
+      workers: pass.workers,
+      ok: result.ok,
+      exitCode: result.code,
+      reproduce: reproduceFor(plan),
+    });
+    ok = ok && result.ok;
+    if (exitCode === 0) exitCode = result.code;
+  }
+  const durationMs = Math.round(performance.now() - started);
+  const fileCount = options.files.length;
+  // The width the RUN reached, which is the widest pass: a mixed selection whose serial half ran
+  // one at a time did not become a one-worker run, and reporting it as one would misname the
+  // reproduce a reader is handed.
+  const workers = Math.max(1, ...passes.map((pass) => pass.workers));
+  const plan = planOf(options, {
+    workers,
+    ...(options.type === undefined ? {} : { type: options.type }),
+  });
+  const type = options.type;
+  const typeParam = type === undefined ? {} : { type };
+  const sample = options.sample;
   const data: JsonValue = {
     ...typeParam,
     workers,
-    files: files.length,
+    files: fileCount,
     durationMs,
     ...(options.filter === undefined ? {} : { filter: options.filter }),
     ...(sample === undefined ? {} : { sample: { kept: sample.kept, total: sample.total } }),
     ...(only === undefined ? {} : { shard: only }),
-    ok: result.ok,
-    exitCode: result.code,
+    // Only when the split made more than one, so a single-pass run's JSON is byte-identical to
+    // what it has always been — and a mixed one can never be read as if it were a single run.
+    ...(spent.length > 1 ? { passes: spent } : {}),
+    ok,
+    exitCode,
     reproduce: reproduceFor(plan),
   };
   return {
-    ok: result.ok,
+    ok,
     command: 'test',
-    summary: result.ok
+    summary: ok
       ? msg(type === undefined ? 'cli.test.pass' : 'cli.test.type.pass', {
           ...typeParam,
-          files: files.length,
+          files: fileCount,
           workers,
           ms: durationMs,
         })
       : msg(type === undefined ? 'cli.test.fail' : 'cli.test.type.fail', {
           ...typeParam,
-          failed: 1,
+          failed: steps.filter((step) => !step.ok).length,
           workers,
         }),
     steps,
@@ -242,6 +316,6 @@ export async function runShards(options: RunShardsOptions): Promise<CommandResul
       ? {}
       : { lines: [msg('cli.test.sampled', { ...sample, type: type ?? 'all' })] }),
     data,
-    exitCode: result.ok ? 0 : 1,
+    exitCode: ok ? 0 : 1,
   };
 }

package/src/verify-checks.ts

@@ -80,7 +80,7 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
   {
     name: 'boundaries',
     summary: "surface, layer and package-tier imports, and the app's own guards",
-    // An app's `guards/` rides here rather than becoming an eighteenth step, for the reason the
+    // An app's `guards/` rides here rather than becoming a step of its own, for the reason the
     // seam already states: a host adds findings to a step, it can never add, remove, reorder or
     // skip one — so "green" keeps meaning exactly what it meant. This is the step whose host slot
     // already carries "rules this repo makes about itself that the framework cannot know" (the
@@ -106,7 +106,7 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
     name: 'package-shape',
     summary: 'every package ships the same contract files',
     applies: (ctx) => hasWorkspacePackages(ctx.root),
-    // The dependency rule rides here rather than becoming a twentieth step because it is this
+    // The dependency rule rides here rather than becoming a step of its own because it is this
     // step's own question — what does a workspace owe the repo it lives in? — asked of the
     // manifest's `dependencies` instead of its `files`. It is deliberately NOT inside
     // `checkPackageShape`: `scripts/release.ts --check` calls that one to ask whether the tree is
@@ -155,7 +155,7 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
     // Source, not database: the gate runs in CI with nothing listening, and the database half is
     // the post-migrate verification `runMigrations` performs where a connection is already open.
     //
-    // The destructive rail rides here rather than becoming an eighteenth step because it asks this
+    // The destructive rail rides here rather than becoming a step of its own because it asks this
     // step's own question — do the committed migrations still describe what the app is doing to its
     // schema? — off the same directory, in the same pass, with no database either.
     applies: async (ctx) => existsSync(join(ctx.root, APP_CONFIG_FILE)),
@@ -166,7 +166,7 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
         // The third rail, and the one the other two cannot see: a hand-written statement is
         // recorded by no snapshot and hashed by no source, so both halves above are green over SQL
         // a squash silently drops. Same directory, same reader, no database — this step's own
-        // question, which is why it is not an eighteenth step.
+        // question, which is why it is not a step of its own.
         ...(await checkUngeneratableMigrations(ctx.root)),
       ]),
   },
@@ -191,7 +191,7 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
     name: 'budgets',
     summary:
       'per-route JS bytes and LCP, the global style layer every document carries, and the routes that boot nothing to receive their live rows',
-    // The global-style assertion rides here rather than becoming an eighteenth step, because this
+    // The global-style assertion rides here rather than becoming a step of its own, because this
     // step already asks the one question it asks: what does the document this build emits actually
     // contain? It is also the same app load — `appManifest` fills render's stylesheet registry on
     // its way through — so a separate step would pay for a second one to answer half a question.
@@ -295,7 +295,7 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
     // once, and says so by finding nothing — but `AGENTS.md` is required of every repo the gate
     // runs in, so the step always has a question to answer and must never report as skipped.
     //
-    // `.env.example` joins this step rather than becoming an eighteenth: the question is the same
+    // `.env.example` joins this step rather than becoming one of its own: the question is the same
     // one — "does a committed, generated file still describe the code?" — and the step list is the
     // definition of shippable, so it grows only when a genuinely new question needs asking.
     //
@@ -304,12 +304,15 @@ export const VERIFY_STEPS: readonly VerifyStep[] = [
     // vanish, so a typo in the floor covers nothing — which is the false green the floor exists to
     // close, and it is only visible if something reads the file for its own sake.
     async run(ctx) {
-      const agents = await checkAgentsMd(ctx.root);
+      // The floor is read FIRST: it is where a repository declares its own `AGENTS.md` budget,
+      // and reading it after the check would enforce the default on a repo that raised it.
+      const floor = await readVerifyFloor(ctx.root);
+      const agents = await checkAgentsMd(ctx.root, floor?.agentsMdMaxBytes);
       const findings = [
         ...manifestMissingFindings(ctx.root),
         ...(await driftFindings(ctx.root)),
         ...(await envExampleFindings(ctx.root)),
-        ...floorProblemFindings(await readVerifyFloor(ctx.root)),
+        ...floorProblemFindings(floor),
         ...agents.findings,
         ...(await hostFindings(ctx, 'manifest')),
       ];

package/src/verify-floor.ts

@@ -8,6 +8,7 @@
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { ERROR_DOCS_URL, renderThrowable } from '@ultimat3/core';
+import { AGENTS_MD_MAX_BYTES } from '@ultimat3/manifest';
 import type { Finding } from './output';
 import { VERIFY_STEP_NAMES } from './verify-step';
 
@@ -17,10 +18,45 @@ export const VERIFY_FLOOR_FILE = 'x.verify.json';
 export interface VerifyFloor {
   /** Declared step names this run may not report as skipped. */
   readonly steps: readonly string[];
+  /**
+   * This repository's `AGENTS.md` budget, in bytes, when it declares one. `@ultimat3/manifest`
+   * has always taken a `maxBytes` and the gate never passed one, so the 12kB default was the only
+   * budget an app could have — and an app whose conventions genuinely need more had no move left
+   * but to delete a rule to make room, which is the opposite of what the budget is for.
+   *
+   * It is here rather than in `x.config.ts` because this is the file that configures the GATE,
+   * and it is read by the very step that enforces the budget. Raising it is a commit a reviewer
+   * sees, which is the whole safeguard: the number is small, visible, and argued for in one place.
+   */
+  readonly agentsMdMaxBytes?: number;
   /** Why part of the file is not a floor. The `manifest` step reports these; nothing swallows them. */
   readonly problems: readonly string[];
 }
 
+/** The floor's budget key. Named once: the problem quotes it and the fix repairs it. */
+export const BUDGET_FIELD = 'agentsMdMaxBytes';
+
+/**
+ * `agentsMdMaxBytes`, or a reason it is not one. A budget that is not a positive whole number is
+ * the caller's bug and must not silently fall back to the default: a floor file that says
+ * `"agentsMdMaxBytes": "16kb"` and is quietly ignored is a repository that believes it raised a
+ * budget it did not, and finds out when the gate goes red on a commit that changed nothing.
+ */
+function readBudget(payload: Record<string, unknown> | undefined): {
+  budget?: number;
+  problems: readonly string[];
+} {
+  const raw = payload?.[BUDGET_FIELD];
+  if (raw === undefined) return { problems: [] };
+  if (typeof raw !== 'number' || !Number.isSafeInteger(raw) || raw <= 0)
+    return {
+      problems: [
+        `"${BUDGET_FIELD}" is ${JSON.stringify(raw)}, which is not a positive whole number of bytes`,
+      ],
+    };
+  return { budget: raw, problems: [] };
+}
+
 const asRecord = (value: unknown): Record<string, unknown> | undefined =>
   typeof value === 'object' && value !== null && !Array.isArray(value)
     ? (value as Record<string, unknown>)
@@ -48,19 +84,27 @@ export function parseVerifyFloor(
     // was meant to make the path safe (`metrics-endpoint.ts` states the same rule over `stringField`).
     return { steps: [], problems: [`it does not parse as JSON (${renderThrowable(error)})`] };
   }
-  const steps = asRecord(payload)?.['steps'];
+  const record = asRecord(payload);
+  const budget = readBudget(record);
+  const steps = record?.['steps'];
   if (!Array.isArray(steps)) {
-    return { steps: [], problems: ['it has no "steps" array of step names'] };
+    return {
+      steps: [],
+      ...(budget.budget === undefined ? {} : { agentsMdMaxBytes: budget.budget }),
+      problems: ['it has no "steps" array of step names', ...budget.problems],
+    };
   }
   const named = steps.filter((step): step is string => typeof step === 'string');
   const unknown = named.filter((step) => !declared.includes(step));
   return {
     steps: named.filter((step) => declared.includes(step)),
+    ...(budget.budget === undefined ? {} : { agentsMdMaxBytes: budget.budget }),
     problems: [
       ...(named.length === steps.length ? [] : ['"steps" holds an entry that is not a string']),
       ...(unknown.length === 0
         ? []
         : [`"steps" names ${unknown.join(', ')}, which x verify does not run`]),
+      ...budget.problems,
     ],
   };
 }
@@ -129,7 +173,19 @@ export const floorProblemFindings = (floor: VerifyFloor | undefined): readonly F
   (floor?.problems ?? []).map((problem) => ({
     code: 'X_CONFIG_INVALID',
     cause: `${VERIFY_FLOOR_FILE} is not a suite floor: ${problem}`,
-    fix: `x verify --json   # then write ${VERIFY_FLOOR_FILE} as {"steps":["unit","contract"]}, naming only steps it ran`,
+    fix: fixFor(problem),
     docs: ERROR_DOCS_URL,
     at: VERIFY_FLOOR_FILE,
   }));
+
+/**
+ * The edit that repairs THIS problem, not the file in general. The steps-shaped fix is useless
+ * against a bad budget — it prints a `{"steps":[…]}` example with no `agentsMdMaxBytes` in it, so
+ * an author who followed it verbatim would still have the value that failed. A finding whose fix
+ * does not fix it is the failure `packages/cli/CLAUDE.md` names, and the budget is the first
+ * problem this file can report that is not about `steps` at all.
+ */
+const fixFor = (problem: string): string =>
+  problem.includes(`"${BUDGET_FIELD}"`)
+    ? `x verify --json   # then set "${BUDGET_FIELD}" in ${VERIFY_FLOOR_FILE} to a positive whole number of bytes, or drop the key for the ${AGENTS_MD_MAX_BYTES}B default`
+    : `x verify --json   # then write ${VERIFY_FLOOR_FILE} as {"steps":["unit","contract"]}, naming only steps it ran`;

package/src/verify-step.ts

@@ -29,19 +29,19 @@ export const VERIFY_STEP_NAMES = [
   'drift',
   'contract-diff',
   'budgets',
-  // Eighteenth, and a deliberate widening of a closed list rather than a `HostCheck`: an SEO gate
+  // A deliberate widening of a closed list rather than a `HostCheck`: an SEO gate
   // is a MECHANISM every app with a `site/` surface wants (axiom 8), not a rule one host repo
   // enforces — and `verifyCommand.run` passes no host checks at all, so the app path could not
   // have carried it. It runs beside `budgets` because both read the app the same load produced.
   'seo',
-  // Nineteenth, by the same test the SEO step above passed and for the same reason it is not a
-  // rider: `boundaries` asks whether an import was LEGAL and this asks whether a declaration
+  // Here by the same test the SEO step above passed, and not a rider for the same reason:
+  // `boundaries` asks whether an import was LEGAL and this asks whether a declaration
   // REACHED the running app, which is a different question with a different fix (axiom 4). It
   // costs no second app load — `budgets` already imported every module, and this reads the
   // registries that load filled. Until it existed, an app could ship every user-facing string as
   // `⟦key⟧` with `x verify` green, because nothing in the gate ever asked (issue #249).
   'i18n',
-  // Twentieth, by the same test `seo` and `i18n` each passed: a rider must ask the SAME question
+  // Here by the same test `seo` and `i18n` each passed: a rider must ask the SAME question
   // off the same data, and "was this import legal?" is not "does the permission this app grants
   // and requires exist?". Reported under `budgets` it would hand the reader a byte budget for an
   // authz defect (axiom 4). Until it existed, `x new` shipped an app that answered HTTP 500 with

package/src/verify-tests.ts

@@ -44,7 +44,9 @@ const TYPED_SUFFIXES = '{contract,live,job,e2e,eval}';
  * four sweeps. `type` is a closed union here and cannot be `'constructor'` today, which is
  * exactly the argument every one of those thirteen had before it stopped being true.
  *
- * Same repair, and the same reason, as `packages/i18n/src/catalog.ts`.
+ * Same repair, and the same reason, as `packages/i18n/src/catalog.ts`. The read itself is guarded
+ * too (`summaryOf`), because a construction three screens above a read is not something a static
+ * rule should have to reason about.
  */
 const SUMMARIES: Readonly<Record<TypedTest, string>> = Object.assign(
   Object.create(null) as Record<TypedTest, string>,
@@ -255,6 +257,16 @@ const evalStep: VerifyStep = {
   },
 };
 
+/**
+ * The one computed read of `SUMMARIES`, guarded. The table is already prototype-free
+ * (`Object.create(null)`), which is what makes the READ safe — and `scripts/proto-index.ts`
+ * recognises a guard on the read itself, not a construction three screens above it. So the guard
+ * is here: a rule that has to reason about how a table was built is one tightening away from
+ * reporting this line, and a pin is a rule with a hole in it.
+ */
+const summaryOf = (type: TypedTest): string =>
+  Object.hasOwn(SUMMARIES, type) ? SUMMARIES[type] : `${type} tests`;
+
 const stepFor = (type: TestType): VerifyStep => {
   if (type === 'unit') {
     return {
@@ -266,7 +278,7 @@ const stepFor = (type: TestType): VerifyStep => {
   if (type === 'eval') return evalStep;
   return {
     name: type,
-    summary: SUMMARIES[type],
+    summary: summaryOf(type),
     applies: async (ctx) => (await filesFor(ctx.root, type)).length > 0,
     run: (ctx) => runType(ctx, type),
   };