@ultimat3/cli 22.6.1 → 22.7.0

package/src/verify-run.ts

@@ -5,7 +5,6 @@
 import { ERROR_DOCS_URL, renderThrowable } from '@ultimat3/core';
 import { msg } from './messages';
 import type { CommandResult, Finding, StepResult } from './output';
-import { sharedWorkers } from './test-workers';
 import {
   floorRequires,
   readVerifyFloor,
@@ -41,23 +40,27 @@ export async function runVerify(
     await pending;
     pending = undefined;
   };
-  // A parallel suite inside that window shares the cores with the static group, so its DEFAULT
-  // width is one per core (`sharedWorkers`); an explicit `--workers` is the caller's and stands.
-  const shared: VerifyContext =
-    ctx.workers === undefined && beside.length > 0 ? { ...ctx, workers: sharedWorkers() } : ctx;
+  // Isolation is the floor file's unless the caller said: absent in both is off (22.7).
+  const base: VerifyContext =
+    ctx.isolate === undefined && floor?.isolate !== undefined
+      ? { ...ctx, isolate: floor.isolate }
+      : ctx;
+  // A parallel suite inside the overlap window used to be narrowed to one worker per core; the
+  // default is at most that already (`test-workers.ts`), so the window runs the same plan.
+  const shared: VerifyContext = base;
   for (const step of selected) {
     if (beside.includes(step)) continue;
     if (step.name === SERIAL_SUITES[0]) {
       pending = Promise.all(
         beside.map(async (other) => {
-          byName.set(other.name, await runStep(other, ctx, floor));
+          byName.set(other.name, await runStep(other, base, floor));
         }),
       ).then(() => undefined);
     } else if (!SERIAL_SUITES.includes(step.name)) {
       await join();
     }
     const inWindow = pending !== undefined && SERIAL_SUITES.includes(step.name);
-    byName.set(step.name, await runStep(step, inWindow ? shared : ctx, floor));
+    byName.set(step.name, await runStep(step, inWindow ? shared : base, floor));
   }
   await join();
   // Reported in the declared order, whatever order the steps finished in: the table, `--json` and
@@ -113,7 +116,7 @@ const onlyList = (only: VerifyContext['only']): readonly string[] | undefined =>
  * vacuous gate stayed invisible. It names the skipped steps, not just how many: "17/17" is worth
  * something only when the gap is visible in the same glance.
  */
-function verifySummary(input: {
+export function verifySummary(input: {
   readonly results: readonly StepResult[];
   readonly failed: readonly string[];
   readonly skipped: readonly string[];
@@ -196,7 +199,9 @@ async function runStep(
   // ONE definition of "nothing ran", read twice, because the floor decides which of the two
   // things it means — exactly as it already does for a step whose `applies` said no.
   const tests = outcome.tests;
-  const nothingRan = tests !== undefined && tests.ran === 0;
+  // A shard's slice may hold nothing, or only skipped tests, and still be a correct slice: the
+  // "did the suite run at all" question is asked by `x verify merge`, on the summed counts.
+  const nothingRan = tests !== undefined && tests.ran === 0 && outcome.shard === undefined;
   const required = floorRequires(floor, step.name);
   // A step the floor requires whose suite executed nothing is the same vanished suite as a step
   // with no files at all — the run just had to finish before it could be seen. Appended to the
@@ -213,6 +218,8 @@ async function runStep(
     findings: [...outcome.findings, ...vanished],
     ...(outcome.output === undefined ? {} : { output: outcome.output }),
     ...(outcome.workers === undefined ? {} : { workers: outcome.workers }),
+    ...(outcome.widthReason === undefined ? {} : { widthReason: outcome.widthReason }),
+    ...(outcome.shard === undefined ? {} : { shard: outcome.shard }),
     ...(tests === undefined ? {} : { tests }),
   };
 }

package/src/verify-shard.ts

@@ -0,0 +1,145 @@
+// `x verify --only <step> --shard i/n`: one CI job's slice of a parallel test step, and the facts
+// `x verify merge` needs to prove that n jobs' slices were the whole corpus exactly once.
+//
+// The split is a PURE function of the sorted file list, n and (optionally) a timings file — never
+// of the machine, the clock or the order a glob yielded — so job 3 of 8 is the same files on every
+// runner, and a failing shard reproduces locally with the same flags.
+
+import { renderFixShellArg } from '@ultimat3/core';
+import { CryptoHasher } from 'bun';
+import type { ShardFacts } from './output';
+import { VerifyShardInvalidError } from './verify-errors';
+import type { VerifyStepName } from './verify-step';
+
+/** The steps a shard may split: the parallel suites. `live` and `e2e` are serial by design. */
+export const SHARDABLE_STEPS: readonly VerifyStepName[] = ['unit', 'contract', 'job'];
+
+export interface ShardSpec {
+  /** 1-based, as a CI matrix counts. */
+  readonly index: number;
+  readonly total: number;
+}
+
+/** `3/8` → `{ index: 3, total: 8 }`, refused unless `1 <= i <= n`. */
+export function parseShard(raw: string): ShardSpec {
+  const match = /^\s*(\d+)\s*\/\s*(\d+)\s*$/.exec(raw);
+  const index = match === null ? Number.NaN : Number(match[1]);
+  const total = match === null ? Number.NaN : Number(match[2]);
+  if (!Number.isSafeInteger(index) || !Number.isSafeInteger(total) || total < 1 || index < 1) {
+    throw new VerifyShardInvalidError({ reason: `"${raw}" is not i/n (e.g. 2/4)` });
+  }
+  if (index > total) {
+    throw new VerifyShardInvalidError({
+      reason: `shard ${String(index)} does not exist in a ${String(total)}-way split (1..${String(total)})`,
+      fix: `x verify --only unit --shard ${renderFixShellArg(`${String(total)}/${String(total)}`, '<i/n>')} --json`,
+    });
+  }
+  return { index, total };
+}
+
+/**
+ * The steps a shard run may name: every one must be shardable. Refused with the step named, and a
+ * fix that keeps the shardable ones — `live`/`e2e` belong in a job of their own.
+ */
+export function assertShardable(only: readonly VerifyStepName[] | undefined): void {
+  if (only === undefined || only.length === 0) {
+    throw new VerifyShardInvalidError({
+      reason: 'a shard is one slice of ONE parallel step and needs --only to name it',
+      fix: 'x verify --only unit --shard 1/4 --json',
+    });
+  }
+  const refused = only.filter((step) => !SHARDABLE_STEPS.includes(step));
+  if (refused.length > 0) {
+    const kept = only.filter((step) => SHARDABLE_STEPS.includes(step));
+    throw new VerifyShardInvalidError({
+      reason: `${refused.join(', ')} cannot be split (only ${SHARDABLE_STEPS.join(', ')}); run ${
+        refused.length === 1 ? 'it' : 'them'
+      } in a job of its own without --shard`,
+      fix:
+        kept.length === 0
+          ? `x verify --only ${renderFixShellArg(refused.join(','), '<step,step>')} --json`
+          : `x verify --only ${renderFixShellArg(kept.join(','), '<step,step>')} --shard 1/4 --json`,
+    });
+  }
+}
+
+/** sha256 of the sorted list, one path a line: equal across shards iff they split one corpus. */
+export const corpusHash = (files: readonly string[]): string => {
+  const hasher = new CryptoHasher('sha256');
+  hasher.update([...files].sort().join('\n'));
+  return hasher.digest('hex');
+};
+
+/** Per-file durations in ms — `{ "path": ms }`, as in Bun's `--timings` file's `files`. */
+export type Timings = Readonly<Record<string, number>>;
+
+/**
+ * Which files shard `index` of `total` runs. Round-robin over the sorted list by default — every
+ * shard gets a slice of every directory. With `timings`, greedy longest-first: each file, slowest
+ * first (ties by path), goes to the shard with the least total so far (ties to the lowest index);
+ * a file the timings do not know is costed at their median, so a new file cannot pile onto one job.
+ */
+export function shardFiles(
+  files: readonly string[],
+  spec: ShardSpec,
+  timings?: Timings,
+): ShardFacts {
+  const sorted = [...new Set(files)].sort();
+  const hash = corpusHash(sorted);
+  let picked: string[];
+  if (timings === undefined) {
+    picked = sorted.filter((_, position) => position % spec.total === spec.index - 1);
+  } else {
+    const known = sorted
+      .map((file) => timings[file])
+      .filter((ms): ms is number => typeof ms === 'number' && Number.isFinite(ms) && ms >= 0)
+      .sort((a, b) => a - b);
+    const median = known.length === 0 ? 1 : (known[Math.floor(known.length / 2)] as number);
+    const cost = (file: string): number => {
+      const ms = Object.hasOwn(timings, file) ? timings[file] : undefined;
+      return typeof ms === 'number' && Number.isFinite(ms) && ms >= 0 ? ms : median;
+    };
+    const order = [...sorted].sort((a, b) => cost(b) - cost(a) || (a < b ? -1 : a > b ? 1 : 0));
+    const loads = Array.from({ length: spec.total }, () => 0);
+    const owner = new Map<string, number>();
+    for (const file of order) {
+      let best = 0;
+      for (let shard = 1; shard < spec.total; shard += 1) {
+        if ((loads[shard] as number) < (loads[best] as number)) best = shard;
+      }
+      loads[best] = (loads[best] as number) + cost(file);
+      owner.set(file, best);
+    }
+    picked = sorted.filter((file) => owner.get(file) === spec.index - 1);
+  }
+  return { index: spec.index, total: spec.total, corpusHash: hash, files: picked };
+}
+
+/** A timings file, or a refusal naming it — a CI job must not silently fall back to round-robin. */
+export async function readTimings(path: string): Promise<Timings> {
+  const file = Bun.file(path);
+  if (!(await file.exists())) {
+    throw new VerifyShardInvalidError({
+      reason: `--timings ${path} does not exist`,
+      fix: 'bun test --timings=timings.json --update-timings   # writes it, then commit or cache it',
+    });
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(await file.text());
+  } catch {
+    parsed = undefined;
+  }
+  // Bun's own `--update-timings` writes `{ "version": 1, "files": { "path": ms } }`; a bare map
+  // is accepted too.
+  if (typeof parsed === 'object' && parsed !== null && 'files' in parsed) {
+    parsed = (parsed as { files: unknown }).files;
+  }
+  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+    throw new VerifyShardInvalidError({
+      reason: `--timings ${path} is not a JSON object of { "path": ms }`,
+      fix: 'bun test --timings=timings.json --update-timings',
+    });
+  }
+  return parsed as Timings;
+}

package/src/verify-step.ts

@@ -5,7 +5,7 @@
 import { ERROR_DOCS_URL } from '@ultimat3/core';
 import type { ExecResult, Runner } from './exec';
 import { execOutput } from './exec';
-import type { Finding } from './output';
+import type { Finding, ShardFacts } from './output';
 import type { TestCounts } from './test-counts';
 
 /**
@@ -78,6 +78,21 @@ export interface VerifyContext {
    * narrowing: no value of it changes which steps run or what "green" means.
    */
   readonly workers?: number;
+  /**
+   * A fresh global per test file (`bun test --isolate`). Absent means `x.verify.json`'s
+   * `"isolate"`, and absent there means off — the default since 22.7.
+   */
+  readonly isolate?: boolean;
+  /**
+   * One slice of an n-way CI split of each selected parallel step's file list (`--shard i/n`,
+   * 1-based `index`). Only with `only`; `x verify merge` reassembles the verdict.
+   */
+  readonly shard?: {
+    readonly index: number;
+    readonly total: number;
+    /** Bun `--timings` durations: greedy longest-first instead of round-robin. */
+    readonly timings?: Readonly<Record<string, number>>;
+  };
   /**
    * Some steps, by name — an iteration loop, and the one thing here that IS a narrowing. The
    * whole gate costs minutes (3m19s at the framework root on 12 cores, `As of 2026-09-23`), which
@@ -103,6 +118,10 @@ export interface StepOutcome {
    * reading the step list's source.
    */
   readonly workers?: number;
+  /** Why `workers` is what it is — `4 workers (budget 4.0 GB)`. Printed on the step line. */
+  readonly widthReason?: string;
+  /** The CI slice this step ran, under `--shard`: which files, and a hash of the whole list. */
+  readonly shard?: ShardFacts;
   /**
    * What the suite executed, for the steps that run one. Absent means this step spawned no test
    * process at all — a check step, or the `eval` step answering with declarations alone — and is

package/src/verify-test-run.ts

@@ -2,13 +2,16 @@
 // files belong to a type and this one owns what happens to them once selected — a wrong file list
 // is never a race, and a race is never a selection bug.
 
+// why: Bun ships no path-join primitive; the timings cache lives under the run's root.
+import { join } from 'node:path';
 import type { Runner } from './exec';
-import { execOutput } from './exec';
 import type { Finding } from './output';
+import { runBatches, testBatches } from './test-batches';
 import { countsOf } from './test-counts';
 import { testEnvOverrides } from './test-dotenv';
 import type { TestFile } from './test-select';
-import { failureOf, testArgs } from './test-shards';
+import { failureOf, ISOLATED_TEST_ENV, TEST_TIMINGS_FILE, testArgs } from './test-shards';
+import { BATCH_FILES_PER_WORKER, SHARED_BATCH_FILES_PER_WORKER } from './test-workers';
 import type { StepOutcome } from './verify-step';
 // Type-only, so nothing here evaluates verify-tests.ts and the two files cannot form a cycle.
 import type { TestType } from './verify-tests';
@@ -23,10 +26,17 @@ export interface ParallelRunOptions {
   readonly type: TestType;
   /** This process's own environment. Optional and defaulted to `Bun.env` — see `VerifyContext.env`. */
   readonly env?: Readonly<Record<string, string | undefined>>;
+  /** A fresh global per file — `x.verify.json` `"isolate": true` or `--isolate`. Default off. */
+  readonly isolate?: boolean;
+  /** Why the width is what it is — printed on the step line. */
+  readonly widthReason?: string;
+  /** The machine-pool lease (`test-slots.ts`), for a default-width run. */
+  readonly lease?: (want: number) => Promise<{ readonly count: number; release(): void }>;
 }
 
 /**
- * ONE `bun test --parallel=N`, not N processes this file spawns and packs itself.
+ * `bun test --parallel=N` per batch (`test-batches.ts`), not N processes this file spawns and
+ * packs itself.
  *
  * Two things make an arbitrary partition safe here and neither is optional. `--parallel` implies
  * `--isolate`, so every FILE gets a fresh module registry — half a dozen registries in this
@@ -41,24 +51,66 @@ export interface ParallelRunOptions {
 export async function runParallel(options: ParallelRunOptions): Promise<StepOutcome> {
   const files = options.files.map((file) => file.path);
   const workers = Math.max(1, Math.min(Math.trunc(options.workers), files.length || 1));
-  const envOverrides = testEnvOverrides(options.root, options.env ?? Bun.env);
-  const result = await options.runner(testArgs({ files, workers }), {
-    cwd: options.root,
-    ...(Object.keys(envOverrides).length === 0 ? {} : { env: envOverrides }),
+  const envOverrides = {
+    ...testEnvOverrides(options.root, options.env ?? Bun.env),
+    ...(options.isolate === true ? { [ISOLATED_TEST_ENV]: '1' } : {}),
+  };
+  // Batched exactly as `x test` batches the same selection at the same width, so the gate's
+  // `fix:` line — `x test <type> --workers N` — reruns the very processes that failed.
+  const batches = testBatches(
+    files,
+    workers,
+    options.isolate === true ? BATCH_FILES_PER_WORKER : SHARED_BATCH_FILES_PER_WORKER,
+  );
+  const result = await runBatches({
+    runner: options.runner,
+    batches,
+    workers,
+    ...(options.lease === undefined || workers === 1 ? {} : { lease: options.lease }),
+    argsFor: (batch, width) =>
+      testArgs({
+        files: batch,
+        workers: width,
+        isolate: options.isolate === true,
+        timings: join(options.root, TEST_TIMINGS_FILE),
+      }),
+    options: {
+      cwd: options.root,
+      ...(Object.keys(envOverrides).length === 0 ? {} : { env: envOverrides }),
+    },
   });
   // `failureOf` is `x test`'s own, imported rather than restated: the two paths report the SAME
   // failed `bun test`, so a second literal here is two `cause:` strings and two `fix:` lines free
   // to drift — and the one that drifts is the gate's, which is the one an agent reads first.
   const findings: readonly Finding[] = result.ok
     ? []
-    : [failureOf(result.code, files.length, { workers, type: options.type })];
+    : [
+        failureOf(result.code, files.length, {
+          workers,
+          type: options.type,
+          ...(options.isolate === true ? { isolate: true } : {}),
+        }),
+      ];
+  // The width the run REACHED: a batch the machine pool narrowed ran narrower than planned, and a
+  // step line that printed the plan would name a width that never ran.
+  const reached = Math.max(1, ...result.widths);
+  const narrowed = reached < workers;
   return {
     ok: findings.length === 0,
     findings,
-    workers,
-    tests: countsOf([result]),
+    workers: reached,
+    ...(options.widthReason === undefined
+      ? {}
+      : {
+          widthReason: narrowed
+            ? `${String(reached)} of ${options.widthReason} — machine slots held by another run`
+            : options.widthReason,
+        }),
+    // Per batch: each process printed its own summary, and one parse of the joined text would
+    // read only the last one.
+    tests: countsOf(result.results),
     // Only on failure: a green run's summary is already the step table's, and the reader of a red
     // gate needs the assertion diff.
-    ...(result.ok ? {} : { output: execOutput(result) }),
+    ...(result.ok ? {} : { output: result.output }),
   };
 }

package/src/verify-tests.ts

@@ -18,8 +18,11 @@ import { countsOf } from './test-counts';
 import { testEnvOverrides } from './test-dotenv';
 import type { TestFile } from './test-select';
 import { discoverTests } from './test-select';
-import { defaultWorkers, SERIAL_TYPES } from './test-workers';
+import { machineLease } from './test-slots';
+import type { WorkerPlan } from './test-workers';
+import { availableCpus, SERIAL_TYPES, totalMemory, workerPlan } from './test-workers';
 import { withE2eApp } from './verify-e2e';
+import { shardFiles } from './verify-shard';
 import type { StepOutcome, VerifyContext, VerifyStep } from './verify-step';
 import { fromExec, fromFindings } from './verify-step';
 import { runParallel } from './verify-test-run';
@@ -212,16 +215,48 @@ const runSerial = async (ctx: VerifyContext, type: TestType): Promise<StepOutcom
 
 const runType = async (ctx: VerifyContext, type: TestType): Promise<StepOutcome> => {
   if (isSerial(type)) return runSerial(ctx, type);
-  const files = await filesFor(ctx.root, type);
+  const discoveredFiles = await filesFor(ctx.root, type);
+  // Under `--shard`, this job runs its slice and says which — `x verify merge` checks the slices.
+  // An EMPTY slice is a pass with zero tests, never the serial fallback below (which would run
+  // the whole type): the zero-tests floor is `merge`'s to apply, on the summed counts.
+  const shard =
+    ctx.shard === undefined
+      ? undefined
+      : shardFiles(
+          discoveredFiles.map((file) => file.path),
+          ctx.shard,
+          ctx.shard.timings,
+        );
+  if (shard !== undefined && shard.files.length === 0) {
+    return { ok: true, findings: [], workers: 0, tests: { ran: 0, skipped: 0 }, shard };
+  }
+  const files =
+    shard === undefined
+      ? discoveredFiles
+      : discoveredFiles.filter((file) => shard.files.includes(file.path));
   if (files.length === 0) return runSerial(ctx, type);
-  return runParallel({
+  const env = ctx.env ?? Bun.env;
+  const plan =
+    ctx.workers === undefined ? workerPlan(availableCpus(), totalMemory(), env) : undefined;
+  const workers = ctx.workers ?? (plan as WorkerPlan).workers;
+  const lease = plan === undefined ? undefined : machineLease(plan.workers, env);
+  const outcome = await runParallel({
     root: ctx.root,
     runner: ctx.runner,
     files,
-    workers: ctx.workers ?? defaultWorkers(),
+    workers,
     type,
+    isolate: ctx.isolate === true,
+    widthReason:
+      plan === undefined
+        ? `${String(Math.min(workers, files.length))} workers (--workers)`
+        : files.length < plan.workers
+          ? `${String(files.length)} of ${plan.reason}`
+          : plan.reason,
+    ...(lease === undefined ? {} : { lease }),
     ...(ctx.env === undefined ? {} : { env: ctx.env }),
   });
+  return shard === undefined ? outcome : { ...outcome, shard };
 };
 
 const isApp = (root: string): boolean => existsSync(join(root, APP_CONFIG_FILE));