@ultimat3/cli 22.6.1 → 22.7.0

package/src/test-shards.ts

@@ -2,6 +2,11 @@
 // reproduces it exactly. Split out of cmd-test.ts because a printed reproduction is only true if
 // it carries every input to the run — that rule is this file's, and argv parsing is that one's.
 //
+// ONE PROCESS PER BATCH, N WORKERS. Until 2026-09-27 it was one process for the whole selection;
+// a worker's heap grows with every file it runs, so that process's peak grew with the corpus, and
+// `test-batches.ts` now spends each pass as a few `--parallel=N` processes in sequence — the
+// measurement is its header. Everything below still holds inside a batch.
+//
 // ONE PROCESS, N WORKERS, `As of 2026-08-27`. This file used to pack the files into N bins itself
 // (largest-first greedy over file SIZE) and `Promise.all` one `bun test` per bin. Bun 1.4 runs the
 // pool itself — `--parallel=N`, which starts each file on the next free worker — so the packer is
@@ -21,10 +26,11 @@
 // file's packer and its `Shard` type gone, one process instead of eight, and Bun's own summary
 // instead of eight merged ones (issue #342).
 //
-// `--timings` IS REFUSED FOR THE SAME REASON. Bun will start the slowest files first from a
+// `--timings` WAS REFUSED FOR THE SAME REASON (until 22.7, below). Bun will start the slowest files first from a
 // recorded timings file, but there is at most the ~5s between the measured 59.8s median and the
 // 54.6s floor in it — against a committed JSON that goes stale on every test edit and that nothing
-// in the gate would notice had gone stale.
+// in the gate would notice had gone stale. 22.7 uses it differently: a LOCAL cache under `.x/`
+// (`TEST_TIMINGS_FILE`), rewritten by every run, so it cannot go stale and nothing is committed.
 //
 // TWO THINGS THE OLD SPLIT OWNED AND BUN NOW OWNS. `--parallel` implies `--isolate`, so the
 // per-FILE module registry that made an arbitrary partition safe at all is unchanged. Isolation is
@@ -36,10 +42,11 @@
 // process (probed on 1.4.0). `ULTIMATE_TEST_WORKER` stays the first key and is what `--worker`
 // still sets, so a single-shard rerun keeps naming its own database.
 
+// why: Bun ships no path-join primitive; the timings cache lives under the run's root.
+import { join } from 'node:path';
 import { ERROR_DOCS_URL } from '@ultimat3/core';
 import type { AffectedSelection } from './affected';
 import type { Runner } from './exec';
-import { execOutput } from './exec';
 import { msg } from './messages';
 import type { CommandResult, Finding, JsonValue, StepResult } from './output';
 import { quoteArg } from './shell-quote';
@@ -48,8 +55,13 @@ import { testEnvOverrides } from './test-dotenv';
 /** `@ultimat3/testing`'s `ISOLATED_ENV`, restated: `cli → testing` is a runtime edge, kept to fixtures. */
 export const ISOLATED_TEST_ENV = 'ULTIMATE_TEST_ISOLATED';
 
+/** Bun's per-file durations, cached under the run's root and refreshed by every parallel run. */
+export const TEST_TIMINGS_FILE = '.x/test-timings.json';
+
+import { runBatches, testBatches } from './test-batches';
 import { testPasses } from './test-passes';
 import type { TestFile } from './test-select';
+import { BATCH_FILES_PER_WORKER, SHARED_BATCH_FILES_PER_WORKER } from './test-workers';
 import type { TestType } from './verify-tests';
 
 /**
@@ -83,17 +95,43 @@ export function testArgs(input: {
    * same in silence: they refuse the `--` instead.
    */
   readonly passthrough?: readonly string[];
+  /**
+   * A fresh global per FILE. Off by default since 22.7 — `--parallel` implies it, so the default
+   * run says `--no-isolate` — and on for a repository whose `x.verify.json` says `"isolate": true`
+   * or a caller's `--isolate`. Measured, 4 workers: the framework's unit corpus 168s isolated,
+   * 77s not; a notificado.co third 85-107s isolated, 20s not (`test-workers.ts`).
+   */
+  readonly isolate?: boolean;
+  /**
+   * A per-file durations cache (`TEST_TIMINGS_FILE`): Bun starts the slowest files first from it
+   * and rewrites it after the run, so the last straggler is never the file that started last.
+   * Parallel form only; a caller's own `--timings` wins.
+   */
+  readonly timings?: string;
 }): readonly string[] {
   const files = [...input.files].sort();
   const extra = input.passthrough ?? [];
+  const isolate =
+    extra.includes('--isolate') || (input.isolate === true && !extra.includes('--no-isolate'));
+  const bare = extra.filter((arg) => arg !== '--isolate' && arg !== '--no-isolate');
   return input.shard === undefined
-    ? ['bun', 'test', `--parallel=${String(input.workers)}`, ...extra, ...files]
+    ? [
+        'bun',
+        'test',
+        `--parallel=${String(input.workers)}`,
+        ...(isolate ? [] : ['--no-isolate']),
+        ...(input.timings === undefined || bare.some((arg) => arg.startsWith('--timings'))
+          ? []
+          : [`--timings=${input.timings}`, '--update-timings']),
+        ...bare,
+        ...files,
+      ]
     : [
         'bun',
         'test',
-        '--isolate',
+        ...(isolate ? ['--isolate'] : []),
         `--shard=${String(input.shard + 1)}/${String(input.workers)}`,
-        ...extra,
+        ...bare,
         ...files,
       ];
 }
@@ -124,6 +162,8 @@ export interface ReproduceOptions {
   readonly shard?: number;
   /** What the caller put after `--`. It reaches `bun test`, so a rerun without it runs differently. */
   readonly passthrough?: readonly string[];
+  /** The run was isolated per file; the rerun must be too. */
+  readonly isolate?: boolean;
 }
 
 /**
@@ -146,6 +186,7 @@ export function reproduceFor(options: ReproduceOptions): string {
     '--workers',
     String(options.workers),
     ...(options.shard === undefined ? [] : ['--worker', String(options.shard)]),
+    ...(options.isolate === true ? ['--isolate'] : []),
     // Last, and after a `--` of its own, because that is where the caller typed it and where the
     // parser will find it again. Quoted for `shell-quote.ts`'s reason: a reproduce line is pasted.
     ...(options.passthrough === undefined || options.passthrough.length === 0
@@ -180,8 +221,19 @@ export interface RunShardsOptions {
    * fallback so much as a seam this file's own tests use to hand it a fixture instead.
    */
   readonly env?: Readonly<Record<string, string | undefined>>;
+  /** A fresh global per file (`x.verify.json` `"isolate": true`, or `--isolate`). Default off. */
+  readonly isolate?: boolean;
+  /** The machine-pool lease (`test-slots.ts`), for a default-width run. */
+  readonly lease?: (want: number) => Promise<{ readonly count: number; release(): void }>;
 }
 
+const isolatedRun = (options: RunShardsOptions): boolean => {
+  const extra = options.passthrough ?? [];
+  return (
+    extra.includes('--isolate') || (options.isolate === true && !extra.includes('--no-isolate'))
+  );
+};
+
 /**
  * The reproduction's inputs for ONE pass: `workers` is that pass's real width, not the ask, and
  * `type` is the pass's own when the split gave it one — `x test live --workers 1` reruns exactly
@@ -197,6 +249,7 @@ const planOf = (
   ...(options.sample === undefined ? {} : { sample: options.sample.kept }),
   ...(options.affected === undefined ? {} : { affected: options.affected }),
   ...(options.only === undefined ? {} : { shard: options.only }),
+  ...(options.isolate === true ? { isolate: true } : {}),
   ...(options.passthrough === undefined || options.passthrough.length === 0
     ? {}
     : { passthrough: options.passthrough }),
@@ -219,10 +272,10 @@ export const failureOf = (code: number, files: number, plan: ReproduceOptions):
 });
 
 /**
- * ONE `bun test` PER PASS, and one pass unless the selection mixes serial files with the rest —
- * `test-passes.ts` decides that, and this spends it. Bun owns the pool inside a pass and hands
- * each free worker the next file, so nothing here decides which file runs where; see this file's
- * header for what that measured.
+ * ONE `bun test` PER BATCH, a pass being as many batches as `test-batches.ts` cuts it into, and
+ * one pass unless the selection mixes serial files with the rest — `test-passes.ts` decides that,
+ * and this spends it. Bun owns the pool inside a batch and hands each free worker the next file, so
+ * nothing here decides which file runs where; see this file's header for what that measured.
  *
  * Sequential, never `Promise.all`: the whole point of a serial pass is that nothing runs beside
  * it. And every pass runs even after one fails — the caller asked for a suite, and a report that
@@ -238,9 +291,13 @@ export async function runShards(options: RunShardsOptions): Promise<CommandResul
   // same parent env, so the leaked-key set cannot differ pass to pass.
   const envOverrides: Record<string, string | undefined> = {
     ...testEnvOverrides(options.root, options.env ?? Bun.env),
-    // Every run this spawns is isolated (`--parallel` or `--isolate`) unless the caller opted out.
-    ...((options.passthrough ?? []).includes('--no-isolate') ? {} : { [ISOLATED_TEST_ENV]: '1' }),
+    // Only an isolated run: the preload clears `Bun.plugin` handlers after each file, which in a
+    // shared global would leave the next `.tsx` compiling with Bun's classic factory.
+    ...(isolatedRun(options) ? { [ISOLATED_TEST_ENV]: '1' } : {}),
   };
+  const passthrough = options.passthrough ?? [];
+  const watching = passthrough.includes('--watch');
+  const bailing = passthrough.some((arg) => arg === '--bail' || arg.startsWith('--bail='));
   const passes = testPasses({
     files: options.files,
     workers: options.workers,
@@ -250,18 +307,43 @@ export async function runShards(options: RunShardsOptions): Promise<CommandResul
   const started = performance.now();
   const steps: StepResult[] = [];
   const spent: JsonValue[] = [];
+  let batchCount = 0;
   let ok = true;
   let exitCode = 0;
   for (const pass of passes) {
     const files = pass.files.map((file) => file.path);
-    const result = await options.runner(
-      testArgs({
-        files,
-        workers: pass.workers,
-        ...(only === undefined ? {} : { shard: only }),
-        ...(options.passthrough === undefined ? {} : { passthrough: options.passthrough }),
-      }),
-      {
+    // A `--worker` rerun is one process over one shard and is never split: shard i of the rerun
+    // must be the same files as shard i of the run it reproduces. Nor is `-- --watch`, which never
+    // exits, so a second batch would never start. Every other pass is spent in `test-batches.ts`'
+    // batches, one `bun test` after another.
+    const batches =
+      only === undefined && !watching
+        ? testBatches(
+            files,
+            pass.workers,
+            isolatedRun(options) ? BATCH_FILES_PER_WORKER : SHARED_BATCH_FILES_PER_WORKER,
+          )
+        : [files];
+    const result = await runBatches({
+      runner: options.runner,
+      batches,
+      stopOnFailure: bailing,
+      workers: pass.workers,
+      // A `--worker` rerun is one process whatever its N says, and a serial pass is one worker:
+      // only a default-width parallel pass leases from the machine pool.
+      ...(only === undefined && pass.workers > 1 && options.lease !== undefined
+        ? { lease: options.lease }
+        : {}),
+      argsFor: (batch, width) =>
+        testArgs({
+          files: batch,
+          workers: only === undefined ? width : pass.workers,
+          ...(only === undefined ? {} : { shard: only }),
+          ...(options.passthrough === undefined ? {} : { passthrough: options.passthrough }),
+          ...(options.isolate === undefined ? {} : { isolate: options.isolate }),
+          timings: join(options.root, TEST_TIMINGS_FILE),
+        }),
+      options: {
         cwd: options.root,
         ...(Object.keys(envOverrides).length === 0 && only === undefined
           ? {}
@@ -272,24 +354,28 @@ export async function runShards(options: RunShardsOptions): Promise<CommandResul
               },
             }),
       },
-    );
+    });
+    batchCount += batches.length;
     const plan = planOf(options, pass);
     const label =
       only === undefined ? `${pass.workers} worker(s)` : `shard ${only} of ${pass.workers}`;
     steps.push({
-      name: `${pass.type === undefined ? label : `${pass.type} · ${label}`} · ${files.length} files`,
+      name: `${pass.type === undefined ? label : `${pass.type} · ${label}`} · ${files.length} files${
+        batches.length > 1 ? msg('cli.test.batches', { batches: batches.length }) : ''
+      }`,
       ok: result.ok,
       durationMs: result.durationMs,
       // `output.ts` documents this field as absent for a NON-test step, so omitting it here made
       // `renderJson` describe the test step as one — recoverable only by parsing `name`.
       workers: pass.workers,
       findings: result.ok ? [] : [failureOf(result.code, files.length, plan)],
-      output: execOutput(result),
+      output: result.output,
     });
     spent.push({
       ...(pass.type === undefined ? {} : { type: pass.type }),
       files: files.length,
       workers: pass.workers,
+      ...(batches.length > 1 ? { batches: batches.length } : {}),
       ok: result.ok,
       exitCode: result.code,
       reproduce: reproduceFor(plan),
@@ -321,6 +407,8 @@ export async function runShards(options: RunShardsOptions): Promise<CommandResul
     // Only when the split made more than one, so a single-pass run's JSON is byte-identical to
     // what it has always been — and a mixed one can never be read as if it were a single run.
     ...(spent.length > 1 ? { passes: spent } : {}),
+    // Only when some pass was split, for the same reason: an unbatched run's JSON is unchanged.
+    ...(batchCount > spent.length ? { batches: batchCount } : {}),
     ok,
     exitCode,
     reproduce: reproduceFor(plan),

package/src/test-slots.ts

@@ -0,0 +1,168 @@
+// The MACHINE's test budget, shared by every `x test` and `x verify` running on it at once.
+//
+// A run's own width (`test-workers.ts`) bounds one run. It cannot bound two: on 2026-09-27 two
+// agents' gates on one 45 GB box each planned 14-18 workers off the same free-memory reading and
+// the kernel OOM-killed the machine with 36 `bun` processes holding ~40 GB. So each batch of
+// workers first LEASES that many slots from a pool of lock files — one file per worker the machine
+// budget allows — and a second run on the same box gets what is left: it runs narrower, or waits
+// for one slot, instead of doubling the machine's memory.
+//
+// A slot is a file created with O_EXCL holding the leasing process's pid. A holder that died
+// without releasing (SIGKILL, OOM, a closed terminal) is detected by its pid no longer existing,
+// and its slot is taken over. Nothing here needs a daemon, and nothing survives a reboot that
+// matters: the directory is under the OS temp dir.
+//
+// Bun ships no O_EXCL create and no pid probe: `openSync(…, 'wx')` and `process.kill(pid, 0)` are
+// the node:fs / process primitives for exactly those two.
+// why: Bun ships no O_EXCL create (writeFileSync flag 'wx') and no sync unlink/mkdir.
+import { mkdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+// why: Bun exposes no tmpdir(); only node:os answers the platform temp root.
+import { tmpdir } from 'node:os';
+// why: Bun ships no path-join primitive.
+import { join } from 'node:path';
+
+/** Where the pool lives. Per user, so two accounts on one box never read each other's pids. */
+export const SLOTS_DIR_ENV = 'ULTIMATE_TEST_SLOTS_DIR';
+
+/** `0` turns the machine pool off — a CI runner that runs one job per VM has nothing to share. */
+export const SLOTS_ENV = 'ULTIMATE_TEST_SLOTS';
+
+/**
+ * Set on every `bun test` a leased batch spawns: the slots are already held for that process
+ * tree, so an `x test` / `x verify` a test runs inside it (the framework's own suites do) must
+ * not lease again — it would wait on its own parent's slots.
+ */
+export const SLOT_HELD_ENV = 'ULTIMATE_TEST_SLOT_HELD';
+
+/** How long a run waits for its first slot before it runs one worker anyway, and says so. */
+export const SLOT_WAIT_MS = 10 * 60 * 1000;
+
+const POLL_MS = 500;
+
+type Env = Readonly<Record<string, string | undefined>>;
+
+export const slotsDir = (env: Env = Bun.env): string =>
+  env[SLOTS_DIR_ENV] ??
+  join(tmpdir(), `ultimate-test-slots-${String(process.getuid?.() ?? 'user')}`);
+
+export const slotsEnabled = (env: Env = Bun.env): boolean =>
+  env[SLOTS_ENV] !== '0' && env[SLOT_HELD_ENV] !== '1' && Bun.env[SLOT_HELD_ENV] !== '1';
+
+/** A lease on `count` workers' worth of the machine. `count` is at least 1. */
+export interface SlotLease {
+  readonly count: number;
+  /** Slots actually held — 0 when the wait ran out and the run went ahead unleased. */
+  readonly held: number;
+  /** How long the lease waited for its first slot. */
+  readonly waitedMs: number;
+  release(): void;
+}
+
+const pidAlive = (pid: number): boolean => {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // EPERM is a live process another user owns — alive, and not ours to take.
+    return (error as { code?: string } | null)?.code === 'EPERM';
+  }
+};
+
+export interface AcquireInput {
+  /** Workers this batch wants. */
+  readonly want: number;
+  /** Slots the machine budget allows in all — `defaultWorkers()` for this box. */
+  readonly capacity: number;
+  readonly dir?: string;
+  readonly pid?: number;
+  readonly alive?: (pid: number) => boolean;
+  readonly waitMs?: number;
+  readonly sleep?: (ms: number) => Promise<void>;
+  readonly now?: () => number;
+}
+
+/** Take one slot file, or take it over from a dead holder. */
+function tryTake(path: string, pid: number, alive: (pid: number) => boolean): boolean {
+  for (let attempt = 0; attempt < 2; attempt += 1) {
+    try {
+      writeFileSync(path, String(pid), { flag: 'wx' });
+      return true;
+    } catch (error) {
+      if ((error as { code?: string } | null)?.code !== 'EEXIST') return false;
+    }
+    let holder: number;
+    try {
+      holder = Number.parseInt(readFileSync(path, 'utf8'), 10);
+    } catch {
+      // Released between the create and the read: try the create once more.
+      continue;
+    }
+    if (alive(holder)) return false;
+    try {
+      unlinkSync(path);
+    } catch {
+      // Another run took it over first; the second create decides.
+    }
+  }
+  return false;
+}
+
+/**
+ * Lease up to `want` slots out of `capacity`. Never zero: with every slot held, it polls until one
+ * frees, and after `waitMs` it runs one worker unleased rather than hang a gate forever.
+ */
+export async function acquireSlots(input: AcquireInput): Promise<SlotLease> {
+  const dir = input.dir ?? slotsDir();
+  const pid = input.pid ?? process.pid;
+  const alive = input.alive ?? pidAlive;
+  const sleep = input.sleep ?? ((ms: number) => Bun.sleep(ms));
+  const now = input.now ?? (() => performance.now());
+  const want = Math.max(1, Math.trunc(input.want));
+  const capacity = Math.max(1, Math.trunc(input.capacity));
+  mkdirSync(dir, { recursive: true });
+  const started = now();
+  const waitMs =
+    input.waitMs !== undefined && Number.isFinite(input.waitMs) && input.waitMs >= 0
+      ? input.waitMs
+      : SLOT_WAIT_MS;
+  const deadline = started + waitMs;
+  for (;;) {
+    const held: string[] = [];
+    for (let index = 0; index < capacity && held.length < want; index += 1) {
+      const path = join(dir, `slot-${String(index)}`);
+      if (tryTake(path, pid, alive)) held.push(path);
+    }
+    if (held.length > 0 || now() >= deadline) {
+      const waitedMs = Math.round(now() - started);
+      let released = false;
+      return {
+        count: Math.max(1, held.length),
+        held: held.length,
+        waitedMs,
+        release: () => {
+          if (released) return;
+          released = true;
+          for (const path of held) {
+            try {
+              if (readFileSync(path, 'utf8') === String(pid)) unlinkSync(path);
+            } catch {
+              // Already gone — a crashed-holder sweep by another run took it.
+            }
+          }
+        },
+      };
+    }
+    await sleep(POLL_MS);
+  }
+}
+
+/**
+ * The lease a default-width run takes per batch, or none: an explicit `--workers` is the caller's
+ * call (as it is over the budget), and `ULTIMATE_TEST_SLOTS=0` turns the pool off.
+ */
+export const machineLease = (
+  capacity: number,
+  env: Env = Bun.env,
+): ((want: number) => Promise<SlotLease>) | undefined =>
+  slotsEnabled(env) ? (want) => acquireSlots({ want, capacity, dir: slotsDir(env) }) : undefined;