@ultimat3/cli 27.10.3 → 27.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/cli",
-  "version": "27.10.3",
+  "version": "27.11.0",
   "description": "The `x` binary: new, dev, build, verify, generate, db, mcp, doctor, deploy",
   "license": "MIT",
   "type": "module",
@@ -39,33 +39,33 @@
   },
   "dependencies": {
     "@babel/core": "^8.0.1",
-    "@ultimat3/action": "27.10.3",
-    "@ultimat3/admin": "27.10.3",
-    "@ultimat3/ai": "27.10.3",
-    "@ultimat3/auth": "27.10.3",
-    "@ultimat3/cache": "27.10.3",
-    "@ultimat3/core": "27.10.3",
-    "@ultimat3/db": "27.10.3",
-    "@ultimat3/entity": "27.10.3",
-    "@ultimat3/flags": "27.10.3",
-    "@ultimat3/http": "27.10.3",
-    "@ultimat3/i18n": "27.10.3",
-    "@ultimat3/jobs": "27.10.3",
-    "@ultimat3/mail": "27.10.3",
-    "@ultimat3/manifest": "27.10.3",
-    "@ultimat3/mcp": "27.10.3",
-    "@ultimat3/money": "27.10.3",
-    "@ultimat3/notify": "27.10.3",
-    "@ultimat3/policy": "27.10.3",
-    "@ultimat3/pwa": "27.10.3",
-    "@ultimat3/query": "27.10.3",
-    "@ultimat3/realtime": "27.10.3",
-    "@ultimat3/render": "27.10.3",
-    "@ultimat3/schema": "27.10.3",
-    "@ultimat3/seo": "27.10.3",
-    "@ultimat3/storage": "27.10.3",
-    "@ultimat3/testing": "27.10.3",
-    "@ultimat3/time": "27.10.3",
+    "@ultimat3/action": "27.11.0",
+    "@ultimat3/admin": "27.11.0",
+    "@ultimat3/ai": "27.11.0",
+    "@ultimat3/auth": "27.11.0",
+    "@ultimat3/cache": "27.11.0",
+    "@ultimat3/core": "27.11.0",
+    "@ultimat3/db": "27.11.0",
+    "@ultimat3/entity": "27.11.0",
+    "@ultimat3/flags": "27.11.0",
+    "@ultimat3/http": "27.11.0",
+    "@ultimat3/i18n": "27.11.0",
+    "@ultimat3/jobs": "27.11.0",
+    "@ultimat3/mail": "27.11.0",
+    "@ultimat3/manifest": "27.11.0",
+    "@ultimat3/mcp": "27.11.0",
+    "@ultimat3/money": "27.11.0",
+    "@ultimat3/notify": "27.11.0",
+    "@ultimat3/policy": "27.11.0",
+    "@ultimat3/pwa": "27.11.0",
+    "@ultimat3/query": "27.11.0",
+    "@ultimat3/realtime": "27.11.0",
+    "@ultimat3/render": "27.11.0",
+    "@ultimat3/schema": "27.11.0",
+    "@ultimat3/seo": "27.11.0",
+    "@ultimat3/storage": "27.11.0",
+    "@ultimat3/testing": "27.11.0",
+    "@ultimat3/time": "27.11.0",
     "babel-preset-solid": "^1.9.15"
   }
 }

package/src/framework-schema-apply.ts

@@ -1,7 +1,8 @@
-// The two ways a boot meets the framework schema (plan 101, s1-con #5): `applyLockedSchema` runs the
-// DDL in ONE transaction behind the migration advisory lock with a bounded `lock_timeout`, then
+// The three ways a boot meets the framework schema (plan 101, s1-con #5): `applyLockedSchema` runs
+// the DDL in ONE transaction behind the migration advisory lock with a bounded `lock_timeout`, then
 // stamps the build; `verifySchema` runs no DDL and refuses a database this build was never applied
-// to. Which boot takes which is `runtime-queue.ts`'s `SchemaMode`.
+// to; `ensureSchema` reads the stamp and applies only when this build is absent from it. Which boot
+// takes which is `runtime-queue.ts`'s `SchemaMode`.
 
 import { frameworkVersion, logger } from '@ultimat3/core';
 import type { DbClient } from '@ultimat3/db';
@@ -70,9 +71,39 @@ export async function verifySchema(client: DbClient): Promise<void> {
   const mine = thisBuild();
   const verdict = stampVerdict(parseStamp(await readStamp(client)), mine);
   if (!verdict.applied) throw new FrameworkSchemaUnappliedError({ mine, newest: verdict.newest });
-  if (verdict.skew === undefined) return;
+  reportSkew(mine, verdict.skew);
+}
+
+/**
+ * A CLI command's boot — `x db backfill`, `x jobs`, `x auth`: no DDL when this build's stamp is
+ * already there, the locked apply when it is not.
+ *
+ * The stamp is read FIRST because re-applying idempotent DDL is not idempotent for every role.
+ * Postgres checks ownership before existence, so `create index if not exists` on a table the role
+ * does not own is `42501 must be owner` even when the index is there. A production runtime role
+ * holds DML only, and the PostSync backfill Job ran as it: every deploy's `x db backfill --all
+ * --write` died in the jobs entry and no sweep was ever enqueued. When the stamp is absent the
+ * apply still runs, so a fresh dev database needs no separate step — and a role that cannot apply
+ * it is refused with `X_FRAMEWORK_SCHEMA_FAILED`, whose fix is the migrate step.
+ */
+export async function ensureSchema(client: DbClient): Promise<void> {
+  const mine = thisBuild();
+  const verdict = stampVerdict(parseStamp(await readStamp(client)), mine);
+  if (!verdict.applied) {
+    await applyLockedSchema(client);
+    return;
+  }
+  reportSkew(mine, verdict.skew);
+}
+
+/**
+ * A fleet straddling a major is reported once and served — refusing it would crash-loop the old
+ * pods of every rolling deploy across a breaking release.
+ */
+function reportSkew(mine: StampEntry, skew: StampEntry | undefined): void {
+  if (skew === undefined) return;
   logger.warn('ultimate framework major skew', {
-    cause: `this process runs ${mine.version} and the database's framework schema was last applied by ${verdict.skew.version}: a fleet across a major boundary is mid-rollout or half rolled back`,
+    cause: `this process runs ${mine.version} and the database's framework schema was last applied by ${skew.version}: a fleet across a major boundary is mid-rollout or half rolled back`,
     fix: 'kubectl rollout status deployment --selector app.kubernetes.io/component=web',
   });
 }

package/src/framework-schema.ts

@@ -1,7 +1,7 @@
 // The framework's OWN tables, as one table: which package declares each, which relations it
 // creates, and the DDL. One list, read by the one applier, so "installed in dev and not in
-// production" is not a state this framework can be in — `startQueue` is on every boot path there
-// is, `ROLE=migrate` included.
+// production" is not a state this framework can be in: `ROLE=migrate` applies it and stamps the
+// build, and every other boot path meets it through `startQueue`'s `SchemaMode`.
 //
 // A framework table an app has to install by hand is a table that will be missing in production on
 // the one code path that needs it, and it surfaces as a Postgres `42P01` from inside a worker.

package/src/hold.ts

@@ -3,7 +3,15 @@
 // before the exit code. Ctrl-C then takes core's own three-phase drain (stop accepting, finish
 // in-flight, close) instead of killing a query mid-round-trip.
 
-import { drain, installSignalHandlers, logger, onShutdown, systemClock } from '@ultimat3/core';
+import {
+  drain,
+  installSignalHandlers,
+  isDraining,
+  isUltimateError,
+  logger,
+  onShutdown,
+  systemClock,
+} from '@ultimat3/core';
 
 export interface HoldOptions {
   /**
@@ -129,7 +137,24 @@ export async function holdWhileBooting<T extends { stop(): Promise<void> }>(
   try {
     app = await boot();
   } catch (error) {
-    armed.disarm();
+    // A boot that failed BECAUSE a drain began under it — a rollout superseding a pod still
+    // booting, whose server then met `markReady()`'s `X_LIFECYCLE_DRAINED` — is a pod doing what
+    // it was told: finish the drain, say so once at info, exit 0. It printed a stack trace and
+    // exited non-zero, twice a week in production, paging on a normal rollout. A failure with no
+    // drain under way is still the boot's own refusal, rethrown untouched.
+    if (isDraining() && options.exit !== undefined) {
+      await drain();
+      armed.disarm();
+      logger.info('ultimate boot interrupted by shutdown', {
+        name,
+        code: isUltimateError(error) ? error.code : 'uncoded',
+      });
+      options.exit(0);
+    } else {
+      armed.disarm();
+    }
+    // Unreachable past a real `process.exit`; a test's `exit` spy returns, and the caller still
+    // sees the boot's error rather than an app that never booted.
     throw error;
   }
   booted = app;

package/src/runtime-cache.ts

@@ -23,6 +23,7 @@ import type { AppConfig, CacheTierName } from '@ultimat3/core';
 import {
   backoffDelay,
   defineConfig,
+  finiteCount,
   isOutageMilestone,
   logger,
   renderThrowable,
@@ -63,8 +64,23 @@ export interface CacheTiersOptions {
   readonly tiers: readonly CacheTierName[];
   /** The wait before re-subscribing after `attempt` failures (1-based). Injected by a test. */
   readonly subscribeRetryMs?: ((attempt: number) => number) | undefined;
+  /**
+   * How long the boot subscribe may be refused before a refusal is worth a warning. Default
+   * `FIRST_DIAL_GRACE_MS`; a test of an OUTAGE passes 0.
+   */
+  readonly firstDialGraceMs?: number | undefined;
 }
 
+/**
+ * The boot subscribe races the bus's FIRST dial, and loses it on almost every pod: in production
+ * every pod warned on attempts 1–2 and was subscribed within 1.3 s — 345 lines in three weeks of
+ * alerting noise. Refusals inside this window are not reported; a bus still refusing after it is
+ * an outage and warns as before (attempt milestones, then `recovered` at info). 10 s is several
+ * times the slowest first dial measured, and far under any window in which a missed peer bust
+ * matters: the process is flushed (`flushDeaf`) when the subscribe lands either way.
+ */
+export const FIRST_DIAL_GRACE_MS = 10_000;
+
 /** 1 s doubling to 30 s, full jitter: a bus that is down is retried without a fleet in lockstep. */
 const defaultSubscribeRetryMs = (attempt: number): number =>
   backoffDelay({ attempt, base: 1_000, max: 30_000, jitter: 'full' });
@@ -163,6 +179,13 @@ function warnUnnamed(options: CacheTiersOptions): void {
  * suppresses it, and `emit` is not a public parameter.
  */
 export function startCacheTiers(options: CacheTiersOptions): () => Promise<void> {
+  // Every option checked before anything is registered: a refusal after `registerTier` or
+  // `registerInvalidationBroadcast` would leave process-global state installed with no release.
+  const graceMs = finiteCount(
+    'startCacheTiers',
+    'firstDialGraceMs',
+    options.firstDialGraceMs ?? FIRST_DIAL_GRACE_MS,
+  );
   // Built before ANY of them is registered: a refusal halfway through a list would leave the
   // process-global registry holding the rungs that came first, with no release returned to drop
   // them. Registration order does not decide read order — `sortTiers` does, by `CACHE_TIERS`.
@@ -276,6 +299,9 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
   const flushDeaf = (source: string): void => {
     void flushProcessTiers(source);
   };
+  const bootedAt = performance.now();
+  /** Whether an operator has been told of this outage — what makes `recovered` worth a line. */
+  let warned = false;
   const subscribing = (async (): Promise<TransportSubscription | undefined> => {
     for (let attempt = 1; !stopped; attempt += 1) {
       try {
@@ -287,15 +313,22 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
         );
         subscribed = true;
         if (attempt > 1) {
-          // The outage's one closing line, beside the thinned warnings it ends.
-          logger.info('cache.broadcast.subscribe-failed recovered', { after: attempt - 1 });
+          // The outage's one closing line, beside the thinned warnings it ends — and only then: a
+          // first dial that landed inside its window had no outage to close.
+          if (warned) {
+            logger.info('cache.broadcast.subscribe-failed recovered', { after: attempt - 1 });
+          }
           flushDeaf('cache.bus-subscribed');
         }
         return subscription;
       } catch (error) {
         // Attempts 1, 2, 4, 8, …: the dial's own thinning (`NatsTransport`), so an outage is not
         // one line per retry per pod for as long as it lasts.
-        if (isOutageMilestone(attempt)) {
+        // Silent inside the first dial's window. Past it, the FIRST refusal warns whatever its
+        // attempt number, then the milestones thin the rest.
+        const inGrace = performance.now() - bootedAt < graceMs;
+        if (!inGrace && (!warned || isOutageMilestone(attempt))) {
+          warned = true;
           logger.warn('cache.broadcast.subscribe-failed', {
             error: broadcastErrorText(error),
             attempt,

package/src/runtime-jobs-retention.ts

@@ -0,0 +1,69 @@
+// `jobs.doneRetentionMs`, `jobs.deadRetentionMs` and `jobs.outboxRetentionMs`, out of the config
+// `startServices` loads once (`app-config-load.ts`), as the windows the `x_jobs` and `x_outbox`
+// purge targets are built with. The `notify` windows' sibling (`runtime-notify-retention.ts`), with
+// one difference: these DEFAULT, because a finished job is bookkeeping and an unread message is not.
+
+import type { AppConfig, PgExecutor } from '@ultimat3/core';
+import {
+  finiteCount,
+  JOBS_DEAD_RETENTION_MS_DEFAULT,
+  JOBS_DONE_RETENTION_MS_DEFAULT,
+  JOBS_OUTBOX_RETENTION_MS_DEFAULT,
+} from '@ultimat3/core';
+import type { PurgeTarget } from '@ultimat3/jobs';
+import { jobsPurgeTarget, outboxPurgeTarget } from '@ultimat3/jobs';
+
+export interface JobsRetention {
+  readonly doneRetentionMs: number;
+  readonly deadRetentionMs: number;
+  readonly outboxRetentionMs: number;
+}
+
+const SUBJECT = 'jobsRetentionOf';
+
+/**
+ * The app's windows, or core's defaults for a boot with no config file or a hand-built
+ * `AppConfig` that left a key out (`JobsConfig` declares the three optional) — the same constants
+ * `defineConfig` fills in, so an app without `app.config.ts` keeps finished jobs exactly as long as
+ * one that never wrote the keys. Core's validator refuses a bad window in a loaded file; this
+ * screen is the layered form for the config that never met it.
+ */
+export function jobsRetentionOf(config: AppConfig | undefined): JobsRetention {
+  const jobs = config?.jobs;
+  return {
+    doneRetentionMs: finiteCount(
+      SUBJECT,
+      'jobs.doneRetentionMs',
+      jobs?.doneRetentionMs ?? JOBS_DONE_RETENTION_MS_DEFAULT,
+      1,
+    ),
+    deadRetentionMs: finiteCount(
+      SUBJECT,
+      'jobs.deadRetentionMs',
+      jobs?.deadRetentionMs ?? JOBS_DEAD_RETENTION_MS_DEFAULT,
+      1,
+    ),
+    outboxRetentionMs: finiteCount(
+      SUBJECT,
+      'jobs.outboxRetentionMs',
+      jobs?.outboxRetentionMs ?? JOBS_OUTBOX_RETENTION_MS_DEFAULT,
+      1,
+    ),
+  };
+}
+
+/** The two queue-table targets `installRetentionSweep` takes, over the boot's own executor. */
+export function jobsRetentionTargets(
+  executor: PgExecutor,
+  config: AppConfig | undefined,
+): { readonly jobs: PurgeTarget; readonly outbox: PurgeTarget } {
+  const windows = jobsRetentionOf(config);
+  return {
+    jobs: jobsPurgeTarget({
+      executor,
+      doneRetentionMs: windows.doneRetentionMs,
+      deadRetentionMs: windows.deadRetentionMs,
+    }),
+    outbox: outboxPurgeTarget({ executor, publishedRetentionMs: windows.outboxRetentionMs }),
+  };
+}

package/src/runtime-purge.ts

@@ -1,11 +1,11 @@
 // The retention sweep this boot owns: the framework tables that grow with traffic, the `purge()`
 // job that empties them and the `task` that fires it hourly.
 //
-// SEVEN TARGETS over seven tables, `As of 2026-10-02` — `x_idempotency`, `x_rate_limit`, the
-// `x_auth` limiter's table, `x_notify_deliveries`, `x_notify_inbox`, `x_notify_digests` and
-// `x_job_events`. Counted nowhere in prose but here: this header said "three" for two releases
-// after the notify tables joined the boot's DDL, which is how `x_notify_inbox` became the one
-// framework table nothing swept.
+// NINE TARGETS, `As of 2026-10-17` — `x_idempotency`, `x_rate_limit`, the `x_auth` limiter's
+// table, `x_notify_deliveries`, `x_notify_inbox`, `x_notify_digests`, `x_job_events`, `x_outbox`
+// (published rows) and `x_jobs` (finished rows, with their `x_job_steps`). Counted nowhere in prose but here: this header said
+// "three" for two releases after the notify tables joined the boot's DDL, which is how
+// `x_notify_inbox` became the one framework table nothing swept — and `x_jobs` was the next one.
 //
 // WHY here and not in the packages that own the tables: `postgresIdempotencyStore` (tier 3),
 // `postgresRateLimitStore` (tier 2) and `postgresAuthLimiter` (tier 2) cannot see each other and
@@ -58,6 +58,15 @@ export interface RetentionStores {
    * growing for the life of the deployment — which it did until this member existed.
    */
   readonly events: EventBus;
+  /**
+   * Finished jobs — `jobsPurgeTarget` over this boot's executor, with the app's
+   * `jobs.doneRetentionMs` / `jobs.deadRetentionMs`. REQUIRED for `events`' reason: until it
+   * existed `x_jobs` and `x_job_steps` kept every job ever run, `mail.send`'s recipient and body
+   * among them.
+   */
+  readonly jobs: PurgeTarget;
+  /** Published outbox rows — `outboxPurgeTarget` at `jobs.outboxRetentionMs`. REQUIRED, as `jobs`. */
+  readonly outbox: PurgeTarget;
   /**
    * The app's own answer to "how long is an unread message kept", read off the config
    * `startServices` loads (`inboxRetentionOf`). Absent windows mean the inbox is never swept,
@@ -143,6 +152,11 @@ function retentionTargets(stores: RetentionStores): readonly PurgeTarget[] {
     ...notifyTargets(stores.inboxRetention ?? NO_INBOX_RETENTION),
     // Its own clock, and so no `nowMs`: every instant in that table is the database's.
     eventsPurgeTarget(stores.events),
+    // Last: the two widest tables, batched, so every small sweep above has run before they spend
+    // the pass. Either order is safe — the outbox sweep takes only PUBLISHED rows, and the `x_jobs`
+    // sweep vetoes only a job whose outbox row is still unpublished.
+    stores.outbox,
+    stores.jobs,
   ];
 }
 

package/src/runtime-queue.ts

@@ -35,7 +35,7 @@ import {
   setJobDriver,
   setJobsFacade,
 } from '@ultimat3/jobs';
-import { applyLockedSchema, verifySchema } from './framework-schema-apply';
+import { applyLockedSchema, ensureSchema, verifySchema } from './framework-schema-apply';
 import { appExtensions } from './migration-extensions';
 import type { DevServices } from './runtime-bindings';
 import type { RuntimeOverrides } from './runtime-overrides';
@@ -124,15 +124,20 @@ export function startDb(services: DevServices, env: ReplicaEnv): StartedDb {
  * schema — the same reason `SQL_JOBS_TABLE` is applied by the boot.
  */
 /**
- * `'apply'` — `ROLE=migrate`, `x dev` and every CLI command — runs the DDL under the migration
- * lock with a bounded `lock_timeout` and stamps the build. `'verify'` — a serving role on an
- * external database — runs none: an `alter table` queued behind one open transaction stalled every
- * enqueue in the fleet on each pod start, and racing `create table`s were a first-deploy `23505`.
+ * `'apply'` — `ROLE=migrate`, `x db migrate`, `x dev` and `x mcp serve` — runs the DDL under the
+ * migration lock with a bounded `lock_timeout` and stamps the build. `'verify'` — a serving role on
+ * an external database — runs none: an `alter table` queued behind one open transaction stalled
+ * every enqueue in the fleet on each pod start, and racing `create table`s were a first-deploy
+ * `23505`. `'ensure'` — every other CLI command, `x db backfill` and `x jobs` among them — runs
+ * none when this build's stamp is present and applies otherwise: those commands run as the
+ * runtime role in production, which owns no framework table and so cannot re-run even
+ * `if not exists` DDL (`framework-schema-apply.ts`).
  */
-export type SchemaMode = 'apply' | 'verify';
+export type SchemaMode = 'apply' | 'ensure' | 'verify';
 
 async function applySchema(client: DevDbClient, mode: SchemaMode): Promise<void> {
   if (mode === 'verify') await verifySchema(client);
+  else if (mode === 'ensure') await ensureSchema(client);
   else await applyLockedSchema(client);
 }
 /**
@@ -248,7 +253,11 @@ export async function startQueue(
    * this file reads it: see `startDb` for what two readers of one question cost.
    */
   env: ReplicaEnv = process.env,
-  schema: SchemaMode = 'apply',
+  /**
+   * `'ensure'` unless the caller IS the step that applies: a command that only reads or enqueues
+   * must not need the privileges of the migrate role (`SchemaMode`).
+   */
+  schema: SchemaMode = 'ensure',
 ): Promise<RunningQueue> {
   const { client: db, replica } = startDb(services, env);
   try {

package/src/runtime-services.ts

@@ -37,6 +37,7 @@ import type { DevServices, Env } from './runtime-bindings';
 import { selectBus } from './runtime-bus';
 import { cacheTiersOf, startCacheTiers } from './runtime-cache';
 import { type WorkerConfig, workerConfigOf } from './runtime-jobs';
+import { jobsRetentionTargets } from './runtime-jobs-retention';
 import { selectAppMailDriver } from './runtime-mail';
 import { inboxRetentionOf } from './runtime-notify-retention';
 import type { RuntimeOverrides } from './runtime-overrides';
@@ -403,6 +404,7 @@ export async function startServices(
         idempotency: queue.idempotency,
         rateLimit: rateLimitStore,
         events,
+        ...jobsRetentionTargets(executor, config),
         inboxRetention: inboxRetentionOf(config),
       }),
     );

package/src/schema-errors.ts

@@ -18,10 +18,13 @@ export class FrameworkSchemaFailedError extends UltimateError {
     super({
       code: 'X_FRAMEWORK_SCHEMA_FAILED',
       cause: `${input.pkg} could not create ${input.tables.join(', ')}: ${renderThrowable(input.cause)}`,
-      // An EDIT plus the command that CONFIRMS it, which is the house shape for a repair the gate
-      // cannot perform: the two real causes are a role without `create`, and a relation of that
-      // name already present with an incompatible shape.
-      fix: `grant create on schema public to the role in DATABASE_URL, then re-run: x db migrate`,
+      // The migrate step, as the role that owns the schema. The common cause is `42501`: a runtime
+      // role with DML only — no `create` on `public`, and not the owner of an existing framework
+      // table, which Postgres demands even of `create index if not exists`. Granting that role
+      // `create` (this line's old advice) cures the first and never the second, and widens the
+      // runtime role besides. The rarer cause, a relation of that name with another shape, is
+      // named in `cause` and reported again by the same command.
+      fix: 'x db migrate   # with DATABASE_URL naming the role that owns schema public, as ROLE=migrate runs',
       meta: { pkg: input.pkg, tables: [...input.tables] },
     });
   }

package/src/serve.ts

@@ -48,10 +48,13 @@ export type { MigratedApp, ServedApp, ServeOptions, StartedApp } from './serve-t
  * is what fails on it.
  */
 export async function runMigrations(options: ServeOptions): Promise<MigratedApp> {
+  // `'apply'`, never the stamp-first default: this IS the step that applies, and an operator
+  // re-running it is how a dropped framework table comes back.
   const queue = await startQueue(
     resolveServices(options.root, options.env),
     options.runtime,
     options.env,
+    'apply',
   );
   try {
     const migrations = await readMigrations(options.root);

package/src/verify-deadline.ts

@@ -87,12 +87,28 @@ export interface InFlightRun {
   readonly command: readonly string[];
   /** Everything it had printed before the kill; absent when it did not settle after it. */
   readonly output?: string;
+  /** How long it had been running when the deadline passed. */
+  readonly runningMs?: number;
 }
 
+/**
+ * The runs a step had already seen settle when its deadline passed. A batched step (a coverage
+ * run's slices) that finished ninety of them did not hang — it outgrew its deadline — and a
+ * finding that cannot tell the two apart sends its reader hunting a stuck file that is not there.
+ * A `type`: it rides in `Finding.meta`, which an interface is not assignable to.
+ */
+export type StepProgress = {
+  readonly runs: number;
+  /** The longest any of them took, as its runner reported it. */
+  readonly slowestMs: number;
+};
+
 /** What a step was doing when its deadline passed — `verify-stalled.ts` turns it into the finding. */
 export interface StepExpiry {
   readonly killed: readonly KilledProcess[];
   readonly inFlight: readonly InFlightRun[];
+  /** Absent is none: a step that never started a child, or a fixture that does not say. */
+  readonly finished?: StepProgress;
 }
 
 /** A killed child's pipes close with it; this is only the bound on a runner that is not `exec`. */
@@ -138,7 +154,10 @@ export function guardStep(
 ): StepGuard {
   let expired = false;
   // Keyed by the promise itself: two batches of one step may run the very same argv.
-  const inFlight = new Map<Promise<ExecResult>, readonly string[]>();
+  const inFlight = new Map<Promise<ExecResult>, Waiting>();
+  // Settled runs, counted only before the deadline: a kill settles the rest, and those are not
+  // progress.
+  let finished = { runs: 0, slowestMs: 0 };
   // Every child's pid, recorded the moment it exists and dropped when its run settles (a settled
   // pid may be reused). Windows reads no process's environment, so this is the step's only handle.
   const spawned = new Map<number, readonly string[]>();
@@ -169,9 +188,16 @@ export function guardStep(
           options.onSpawn?.(spawnedPid);
         },
       });
-      inFlight.set(run, command);
+      inFlight.set(run, { command, startedAt: performance.now() });
       try {
-        return await run;
+        const result = await run;
+        if (!expired) {
+          finished = {
+            runs: finished.runs + 1,
+            slowestMs: Math.max(finished.slowestMs, result.durationMs),
+          };
+        }
+        return result;
       } finally {
         inFlight.delete(run);
         if (pid !== undefined) spawned.delete(pid);
@@ -179,11 +205,16 @@ export function guardStep(
     },
     async expire(): Promise<StepExpiry> {
       expired = true;
+      const progress = finished.runs === 0 ? {} : { finished };
       // Before the kill: a killed child settles its runner call, which takes it off the map.
-      const waiting = [...inFlight];
+      const at = performance.now();
+      const waiting = [...inFlight].map(
+        ([run, { command, startedAt }]) =>
+          [run, command, Math.round(at - startedAt)] as const satisfies WaitingRun,
+      );
       if (windows) {
         const killed = await killTrees(spawned, platform.killTree ?? taskkillTree);
-        return { killed, inFlight: await settleAll(waiting) };
+        return { killed, inFlight: await settleAll(waiting), ...progress };
       }
       const workers = await killTagged(tag, undefined, isTestWorker);
       if (workers.length > 0) {
@@ -193,19 +224,27 @@ export function guardStep(
       const killed = [...workers, ...rest.filter((one) => workers.every((w) => w.pid !== one.pid))];
       // After it: the kill closed the child's pipes, so its call now resolves with everything it
       // had printed — for `bun test`, the crash line of each file a worker was holding.
-      return { killed, inFlight: await settleAll(waiting) };
+      return { killed, inFlight: await settleAll(waiting), ...progress };
     },
   };
 }
 
+interface Waiting {
+  readonly command: readonly string[];
+  readonly startedAt: number;
+}
+
+/** A run the deadline found in flight: its call, its argv, and how long it had been running. */
+type WaitingRun = readonly [Promise<ExecResult>, readonly string[], number];
+
 /** What each child had printed, once the kill closed its pipes — or just its argv past `SETTLE_MS`. */
-function settleAll(
-  waiting: readonly (readonly [Promise<ExecResult>, readonly string[]])[],
-): Promise<readonly InFlightRun[]> {
+function settleAll(waiting: readonly WaitingRun[]): Promise<readonly InFlightRun[]> {
   return Promise.all(
-    waiting.map(async ([run, command]): Promise<InFlightRun> => {
+    waiting.map(async ([run, command, runningMs]): Promise<InFlightRun> => {
       const raced = await raceDeadline(run.then(execOutput), SETTLE_MS).catch(() => undefined);
-      return raced === undefined || raced.timedOut ? { command } : { command, output: raced.value };
+      return raced === undefined || raced.timedOut
+        ? { command, runningMs }
+        : { command, output: raced.value, runningMs };
     }),
   );
 }

package/src/verify-stalled.ts

@@ -5,7 +5,7 @@
 import { ERROR_DOCS_URL } from '@ultimat3/core';
 import type { Finding } from './output';
 import { quoteArg } from './shell-quote';
-import type { InFlightRun, StepExpiry } from './verify-deadline';
+import type { InFlightRun, StepExpiry, StepProgress } from './verify-deadline';
 import { STEP_TIMEOUT_FIELD } from './verify-deadline';
 import type { VerifyStepName } from './verify-step';
 
@@ -16,12 +16,14 @@ import type { VerifyStepName } from './verify-step';
 export type StalledRun = {
   /** The argv without its file list: `bun test --parallel=3`. */
   readonly command: string;
-  /** Files the run was handed; 0 for a serial run, which selects by filter. */
+  /** Files the run was handed by name; 0 for a serial run that selects by filter. */
   readonly files: number;
   /** `--parallel`'s width, and 1 for a serial run. */
   readonly workers: number;
   /** The files that were still running, as bun named them. Empty when it named none. */
   readonly stuck: readonly string[];
+  /** How long it had been running when it was stopped. */
+  readonly runningMs?: number;
 };
 
 /** `X_VERIFY_STEP_TIMEOUT`'s `meta`. A `type`: an interface is not assignable to `Finding.meta`. */
@@ -30,6 +32,8 @@ export type StepTimeoutMeta = {
   readonly deadlineMs: number;
   readonly killed: readonly { readonly pid: number; readonly command: string }[];
   readonly inFlight: readonly StalledRun[];
+  /** The runs that had finished before the deadline — the step's progress. */
+  readonly finished: StepProgress;
 };
 
 /**
@@ -59,15 +63,18 @@ export function stalledRun(run: InFlightRun): StalledRun | undefined {
   const width = rest.find((arg) => arg.startsWith(PARALLEL));
   const flags = rest.filter((arg) => arg.startsWith('--') && !arg.startsWith('--timings'));
   const command = ['bun', 'test', ...flags].join(' ');
+  const age = run.runningMs === undefined ? {} : { runningMs: run.runningMs };
   if (width === undefined) {
     const last = lines.findLastIndex((line) => FILE_LINE.test(line));
     const started = last < 0 ? undefined : FILE_LINE.exec(lines[last] ?? '')?.[1];
     const failed = lines.slice(last + 1).some((line) => line.startsWith('(fail)'));
     return {
       command,
-      files: 0,
+      // A coverage slice names its files; a serial suite selects by a filter that is not a path.
+      files: rest.filter((arg) => TEST_FILE.test(arg)).length,
       workers: 1,
       stuck: started === undefined || failed ? [] : [started],
+      ...age,
     };
   }
   const files = rest.filter((arg) => !arg.startsWith('--'));
@@ -81,9 +88,32 @@ export function stalledRun(run: InFlightRun): StalledRun | undefined {
     files: files.length,
     workers: Math.max(1, Number(width.slice(PARALLEL.length)) || 1),
     stuck: [...new Set(crashed)],
+    ...age,
   };
 }
 
+const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
+
+/**
+ * A batched step that had finished runs, none of those still in flight older than twice the
+ * slowest that finished, was making progress when its deadline passed: it outgrew the deadline
+ * at this width, and no file is stuck. Anything older than that is read as the hang it may be.
+ */
+const PROGRESS_SLACK = 2;
+
+export function outgrewDeadline(runs: readonly StalledRun[], expiry: StepExpiry): boolean {
+  const finished = expiry.finished;
+  if (finished === undefined || finished.runs === 0) return false;
+  if (runs.some((run) => run.stuck.length > 0)) return false;
+  const bound = finished.slowestMs * PROGRESS_SLACK;
+  return expiry.inFlight.every((run) => run.runningMs !== undefined && run.runningMs <= bound);
+}
+
+const seconds = (ms: number): string => String(Math.round(ms / 1000));
+
+const oldest = (expiry: StepExpiry): number =>
+  Math.max(0, ...expiry.inFlight.map((run) => run.runningMs ?? 0));
+
 /** How many paths a `cause:` spells out — the rest are in `meta`. */
 const NAMED_IN_CAUSE = 5;
 
@@ -99,6 +129,10 @@ export function stalledOutput(expiry: StepExpiry): string | undefined {
 }
 
 const inFlightClause = (runs: readonly StalledRun[], expiry: StepExpiry): string => {
+  if (outgrewDeadline(runs, expiry)) {
+    const finished = expiry.finished as StepProgress;
+    return `; it was not hung: ${String(finished.runs)} run(s) it started had finished (the slowest in ${seconds(finished.slowestMs)} s) and the longest still running had been going ${seconds(oldest(expiry))} s — the step needs longer than its deadline at this width`;
+  }
   const files = runs.flatMap((run) => run.stuck);
   const named = runs.find((run) => run.stuck.length > 0);
   if (named !== undefined) {
@@ -110,7 +144,7 @@ const inFlightClause = (runs: readonly StalledRun[], expiry: StepExpiry): string
   if (waiting === undefined) return '';
   return runs[0] === undefined
     ? `; it was waiting on: ${waiting.command.slice(0, 4).join(' ')}`
-    : `; its running "${runs[0].command}" named no file still in flight, so the run itself did not exit`;
+    : `; its running "${runs[0].command}" named no file still in flight, so the run itself did not exit${waiting.runningMs === undefined ? '' : ` after ${seconds(waiting.runningMs)} s`}`;
 };
 
 /**
@@ -137,12 +171,15 @@ export const stepTimeoutFinding = (
     deadlineMs: ms,
     killed: expiry.killed.map((one) => ({ pid: one.pid, command: one.command })),
     inFlight: runs,
+    finished: expiry.finished ?? { runs: 0, slowestMs: 0 },
   };
+  const wider = `${command} --only ${step} --workers <n> --json`;
   return {
     code: 'X_VERIFY_STEP_TIMEOUT',
     cause: `step "${step}" did not finish within its ${String(ms)} ms deadline, so it was stopped and ${String(expiry.killed.length)} process(es) it had started were killed${inFlightClause(runs, expiry)}`,
-    fix:
-      stuck.length > 0
+    fix: outgrewDeadline(runs, expiry)
+      ? `${wider}   # it outgrew its deadline at this width, so run it wider; or raise the deadline in x.verify.json — add "${STEP_TIMEOUT_FIELD}": { "${step}": <milliseconds> } above the ${String(ms)} it was given`
+      : stuck.length > 0
         ? `${alone}   # what had not finished, alone; the whole step: ${rerun}; ${longer}`
         : `${rerun}   # reproduce it alone; ${longer}`,
     docs: ERROR_DOCS_URL,