@ultimat3/cli 27.10.3 → 27.10.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/cli",
-  "version": "27.10.3",
+  "version": "27.10.4",
   "description": "The `x` binary: new, dev, build, verify, generate, db, mcp, doctor, deploy",
   "license": "MIT",
   "type": "module",
@@ -39,33 +39,33 @@
   },
   "dependencies": {
     "@babel/core": "^8.0.1",
-    "@ultimat3/action": "27.10.3",
-    "@ultimat3/admin": "27.10.3",
-    "@ultimat3/ai": "27.10.3",
-    "@ultimat3/auth": "27.10.3",
-    "@ultimat3/cache": "27.10.3",
-    "@ultimat3/core": "27.10.3",
-    "@ultimat3/db": "27.10.3",
-    "@ultimat3/entity": "27.10.3",
-    "@ultimat3/flags": "27.10.3",
-    "@ultimat3/http": "27.10.3",
-    "@ultimat3/i18n": "27.10.3",
-    "@ultimat3/jobs": "27.10.3",
-    "@ultimat3/mail": "27.10.3",
-    "@ultimat3/manifest": "27.10.3",
-    "@ultimat3/mcp": "27.10.3",
-    "@ultimat3/money": "27.10.3",
-    "@ultimat3/notify": "27.10.3",
-    "@ultimat3/policy": "27.10.3",
-    "@ultimat3/pwa": "27.10.3",
-    "@ultimat3/query": "27.10.3",
-    "@ultimat3/realtime": "27.10.3",
-    "@ultimat3/render": "27.10.3",
-    "@ultimat3/schema": "27.10.3",
-    "@ultimat3/seo": "27.10.3",
-    "@ultimat3/storage": "27.10.3",
-    "@ultimat3/testing": "27.10.3",
-    "@ultimat3/time": "27.10.3",
+    "@ultimat3/action": "27.10.4",
+    "@ultimat3/admin": "27.10.4",
+    "@ultimat3/ai": "27.10.4",
+    "@ultimat3/auth": "27.10.4",
+    "@ultimat3/cache": "27.10.4",
+    "@ultimat3/core": "27.10.4",
+    "@ultimat3/db": "27.10.4",
+    "@ultimat3/entity": "27.10.4",
+    "@ultimat3/flags": "27.10.4",
+    "@ultimat3/http": "27.10.4",
+    "@ultimat3/i18n": "27.10.4",
+    "@ultimat3/jobs": "27.10.4",
+    "@ultimat3/mail": "27.10.4",
+    "@ultimat3/manifest": "27.10.4",
+    "@ultimat3/mcp": "27.10.4",
+    "@ultimat3/money": "27.10.4",
+    "@ultimat3/notify": "27.10.4",
+    "@ultimat3/policy": "27.10.4",
+    "@ultimat3/pwa": "27.10.4",
+    "@ultimat3/query": "27.10.4",
+    "@ultimat3/realtime": "27.10.4",
+    "@ultimat3/render": "27.10.4",
+    "@ultimat3/schema": "27.10.4",
+    "@ultimat3/seo": "27.10.4",
+    "@ultimat3/storage": "27.10.4",
+    "@ultimat3/testing": "27.10.4",
+    "@ultimat3/time": "27.10.4",
     "babel-preset-solid": "^1.9.15"
   }
 }

package/src/verify-deadline.ts

@@ -87,12 +87,28 @@ export interface InFlightRun {
   readonly command: readonly string[];
   /** Everything it had printed before the kill; absent when it did not settle after it. */
   readonly output?: string;
+  /** How long it had been running when the deadline passed. */
+  readonly runningMs?: number;
 }
 
+/**
+ * The runs a step had already seen settle when its deadline passed. A batched step (a coverage
+ * run's slices) that finished ninety of them did not hang — it outgrew its deadline — and a
+ * finding that cannot tell the two apart sends its reader hunting a stuck file that is not there.
+ * A `type`: it rides in `Finding.meta`, which an interface is not assignable to.
+ */
+export type StepProgress = {
+  readonly runs: number;
+  /** The longest any of them took, as its runner reported it. */
+  readonly slowestMs: number;
+};
+
 /** What a step was doing when its deadline passed — `verify-stalled.ts` turns it into the finding. */
 export interface StepExpiry {
   readonly killed: readonly KilledProcess[];
   readonly inFlight: readonly InFlightRun[];
+  /** Absent is none: a step that never started a child, or a fixture that does not say. */
+  readonly finished?: StepProgress;
 }
 
 /** A killed child's pipes close with it; this is only the bound on a runner that is not `exec`. */
@@ -138,7 +154,10 @@ export function guardStep(
 ): StepGuard {
   let expired = false;
   // Keyed by the promise itself: two batches of one step may run the very same argv.
-  const inFlight = new Map<Promise<ExecResult>, readonly string[]>();
+  const inFlight = new Map<Promise<ExecResult>, Waiting>();
+  // Settled runs, counted only before the deadline: a kill settles the rest, and those are not
+  // progress.
+  let finished = { runs: 0, slowestMs: 0 };
   // Every child's pid, recorded the moment it exists and dropped when its run settles (a settled
   // pid may be reused). Windows reads no process's environment, so this is the step's only handle.
   const spawned = new Map<number, readonly string[]>();
@@ -169,9 +188,16 @@ export function guardStep(
           options.onSpawn?.(spawnedPid);
         },
       });
-      inFlight.set(run, command);
+      inFlight.set(run, { command, startedAt: performance.now() });
       try {
-        return await run;
+        const result = await run;
+        if (!expired) {
+          finished = {
+            runs: finished.runs + 1,
+            slowestMs: Math.max(finished.slowestMs, result.durationMs),
+          };
+        }
+        return result;
       } finally {
         inFlight.delete(run);
         if (pid !== undefined) spawned.delete(pid);
@@ -179,11 +205,16 @@ export function guardStep(
     },
     async expire(): Promise<StepExpiry> {
       expired = true;
+      const progress = finished.runs === 0 ? {} : { finished };
       // Before the kill: a killed child settles its runner call, which takes it off the map.
-      const waiting = [...inFlight];
+      const at = performance.now();
+      const waiting = [...inFlight].map(
+        ([run, { command, startedAt }]) =>
+          [run, command, Math.round(at - startedAt)] as const satisfies WaitingRun,
+      );
       if (windows) {
         const killed = await killTrees(spawned, platform.killTree ?? taskkillTree);
-        return { killed, inFlight: await settleAll(waiting) };
+        return { killed, inFlight: await settleAll(waiting), ...progress };
       }
       const workers = await killTagged(tag, undefined, isTestWorker);
       if (workers.length > 0) {
@@ -193,19 +224,27 @@ export function guardStep(
       const killed = [...workers, ...rest.filter((one) => workers.every((w) => w.pid !== one.pid))];
       // After it: the kill closed the child's pipes, so its call now resolves with everything it
       // had printed — for `bun test`, the crash line of each file a worker was holding.
-      return { killed, inFlight: await settleAll(waiting) };
+      return { killed, inFlight: await settleAll(waiting), ...progress };
     },
   };
 }
 
+interface Waiting {
+  readonly command: readonly string[];
+  readonly startedAt: number;
+}
+
+/** A run the deadline found in flight: its call, its argv, and how long it had been running. */
+type WaitingRun = readonly [Promise<ExecResult>, readonly string[], number];
+
 /** What each child had printed, once the kill closed its pipes — or just its argv past `SETTLE_MS`. */
-function settleAll(
-  waiting: readonly (readonly [Promise<ExecResult>, readonly string[]])[],
-): Promise<readonly InFlightRun[]> {
+function settleAll(waiting: readonly WaitingRun[]): Promise<readonly InFlightRun[]> {
   return Promise.all(
-    waiting.map(async ([run, command]): Promise<InFlightRun> => {
+    waiting.map(async ([run, command, runningMs]): Promise<InFlightRun> => {
       const raced = await raceDeadline(run.then(execOutput), SETTLE_MS).catch(() => undefined);
-      return raced === undefined || raced.timedOut ? { command } : { command, output: raced.value };
+      return raced === undefined || raced.timedOut
+        ? { command, runningMs }
+        : { command, output: raced.value, runningMs };
     }),
   );
 }

package/src/verify-stalled.ts

@@ -5,7 +5,7 @@
 import { ERROR_DOCS_URL } from '@ultimat3/core';
 import type { Finding } from './output';
 import { quoteArg } from './shell-quote';
-import type { InFlightRun, StepExpiry } from './verify-deadline';
+import type { InFlightRun, StepExpiry, StepProgress } from './verify-deadline';
 import { STEP_TIMEOUT_FIELD } from './verify-deadline';
 import type { VerifyStepName } from './verify-step';
 
@@ -16,12 +16,14 @@ import type { VerifyStepName } from './verify-step';
 export type StalledRun = {
   /** The argv without its file list: `bun test --parallel=3`. */
   readonly command: string;
-  /** Files the run was handed; 0 for a serial run, which selects by filter. */
+  /** Files the run was handed by name; 0 for a serial run that selects by filter. */
   readonly files: number;
   /** `--parallel`'s width, and 1 for a serial run. */
   readonly workers: number;
   /** The files that were still running, as bun named them. Empty when it named none. */
   readonly stuck: readonly string[];
+  /** How long it had been running when it was stopped. */
+  readonly runningMs?: number;
 };
 
 /** `X_VERIFY_STEP_TIMEOUT`'s `meta`. A `type`: an interface is not assignable to `Finding.meta`. */
@@ -30,6 +32,8 @@ export type StepTimeoutMeta = {
   readonly deadlineMs: number;
   readonly killed: readonly { readonly pid: number; readonly command: string }[];
   readonly inFlight: readonly StalledRun[];
+  /** The runs that had finished before the deadline — the step's progress. */
+  readonly finished: StepProgress;
 };
 
 /**
@@ -59,15 +63,18 @@ export function stalledRun(run: InFlightRun): StalledRun | undefined {
   const width = rest.find((arg) => arg.startsWith(PARALLEL));
   const flags = rest.filter((arg) => arg.startsWith('--') && !arg.startsWith('--timings'));
   const command = ['bun', 'test', ...flags].join(' ');
+  const age = run.runningMs === undefined ? {} : { runningMs: run.runningMs };
   if (width === undefined) {
     const last = lines.findLastIndex((line) => FILE_LINE.test(line));
     const started = last < 0 ? undefined : FILE_LINE.exec(lines[last] ?? '')?.[1];
     const failed = lines.slice(last + 1).some((line) => line.startsWith('(fail)'));
     return {
       command,
-      files: 0,
+      // A coverage slice names its files; a serial suite selects by a filter that is not a path.
+      files: rest.filter((arg) => TEST_FILE.test(arg)).length,
       workers: 1,
       stuck: started === undefined || failed ? [] : [started],
+      ...age,
     };
   }
   const files = rest.filter((arg) => !arg.startsWith('--'));
@@ -81,9 +88,32 @@ export function stalledRun(run: InFlightRun): StalledRun | undefined {
     files: files.length,
     workers: Math.max(1, Number(width.slice(PARALLEL.length)) || 1),
     stuck: [...new Set(crashed)],
+    ...age,
   };
 }
 
+const TEST_FILE = /\.test\.[cm]?[jt]sx?$/;
+
+/**
+ * A batched step that had finished runs, none of those still in flight older than twice the
+ * slowest that finished, was making progress when its deadline passed: it outgrew the deadline
+ * at this width, and no file is stuck. Anything older than that is read as the hang it may be.
+ */
+const PROGRESS_SLACK = 2;
+
+export function outgrewDeadline(runs: readonly StalledRun[], expiry: StepExpiry): boolean {
+  const finished = expiry.finished;
+  if (finished === undefined || finished.runs === 0) return false;
+  if (runs.some((run) => run.stuck.length > 0)) return false;
+  const bound = finished.slowestMs * PROGRESS_SLACK;
+  return expiry.inFlight.every((run) => run.runningMs !== undefined && run.runningMs <= bound);
+}
+
+const seconds = (ms: number): string => String(Math.round(ms / 1000));
+
+const oldest = (expiry: StepExpiry): number =>
+  Math.max(0, ...expiry.inFlight.map((run) => run.runningMs ?? 0));
+
 /** How many paths a `cause:` spells out — the rest are in `meta`. */
 const NAMED_IN_CAUSE = 5;
 
@@ -99,6 +129,10 @@ export function stalledOutput(expiry: StepExpiry): string | undefined {
 }
 
 const inFlightClause = (runs: readonly StalledRun[], expiry: StepExpiry): string => {
+  if (outgrewDeadline(runs, expiry)) {
+    const finished = expiry.finished as StepProgress;
+    return `; it was not hung: ${String(finished.runs)} run(s) it started had finished (the slowest in ${seconds(finished.slowestMs)} s) and the longest still running had been going ${seconds(oldest(expiry))} s — the step needs longer than its deadline at this width`;
+  }
   const files = runs.flatMap((run) => run.stuck);
   const named = runs.find((run) => run.stuck.length > 0);
   if (named !== undefined) {
@@ -110,7 +144,7 @@ const inFlightClause = (runs: readonly StalledRun[], expiry: StepExpiry): string
   if (waiting === undefined) return '';
   return runs[0] === undefined
     ? `; it was waiting on: ${waiting.command.slice(0, 4).join(' ')}`
-    : `; its running "${runs[0].command}" named no file still in flight, so the run itself did not exit`;
+    : `; its running "${runs[0].command}" named no file still in flight, so the run itself did not exit${waiting.runningMs === undefined ? '' : ` after ${seconds(waiting.runningMs)} s`}`;
 };
 
 /**
@@ -137,12 +171,15 @@ export const stepTimeoutFinding = (
     deadlineMs: ms,
     killed: expiry.killed.map((one) => ({ pid: one.pid, command: one.command })),
     inFlight: runs,
+    finished: expiry.finished ?? { runs: 0, slowestMs: 0 },
   };
+  const wider = `${command} --only ${step} --workers <n> --json`;
   return {
     code: 'X_VERIFY_STEP_TIMEOUT',
     cause: `step "${step}" did not finish within its ${String(ms)} ms deadline, so it was stopped and ${String(expiry.killed.length)} process(es) it had started were killed${inFlightClause(runs, expiry)}`,
-    fix:
-      stuck.length > 0
+    fix: outgrewDeadline(runs, expiry)
+      ? `${wider}   # it outgrew its deadline at this width, so run it wider; or raise the deadline in x.verify.json — add "${STEP_TIMEOUT_FIELD}": { "${step}": <milliseconds> } above the ${String(ms)} it was given`
+      : stuck.length > 0
         ? `${alone}   # what had not finished, alone; the whole step: ${rerun}; ${longer}`
         : `${rerun}   # reproduce it alone; ${longer}`,
     docs: ERROR_DOCS_URL,