@ultimat3/cli 27.10.4 → 27.11.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/cli",
-  "version": "27.10.4",
+  "version": "27.11.0",
   "description": "The `x` binary: new, dev, build, verify, generate, db, mcp, doctor, deploy",
   "license": "MIT",
   "type": "module",
@@ -39,33 +39,33 @@
   },
   "dependencies": {
     "@babel/core": "^8.0.1",
-    "@ultimat3/action": "27.10.4",
-    "@ultimat3/admin": "27.10.4",
-    "@ultimat3/ai": "27.10.4",
-    "@ultimat3/auth": "27.10.4",
-    "@ultimat3/cache": "27.10.4",
-    "@ultimat3/core": "27.10.4",
-    "@ultimat3/db": "27.10.4",
-    "@ultimat3/entity": "27.10.4",
-    "@ultimat3/flags": "27.10.4",
-    "@ultimat3/http": "27.10.4",
-    "@ultimat3/i18n": "27.10.4",
-    "@ultimat3/jobs": "27.10.4",
-    "@ultimat3/mail": "27.10.4",
-    "@ultimat3/manifest": "27.10.4",
-    "@ultimat3/mcp": "27.10.4",
-    "@ultimat3/money": "27.10.4",
-    "@ultimat3/notify": "27.10.4",
-    "@ultimat3/policy": "27.10.4",
-    "@ultimat3/pwa": "27.10.4",
-    "@ultimat3/query": "27.10.4",
-    "@ultimat3/realtime": "27.10.4",
-    "@ultimat3/render": "27.10.4",
-    "@ultimat3/schema": "27.10.4",
-    "@ultimat3/seo": "27.10.4",
-    "@ultimat3/storage": "27.10.4",
-    "@ultimat3/testing": "27.10.4",
-    "@ultimat3/time": "27.10.4",
+    "@ultimat3/action": "27.11.0",
+    "@ultimat3/admin": "27.11.0",
+    "@ultimat3/ai": "27.11.0",
+    "@ultimat3/auth": "27.11.0",
+    "@ultimat3/cache": "27.11.0",
+    "@ultimat3/core": "27.11.0",
+    "@ultimat3/db": "27.11.0",
+    "@ultimat3/entity": "27.11.0",
+    "@ultimat3/flags": "27.11.0",
+    "@ultimat3/http": "27.11.0",
+    "@ultimat3/i18n": "27.11.0",
+    "@ultimat3/jobs": "27.11.0",
+    "@ultimat3/mail": "27.11.0",
+    "@ultimat3/manifest": "27.11.0",
+    "@ultimat3/mcp": "27.11.0",
+    "@ultimat3/money": "27.11.0",
+    "@ultimat3/notify": "27.11.0",
+    "@ultimat3/policy": "27.11.0",
+    "@ultimat3/pwa": "27.11.0",
+    "@ultimat3/query": "27.11.0",
+    "@ultimat3/realtime": "27.11.0",
+    "@ultimat3/render": "27.11.0",
+    "@ultimat3/schema": "27.11.0",
+    "@ultimat3/seo": "27.11.0",
+    "@ultimat3/storage": "27.11.0",
+    "@ultimat3/testing": "27.11.0",
+    "@ultimat3/time": "27.11.0",
     "babel-preset-solid": "^1.9.15"
   }
 }

package/src/framework-schema-apply.ts

@@ -1,7 +1,8 @@
-// The two ways a boot meets the framework schema (plan 101, s1-con #5): `applyLockedSchema` runs the
-// DDL in ONE transaction behind the migration advisory lock with a bounded `lock_timeout`, then
+// The three ways a boot meets the framework schema (plan 101, s1-con #5): `applyLockedSchema` runs
+// the DDL in ONE transaction behind the migration advisory lock with a bounded `lock_timeout`, then
 // stamps the build; `verifySchema` runs no DDL and refuses a database this build was never applied
-// to. Which boot takes which is `runtime-queue.ts`'s `SchemaMode`.
+// to; `ensureSchema` reads the stamp and applies only when this build is absent from it. Which boot
+// takes which is `runtime-queue.ts`'s `SchemaMode`.
 
 import { frameworkVersion, logger } from '@ultimat3/core';
 import type { DbClient } from '@ultimat3/db';
@@ -70,9 +71,39 @@ export async function verifySchema(client: DbClient): Promise<void> {
   const mine = thisBuild();
   const verdict = stampVerdict(parseStamp(await readStamp(client)), mine);
   if (!verdict.applied) throw new FrameworkSchemaUnappliedError({ mine, newest: verdict.newest });
-  if (verdict.skew === undefined) return;
+  reportSkew(mine, verdict.skew);
+}
+
+/**
+ * A CLI command's boot — `x db backfill`, `x jobs`, `x auth`: no DDL when this build's stamp is
+ * already there, the locked apply when it is not.
+ *
+ * The stamp is read FIRST because re-applying idempotent DDL is not idempotent for every role.
+ * Postgres checks ownership before existence, so `create index if not exists` on a table the role
+ * does not own is `42501 must be owner` even when the index is there. A production runtime role
+ * holds DML only, and the PostSync backfill Job ran as it: every deploy's `x db backfill --all
+ * --write` died in the jobs entry and no sweep was ever enqueued. When the stamp is absent the
+ * apply still runs, so a fresh dev database needs no separate step — and a role that cannot apply
+ * it is refused with `X_FRAMEWORK_SCHEMA_FAILED`, whose fix is the migrate step.
+ */
+export async function ensureSchema(client: DbClient): Promise<void> {
+  const mine = thisBuild();
+  const verdict = stampVerdict(parseStamp(await readStamp(client)), mine);
+  if (!verdict.applied) {
+    await applyLockedSchema(client);
+    return;
+  }
+  reportSkew(mine, verdict.skew);
+}
+
+/**
+ * A fleet straddling a major is reported once and served — refusing it would crash-loop the old
+ * pods of every rolling deploy across a breaking release.
+ */
+function reportSkew(mine: StampEntry, skew: StampEntry | undefined): void {
+  if (skew === undefined) return;
   logger.warn('ultimate framework major skew', {
-    cause: `this process runs ${mine.version} and the database's framework schema was last applied by ${verdict.skew.version}: a fleet across a major boundary is mid-rollout or half rolled back`,
+    cause: `this process runs ${mine.version} and the database's framework schema was last applied by ${skew.version}: a fleet across a major boundary is mid-rollout or half rolled back`,
     fix: 'kubectl rollout status deployment --selector app.kubernetes.io/component=web',
   });
 }

package/src/framework-schema.ts

@@ -1,7 +1,7 @@
 // The framework's OWN tables, as one table: which package declares each, which relations it
 // creates, and the DDL. One list, read by the one applier, so "installed in dev and not in
-// production" is not a state this framework can be in — `startQueue` is on every boot path there
-// is, `ROLE=migrate` included.
+// production" is not a state this framework can be in: `ROLE=migrate` applies it and stamps the
+// build, and every other boot path meets it through `startQueue`'s `SchemaMode`.
 //
 // A framework table an app has to install by hand is a table that will be missing in production on
 // the one code path that needs it, and it surfaces as a Postgres `42P01` from inside a worker.

package/src/hold.ts

@@ -3,7 +3,15 @@
 // before the exit code. Ctrl-C then takes core's own three-phase drain (stop accepting, finish
 // in-flight, close) instead of killing a query mid-round-trip.
 
-import { drain, installSignalHandlers, logger, onShutdown, systemClock } from '@ultimat3/core';
+import {
+  drain,
+  installSignalHandlers,
+  isDraining,
+  isUltimateError,
+  logger,
+  onShutdown,
+  systemClock,
+} from '@ultimat3/core';
 
 export interface HoldOptions {
   /**
@@ -129,7 +137,24 @@ export async function holdWhileBooting<T extends { stop(): Promise<void> }>(
   try {
     app = await boot();
   } catch (error) {
-    armed.disarm();
+    // A boot that failed BECAUSE a drain began under it — a rollout superseding a pod still
+    // booting, whose server then met `markReady()`'s `X_LIFECYCLE_DRAINED` — is a pod doing what
+    // it was told: finish the drain, say so once at info, exit 0. It printed a stack trace and
+    // exited non-zero, twice a week in production, paging on a normal rollout. A failure with no
+    // drain under way is still the boot's own refusal, rethrown untouched.
+    if (isDraining() && options.exit !== undefined) {
+      await drain();
+      armed.disarm();
+      logger.info('ultimate boot interrupted by shutdown', {
+        name,
+        code: isUltimateError(error) ? error.code : 'uncoded',
+      });
+      options.exit(0);
+    } else {
+      armed.disarm();
+    }
+    // Unreachable past a real `process.exit`; a test's `exit` spy returns, and the caller still
+    // sees the boot's error rather than an app that never booted.
     throw error;
   }
   booted = app;

package/src/runtime-cache.ts

@@ -23,6 +23,7 @@ import type { AppConfig, CacheTierName } from '@ultimat3/core';
 import {
   backoffDelay,
   defineConfig,
+  finiteCount,
   isOutageMilestone,
   logger,
   renderThrowable,
@@ -63,8 +64,23 @@ export interface CacheTiersOptions {
   readonly tiers: readonly CacheTierName[];
   /** The wait before re-subscribing after `attempt` failures (1-based). Injected by a test. */
   readonly subscribeRetryMs?: ((attempt: number) => number) | undefined;
+  /**
+   * How long the boot subscribe may be refused before a refusal is worth a warning. Default
+   * `FIRST_DIAL_GRACE_MS`; a test of an OUTAGE passes 0.
+   */
+  readonly firstDialGraceMs?: number | undefined;
 }
 
+/**
+ * The boot subscribe races the bus's FIRST dial, and loses it on almost every pod: in production
+ * every pod warned on attempts 1–2 and was subscribed within 1.3 s — 345 lines in three weeks of
+ * alerting noise. Refusals inside this window are not reported; a bus still refusing after it is
+ * an outage and warns as before (attempt milestones, then `recovered` at info). 10 s is several
+ * times the slowest first dial measured, and far under any window in which a missed peer bust
+ * matters: the process is flushed (`flushDeaf`) when the subscribe lands either way.
+ */
+export const FIRST_DIAL_GRACE_MS = 10_000;
+
 /** 1 s doubling to 30 s, full jitter: a bus that is down is retried without a fleet in lockstep. */
 const defaultSubscribeRetryMs = (attempt: number): number =>
   backoffDelay({ attempt, base: 1_000, max: 30_000, jitter: 'full' });
@@ -163,6 +179,13 @@ function warnUnnamed(options: CacheTiersOptions): void {
  * suppresses it, and `emit` is not a public parameter.
  */
 export function startCacheTiers(options: CacheTiersOptions): () => Promise<void> {
+  // Every option checked before anything is registered: a refusal after `registerTier` or
+  // `registerInvalidationBroadcast` would leave process-global state installed with no release.
+  const graceMs = finiteCount(
+    'startCacheTiers',
+    'firstDialGraceMs',
+    options.firstDialGraceMs ?? FIRST_DIAL_GRACE_MS,
+  );
   // Built before ANY of them is registered: a refusal halfway through a list would leave the
   // process-global registry holding the rungs that came first, with no release returned to drop
   // them. Registration order does not decide read order — `sortTiers` does, by `CACHE_TIERS`.
@@ -276,6 +299,9 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
   const flushDeaf = (source: string): void => {
     void flushProcessTiers(source);
   };
+  const bootedAt = performance.now();
+  /** Whether an operator has been told of this outage — what makes `recovered` worth a line. */
+  let warned = false;
   const subscribing = (async (): Promise<TransportSubscription | undefined> => {
     for (let attempt = 1; !stopped; attempt += 1) {
       try {
@@ -287,15 +313,22 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
         );
         subscribed = true;
         if (attempt > 1) {
-          // The outage's one closing line, beside the thinned warnings it ends.
-          logger.info('cache.broadcast.subscribe-failed recovered', { after: attempt - 1 });
+          // The outage's one closing line, beside the thinned warnings it ends — and only then: a
+          // first dial that landed inside its window had no outage to close.
+          if (warned) {
+            logger.info('cache.broadcast.subscribe-failed recovered', { after: attempt - 1 });
+          }
           flushDeaf('cache.bus-subscribed');
         }
         return subscription;
       } catch (error) {
         // Attempts 1, 2, 4, 8, …: the dial's own thinning (`NatsTransport`), so an outage is not
         // one line per retry per pod for as long as it lasts.
-        if (isOutageMilestone(attempt)) {
+        // Silent inside the first dial's window. Past it, the FIRST refusal warns whatever its
+        // attempt number, then the milestones thin the rest.
+        const inGrace = performance.now() - bootedAt < graceMs;
+        if (!inGrace && (!warned || isOutageMilestone(attempt))) {
+          warned = true;
           logger.warn('cache.broadcast.subscribe-failed', {
             error: broadcastErrorText(error),
             attempt,

package/src/runtime-jobs-retention.ts

@@ -0,0 +1,69 @@
+// `jobs.doneRetentionMs`, `jobs.deadRetentionMs` and `jobs.outboxRetentionMs`, out of the config
+// `startServices` loads once (`app-config-load.ts`), as the windows the `x_jobs` and `x_outbox`
+// purge targets are built with. The `notify` windows' sibling (`runtime-notify-retention.ts`), with
+// one difference: these DEFAULT, because a finished job is bookkeeping and an unread message is not.
+
+import type { AppConfig, PgExecutor } from '@ultimat3/core';
+import {
+  finiteCount,
+  JOBS_DEAD_RETENTION_MS_DEFAULT,
+  JOBS_DONE_RETENTION_MS_DEFAULT,
+  JOBS_OUTBOX_RETENTION_MS_DEFAULT,
+} from '@ultimat3/core';
+import type { PurgeTarget } from '@ultimat3/jobs';
+import { jobsPurgeTarget, outboxPurgeTarget } from '@ultimat3/jobs';
+
+export interface JobsRetention {
+  readonly doneRetentionMs: number;
+  readonly deadRetentionMs: number;
+  readonly outboxRetentionMs: number;
+}
+
+const SUBJECT = 'jobsRetentionOf';
+
+/**
+ * The app's windows, or core's defaults for a boot with no config file or a hand-built
+ * `AppConfig` that left a key out (`JobsConfig` declares the three optional) — the same constants
+ * `defineConfig` fills in, so an app without `app.config.ts` keeps finished jobs exactly as long as
+ * one that never wrote the keys. Core's validator refuses a bad window in a loaded file; this
+ * screen is the layered form for the config that never met it.
+ */
+export function jobsRetentionOf(config: AppConfig | undefined): JobsRetention {
+  const jobs = config?.jobs;
+  return {
+    doneRetentionMs: finiteCount(
+      SUBJECT,
+      'jobs.doneRetentionMs',
+      jobs?.doneRetentionMs ?? JOBS_DONE_RETENTION_MS_DEFAULT,
+      1,
+    ),
+    deadRetentionMs: finiteCount(
+      SUBJECT,
+      'jobs.deadRetentionMs',
+      jobs?.deadRetentionMs ?? JOBS_DEAD_RETENTION_MS_DEFAULT,
+      1,
+    ),
+    outboxRetentionMs: finiteCount(
+      SUBJECT,
+      'jobs.outboxRetentionMs',
+      jobs?.outboxRetentionMs ?? JOBS_OUTBOX_RETENTION_MS_DEFAULT,
+      1,
+    ),
+  };
+}
+
+/** The two queue-table targets `installRetentionSweep` takes, over the boot's own executor. */
+export function jobsRetentionTargets(
+  executor: PgExecutor,
+  config: AppConfig | undefined,
+): { readonly jobs: PurgeTarget; readonly outbox: PurgeTarget } {
+  const windows = jobsRetentionOf(config);
+  return {
+    jobs: jobsPurgeTarget({
+      executor,
+      doneRetentionMs: windows.doneRetentionMs,
+      deadRetentionMs: windows.deadRetentionMs,
+    }),
+    outbox: outboxPurgeTarget({ executor, publishedRetentionMs: windows.outboxRetentionMs }),
+  };
+}

package/src/runtime-purge.ts

@@ -1,11 +1,11 @@
 // The retention sweep this boot owns: the framework tables that grow with traffic, the `purge()`
 // job that empties them and the `task` that fires it hourly.
 //
-// SEVEN TARGETS over seven tables, `As of 2026-10-02` — `x_idempotency`, `x_rate_limit`, the
-// `x_auth` limiter's table, `x_notify_deliveries`, `x_notify_inbox`, `x_notify_digests` and
-// `x_job_events`. Counted nowhere in prose but here: this header said "three" for two releases
-// after the notify tables joined the boot's DDL, which is how `x_notify_inbox` became the one
-// framework table nothing swept.
+// NINE TARGETS, `As of 2026-10-17` — `x_idempotency`, `x_rate_limit`, the `x_auth` limiter's
+// table, `x_notify_deliveries`, `x_notify_inbox`, `x_notify_digests`, `x_job_events`, `x_outbox`
+// (published rows) and `x_jobs` (finished rows, with their `x_job_steps`). Counted nowhere in prose but here: this header said
+// "three" for two releases after the notify tables joined the boot's DDL, which is how
+// `x_notify_inbox` became the one framework table nothing swept — and `x_jobs` was the next one.
 //
 // WHY here and not in the packages that own the tables: `postgresIdempotencyStore` (tier 3),
 // `postgresRateLimitStore` (tier 2) and `postgresAuthLimiter` (tier 2) cannot see each other and
@@ -58,6 +58,15 @@ export interface RetentionStores {
    * growing for the life of the deployment — which it did until this member existed.
    */
   readonly events: EventBus;
+  /**
+   * Finished jobs — `jobsPurgeTarget` over this boot's executor, with the app's
+   * `jobs.doneRetentionMs` / `jobs.deadRetentionMs`. REQUIRED for `events`' reason: until it
+   * existed `x_jobs` and `x_job_steps` kept every job ever run, `mail.send`'s recipient and body
+   * among them.
+   */
+  readonly jobs: PurgeTarget;
+  /** Published outbox rows — `outboxPurgeTarget` at `jobs.outboxRetentionMs`. REQUIRED, as `jobs`. */
+  readonly outbox: PurgeTarget;
   /**
    * The app's own answer to "how long is an unread message kept", read off the config
    * `startServices` loads (`inboxRetentionOf`). Absent windows mean the inbox is never swept,
@@ -143,6 +152,11 @@ function retentionTargets(stores: RetentionStores): readonly PurgeTarget[] {
     ...notifyTargets(stores.inboxRetention ?? NO_INBOX_RETENTION),
     // Its own clock, and so no `nowMs`: every instant in that table is the database's.
     eventsPurgeTarget(stores.events),
+    // Last: the two widest tables, batched, so every small sweep above has run before they spend
+    // the pass. Either order is safe — the outbox sweep takes only PUBLISHED rows, and the `x_jobs`
+    // sweep vetoes only a job whose outbox row is still unpublished.
+    stores.outbox,
+    stores.jobs,
   ];
 }
 

package/src/runtime-queue.ts

@@ -35,7 +35,7 @@ import {
   setJobDriver,
   setJobsFacade,
 } from '@ultimat3/jobs';
-import { applyLockedSchema, verifySchema } from './framework-schema-apply';
+import { applyLockedSchema, ensureSchema, verifySchema } from './framework-schema-apply';
 import { appExtensions } from './migration-extensions';
 import type { DevServices } from './runtime-bindings';
 import type { RuntimeOverrides } from './runtime-overrides';
@@ -124,15 +124,20 @@ export function startDb(services: DevServices, env: ReplicaEnv): StartedDb {
  * schema — the same reason `SQL_JOBS_TABLE` is applied by the boot.
  */
 /**
- * `'apply'` — `ROLE=migrate`, `x dev` and every CLI command — runs the DDL under the migration
- * lock with a bounded `lock_timeout` and stamps the build. `'verify'` — a serving role on an
- * external database — runs none: an `alter table` queued behind one open transaction stalled every
- * enqueue in the fleet on each pod start, and racing `create table`s were a first-deploy `23505`.
+ * `'apply'` — `ROLE=migrate`, `x db migrate`, `x dev` and `x mcp serve` — runs the DDL under the
+ * migration lock with a bounded `lock_timeout` and stamps the build. `'verify'` — a serving role on
+ * an external database — runs none: an `alter table` queued behind one open transaction stalled
+ * every enqueue in the fleet on each pod start, and racing `create table`s were a first-deploy
+ * `23505`. `'ensure'` — every other CLI command, `x db backfill` and `x jobs` among them — runs
+ * none when this build's stamp is present and applies otherwise: those commands run as the
+ * runtime role in production, which owns no framework table and so cannot re-run even
+ * `if not exists` DDL (`framework-schema-apply.ts`).
  */
-export type SchemaMode = 'apply' | 'verify';
+export type SchemaMode = 'apply' | 'ensure' | 'verify';
 
 async function applySchema(client: DevDbClient, mode: SchemaMode): Promise<void> {
   if (mode === 'verify') await verifySchema(client);
+  else if (mode === 'ensure') await ensureSchema(client);
   else await applyLockedSchema(client);
 }
 /**
@@ -248,7 +253,11 @@ export async function startQueue(
    * this file reads it: see `startDb` for what two readers of one question cost.
    */
   env: ReplicaEnv = process.env,
-  schema: SchemaMode = 'apply',
+  /**
+   * `'ensure'` unless the caller IS the step that applies: a command that only reads or enqueues
+   * must not need the privileges of the migrate role (`SchemaMode`).
+   */
+  schema: SchemaMode = 'ensure',
 ): Promise<RunningQueue> {
   const { client: db, replica } = startDb(services, env);
   try {

package/src/runtime-services.ts

@@ -37,6 +37,7 @@ import type { DevServices, Env } from './runtime-bindings';
 import { selectBus } from './runtime-bus';
 import { cacheTiersOf, startCacheTiers } from './runtime-cache';
 import { type WorkerConfig, workerConfigOf } from './runtime-jobs';
+import { jobsRetentionTargets } from './runtime-jobs-retention';
 import { selectAppMailDriver } from './runtime-mail';
 import { inboxRetentionOf } from './runtime-notify-retention';
 import type { RuntimeOverrides } from './runtime-overrides';
@@ -403,6 +404,7 @@ export async function startServices(
         idempotency: queue.idempotency,
         rateLimit: rateLimitStore,
         events,
+        ...jobsRetentionTargets(executor, config),
         inboxRetention: inboxRetentionOf(config),
       }),
     );

package/src/schema-errors.ts

@@ -18,10 +18,13 @@ export class FrameworkSchemaFailedError extends UltimateError {
     super({
       code: 'X_FRAMEWORK_SCHEMA_FAILED',
       cause: `${input.pkg} could not create ${input.tables.join(', ')}: ${renderThrowable(input.cause)}`,
-      // An EDIT plus the command that CONFIRMS it, which is the house shape for a repair the gate
-      // cannot perform: the two real causes are a role without `create`, and a relation of that
-      // name already present with an incompatible shape.
-      fix: `grant create on schema public to the role in DATABASE_URL, then re-run: x db migrate`,
+      // The migrate step, as the role that owns the schema. The common cause is `42501`: a runtime
+      // role with DML only — no `create` on `public`, and not the owner of an existing framework
+      // table, which Postgres demands even of `create index if not exists`. Granting that role
+      // `create` (this line's old advice) cures the first and never the second, and widens the
+      // runtime role besides. The rarer cause, a relation of that name with another shape, is
+      // named in `cause` and reported again by the same command.
+      fix: 'x db migrate   # with DATABASE_URL naming the role that owns schema public, as ROLE=migrate runs',
       meta: { pkg: input.pkg, tables: [...input.tables] },
     });
   }

package/src/serve.ts

@@ -48,10 +48,13 @@ export type { MigratedApp, ServedApp, ServeOptions, StartedApp } from './serve-t
  * is what fails on it.
  */
 export async function runMigrations(options: ServeOptions): Promise<MigratedApp> {
+  // `'apply'`, never the stamp-first default: this IS the step that applies, and an operator
+  // re-running it is how a dropped framework table comes back.
   const queue = await startQueue(
     resolveServices(options.root, options.env),
     options.runtime,
     options.env,
+    'apply',
   );
   try {
     const migrations = await readMigrations(options.root);