@ultimat3/cli 27.10.4 → 27.12.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/cli",
-  "version": "27.10.4",
+  "version": "27.12.0",
   "description": "The `x` binary: new, dev, build, verify, generate, db, mcp, doctor, deploy",
   "license": "MIT",
   "type": "module",
@@ -39,33 +39,33 @@
   },
   "dependencies": {
     "@babel/core": "^8.0.1",
-    "@ultimat3/action": "27.10.4",
-    "@ultimat3/admin": "27.10.4",
-    "@ultimat3/ai": "27.10.4",
-    "@ultimat3/auth": "27.10.4",
-    "@ultimat3/cache": "27.10.4",
-    "@ultimat3/core": "27.10.4",
-    "@ultimat3/db": "27.10.4",
-    "@ultimat3/entity": "27.10.4",
-    "@ultimat3/flags": "27.10.4",
-    "@ultimat3/http": "27.10.4",
-    "@ultimat3/i18n": "27.10.4",
-    "@ultimat3/jobs": "27.10.4",
-    "@ultimat3/mail": "27.10.4",
-    "@ultimat3/manifest": "27.10.4",
-    "@ultimat3/mcp": "27.10.4",
-    "@ultimat3/money": "27.10.4",
-    "@ultimat3/notify": "27.10.4",
-    "@ultimat3/policy": "27.10.4",
-    "@ultimat3/pwa": "27.10.4",
-    "@ultimat3/query": "27.10.4",
-    "@ultimat3/realtime": "27.10.4",
-    "@ultimat3/render": "27.10.4",
-    "@ultimat3/schema": "27.10.4",
-    "@ultimat3/seo": "27.10.4",
-    "@ultimat3/storage": "27.10.4",
-    "@ultimat3/testing": "27.10.4",
-    "@ultimat3/time": "27.10.4",
+    "@ultimat3/action": "27.12.0",
+    "@ultimat3/admin": "27.12.0",
+    "@ultimat3/ai": "27.12.0",
+    "@ultimat3/auth": "27.12.0",
+    "@ultimat3/cache": "27.12.0",
+    "@ultimat3/core": "27.12.0",
+    "@ultimat3/db": "27.12.0",
+    "@ultimat3/entity": "27.12.0",
+    "@ultimat3/flags": "27.12.0",
+    "@ultimat3/http": "27.12.0",
+    "@ultimat3/i18n": "27.12.0",
+    "@ultimat3/jobs": "27.12.0",
+    "@ultimat3/mail": "27.12.0",
+    "@ultimat3/manifest": "27.12.0",
+    "@ultimat3/mcp": "27.12.0",
+    "@ultimat3/money": "27.12.0",
+    "@ultimat3/notify": "27.12.0",
+    "@ultimat3/policy": "27.12.0",
+    "@ultimat3/pwa": "27.12.0",
+    "@ultimat3/query": "27.12.0",
+    "@ultimat3/realtime": "27.12.0",
+    "@ultimat3/render": "27.12.0",
+    "@ultimat3/schema": "27.12.0",
+    "@ultimat3/seo": "27.12.0",
+    "@ultimat3/storage": "27.12.0",
+    "@ultimat3/testing": "27.12.0",
+    "@ultimat3/time": "27.12.0",
     "babel-preset-solid": "^1.9.15"
   }
 }

package/src/framework-schema-apply.ts

@@ -1,7 +1,8 @@
-// The two ways a boot meets the framework schema (plan 101, s1-con #5): `applyLockedSchema` runs the
-// DDL in ONE transaction behind the migration advisory lock with a bounded `lock_timeout`, then
+// The three ways a boot meets the framework schema (plan 101, s1-con #5): `applyLockedSchema` runs
+// the DDL in ONE transaction behind the migration advisory lock with a bounded `lock_timeout`, then
 // stamps the build; `verifySchema` runs no DDL and refuses a database this build was never applied
-// to. Which boot takes which is `runtime-queue.ts`'s `SchemaMode`.
+// to; `ensureSchema` reads the stamp and applies only when this build is absent from it. Which boot
+// takes which is `runtime-queue.ts`'s `SchemaMode`.
 
 import { frameworkVersion, logger } from '@ultimat3/core';
 import type { DbClient } from '@ultimat3/db';
@@ -70,9 +71,39 @@ export async function verifySchema(client: DbClient): Promise<void> {
   const mine = thisBuild();
   const verdict = stampVerdict(parseStamp(await readStamp(client)), mine);
   if (!verdict.applied) throw new FrameworkSchemaUnappliedError({ mine, newest: verdict.newest });
-  if (verdict.skew === undefined) return;
+  reportSkew(mine, verdict.skew);
+}
+
+/**
+ * A CLI command's boot — `x db backfill`, `x jobs`, `x auth`: no DDL when this build's stamp is
+ * already there, the locked apply when it is not.
+ *
+ * The stamp is read FIRST because re-applying idempotent DDL is not idempotent for every role.
+ * Postgres checks ownership before existence, so `create index if not exists` on a table the role
+ * does not own is `42501 must be owner` even when the index is there. A production runtime role
+ * holds DML only, and the PostSync backfill Job ran as it: every deploy's `x db backfill --all
+ * --write` died in the jobs entry and no sweep was ever enqueued. When the stamp is absent the
+ * apply still runs, so a fresh dev database needs no separate step — and a role that cannot apply
+ * it is refused with `X_FRAMEWORK_SCHEMA_FAILED`, whose fix is the migrate step.
+ */
+export async function ensureSchema(client: DbClient): Promise<void> {
+  const mine = thisBuild();
+  const verdict = stampVerdict(parseStamp(await readStamp(client)), mine);
+  if (!verdict.applied) {
+    await applyLockedSchema(client);
+    return;
+  }
+  reportSkew(mine, verdict.skew);
+}
+
+/**
+ * A fleet straddling a major is reported once and served — refusing it would crash-loop the old
+ * pods of every rolling deploy across a breaking release.
+ */
+function reportSkew(mine: StampEntry, skew: StampEntry | undefined): void {
+  if (skew === undefined) return;
   logger.warn('ultimate framework major skew', {
-    cause: `this process runs ${mine.version} and the database's framework schema was last applied by ${verdict.skew.version}: a fleet across a major boundary is mid-rollout or half rolled back`,
+    cause: `this process runs ${mine.version} and the database's framework schema was last applied by ${skew.version}: a fleet across a major boundary is mid-rollout or half rolled back`,
     fix: 'kubectl rollout status deployment --selector app.kubernetes.io/component=web',
   });
 }

package/src/framework-schema.ts

@@ -1,7 +1,7 @@
 // The framework's OWN tables, as one table: which package declares each, which relations it
 // creates, and the DDL. One list, read by the one applier, so "installed in dev and not in
-// production" is not a state this framework can be in — `startQueue` is on every boot path there
-// is, `ROLE=migrate` included.
+// production" is not a state this framework can be in: `ROLE=migrate` applies it and stamps the
+// build, and every other boot path meets it through `startQueue`'s `SchemaMode`.
 //
 // A framework table an app has to install by hand is a table that will be missing in production on
 // the one code path that needs it, and it surfaces as a Postgres `42P01` from inside a worker.

package/src/hold.ts

@@ -3,7 +3,15 @@
 // before the exit code. Ctrl-C then takes core's own three-phase drain (stop accepting, finish
 // in-flight, close) instead of killing a query mid-round-trip.
 
-import { drain, installSignalHandlers, logger, onShutdown, systemClock } from '@ultimat3/core';
+import {
+  drain,
+  installSignalHandlers,
+  isDraining,
+  isUltimateError,
+  logger,
+  onShutdown,
+  systemClock,
+} from '@ultimat3/core';
 
 export interface HoldOptions {
   /**
@@ -129,7 +137,24 @@ export async function holdWhileBooting<T extends { stop(): Promise<void> }>(
   try {
     app = await boot();
   } catch (error) {
-    armed.disarm();
+    // A boot that failed BECAUSE a drain began under it — a rollout superseding a pod still
+    // booting, whose server then met `markReady()`'s `X_LIFECYCLE_DRAINED` — is a pod doing what
+    // it was told: finish the drain, say so once at info, exit 0. It printed a stack trace and
+    // exited non-zero, twice a week in production, paging on a normal rollout. A failure with no
+    // drain under way is still the boot's own refusal, rethrown untouched.
+    if (isDraining() && options.exit !== undefined) {
+      await drain();
+      armed.disarm();
+      logger.info('ultimate boot interrupted by shutdown', {
+        name,
+        code: isUltimateError(error) ? error.code : 'uncoded',
+      });
+      options.exit(0);
+    } else {
+      armed.disarm();
+    }
+    // Unreachable past a real `process.exit`; a test's `exit` spy returns, and the caller still
+    // sees the boot's error rather than an app that never booted.
     throw error;
   }
   booted = app;

package/src/runtime-cache.ts

@@ -23,6 +23,7 @@ import type { AppConfig, CacheTierName } from '@ultimat3/core';
 import {
   backoffDelay,
   defineConfig,
+  finiteCount,
   isOutageMilestone,
   logger,
   renderThrowable,
@@ -63,8 +64,23 @@ export interface CacheTiersOptions {
   readonly tiers: readonly CacheTierName[];
   /** The wait before re-subscribing after `attempt` failures (1-based). Injected by a test. */
   readonly subscribeRetryMs?: ((attempt: number) => number) | undefined;
+  /**
+   * How long the boot subscribe may be refused before a refusal is worth a warning. Default
+   * `FIRST_DIAL_GRACE_MS`; a test of an OUTAGE passes 0.
+   */
+  readonly firstDialGraceMs?: number | undefined;
 }
 
+/**
+ * The boot subscribe races the bus's FIRST dial, and loses it on almost every pod: in production
+ * every pod warned on attempts 1–2 and was subscribed within 1.3 s — 345 lines in three weeks of
+ * alerting noise. Refusals inside this window are not reported; a bus still refusing after it is
+ * an outage and warns as before (attempt milestones, then `recovered` at info). 10 s is several
+ * times the slowest first dial measured, and far under any window in which a missed peer bust
+ * matters: the process is flushed (`flushDeaf`) when the subscribe lands either way.
+ */
+export const FIRST_DIAL_GRACE_MS = 10_000;
+
 /** 1 s doubling to 30 s, full jitter: a bus that is down is retried without a fleet in lockstep. */
 const defaultSubscribeRetryMs = (attempt: number): number =>
   backoffDelay({ attempt, base: 1_000, max: 30_000, jitter: 'full' });
@@ -163,6 +179,13 @@ function warnUnnamed(options: CacheTiersOptions): void {
  * suppresses it, and `emit` is not a public parameter.
  */
 export function startCacheTiers(options: CacheTiersOptions): () => Promise<void> {
+  // Every option checked before anything is registered: a refusal after `registerTier` or
+  // `registerInvalidationBroadcast` would leave process-global state installed with no release.
+  const graceMs = finiteCount(
+    'startCacheTiers',
+    'firstDialGraceMs',
+    options.firstDialGraceMs ?? FIRST_DIAL_GRACE_MS,
+  );
   // Built before ANY of them is registered: a refusal halfway through a list would leave the
   // process-global registry holding the rungs that came first, with no release returned to drop
   // them. Registration order does not decide read order — `sortTiers` does, by `CACHE_TIERS`.
@@ -276,6 +299,9 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
   const flushDeaf = (source: string): void => {
     void flushProcessTiers(source);
   };
+  const bootedAt = performance.now();
+  /** Whether an operator has been told of this outage — what makes `recovered` worth a line. */
+  let warned = false;
   const subscribing = (async (): Promise<TransportSubscription | undefined> => {
     for (let attempt = 1; !stopped; attempt += 1) {
       try {
@@ -287,15 +313,22 @@ export function startCacheTiers(options: CacheTiersOptions): () => Promise<void>
         );
         subscribed = true;
         if (attempt > 1) {
-          // The outage's one closing line, beside the thinned warnings it ends.
-          logger.info('cache.broadcast.subscribe-failed recovered', { after: attempt - 1 });
+          // The outage's one closing line, beside the thinned warnings it ends — and only then: a
+          // first dial that landed inside its window had no outage to close.
+          if (warned) {
+            logger.info('cache.broadcast.subscribe-failed recovered', { after: attempt - 1 });
+          }
           flushDeaf('cache.bus-subscribed');
         }
         return subscription;
       } catch (error) {
         // Attempts 1, 2, 4, 8, …: the dial's own thinning (`NatsTransport`), so an outage is not
         // one line per retry per pod for as long as it lasts.
-        if (isOutageMilestone(attempt)) {
+        // Silent inside the first dial's window. Past it, the FIRST refusal warns whatever its
+        // attempt number, then the milestones thin the rest.
+        const inGrace = performance.now() - bootedAt < graceMs;
+        if (!inGrace && (!warned || isOutageMilestone(attempt))) {
+          warned = true;
           logger.warn('cache.broadcast.subscribe-failed', {
             error: broadcastErrorText(error),
             attempt,

package/src/runtime-jobs-retention.ts

@@ -0,0 +1,69 @@
+// `jobs.doneRetentionMs`, `jobs.deadRetentionMs` and `jobs.outboxRetentionMs`, out of the config
+// `startServices` loads once (`app-config-load.ts`), as the windows the `x_jobs` and `x_outbox`
+// purge targets are built with. The `notify` windows' sibling (`runtime-notify-retention.ts`), with
+// one difference: these DEFAULT, because a finished job is bookkeeping and an unread message is not.
+
+import type { AppConfig, PgExecutor } from '@ultimat3/core';
+import {
+  finiteCount,
+  JOBS_DEAD_RETENTION_MS_DEFAULT,
+  JOBS_DONE_RETENTION_MS_DEFAULT,
+  JOBS_OUTBOX_RETENTION_MS_DEFAULT,
+} from '@ultimat3/core';
+import type { PurgeTarget } from '@ultimat3/jobs';
+import { jobsPurgeTarget, outboxPurgeTarget } from '@ultimat3/jobs';
+
+export interface JobsRetention {
+  readonly doneRetentionMs: number;
+  readonly deadRetentionMs: number;
+  readonly outboxRetentionMs: number;
+}
+
+const SUBJECT = 'jobsRetentionOf';
+
+/**
+ * The app's windows, or core's defaults for a boot with no config file or a hand-built
+ * `AppConfig` that left a key out (`JobsConfig` declares the three optional) — the same constants
+ * `defineConfig` fills in, so an app without `app.config.ts` keeps finished jobs exactly as long as
+ * one that never wrote the keys. Core's validator refuses a bad window in a loaded file; this
+ * screen is the layered form for the config that never met it.
+ */
+export function jobsRetentionOf(config: AppConfig | undefined): JobsRetention {
+  const jobs = config?.jobs;
+  return {
+    doneRetentionMs: finiteCount(
+      SUBJECT,
+      'jobs.doneRetentionMs',
+      jobs?.doneRetentionMs ?? JOBS_DONE_RETENTION_MS_DEFAULT,
+      1,
+    ),
+    deadRetentionMs: finiteCount(
+      SUBJECT,
+      'jobs.deadRetentionMs',
+      jobs?.deadRetentionMs ?? JOBS_DEAD_RETENTION_MS_DEFAULT,
+      1,
+    ),
+    outboxRetentionMs: finiteCount(
+      SUBJECT,
+      'jobs.outboxRetentionMs',
+      jobs?.outboxRetentionMs ?? JOBS_OUTBOX_RETENTION_MS_DEFAULT,
+      1,
+    ),
+  };
+}
+
+/** The two queue-table targets `installRetentionSweep` takes, over the boot's own executor. */
+export function jobsRetentionTargets(
+  executor: PgExecutor,
+  config: AppConfig | undefined,
+): { readonly jobs: PurgeTarget; readonly outbox: PurgeTarget } {
+  const windows = jobsRetentionOf(config);
+  return {
+    jobs: jobsPurgeTarget({
+      executor,
+      doneRetentionMs: windows.doneRetentionMs,
+      deadRetentionMs: windows.deadRetentionMs,
+    }),
+    outbox: outboxPurgeTarget({ executor, publishedRetentionMs: windows.outboxRetentionMs }),
+  };
+}

package/src/runtime-purge.ts

@@ -1,11 +1,11 @@
 // The retention sweep this boot owns: the framework tables that grow with traffic, the `purge()`
 // job that empties them and the `task` that fires it hourly.
 //
-// SEVEN TARGETS over seven tables, `As of 2026-10-02` — `x_idempotency`, `x_rate_limit`, the
-// `x_auth` limiter's table, `x_notify_deliveries`, `x_notify_inbox`, `x_notify_digests` and
-// `x_job_events`. Counted nowhere in prose but here: this header said "three" for two releases
-// after the notify tables joined the boot's DDL, which is how `x_notify_inbox` became the one
-// framework table nothing swept.
+// NINE TARGETS, `As of 2026-10-17` — `x_idempotency`, `x_rate_limit`, the `x_auth` limiter's
+// table, `x_notify_deliveries`, `x_notify_inbox`, `x_notify_digests`, `x_job_events`, `x_outbox`
+// (published rows) and `x_jobs` (finished rows, with their `x_job_steps`). Counted nowhere in prose but here: this header said
+// "three" for two releases after the notify tables joined the boot's DDL, which is how
+// `x_notify_inbox` became the one framework table nothing swept — and `x_jobs` was the next one.
 //
 // WHY here and not in the packages that own the tables: `postgresIdempotencyStore` (tier 3),
 // `postgresRateLimitStore` (tier 2) and `postgresAuthLimiter` (tier 2) cannot see each other and
@@ -58,6 +58,15 @@ export interface RetentionStores {
    * growing for the life of the deployment — which it did until this member existed.
    */
   readonly events: EventBus;
+  /**
+   * Finished jobs — `jobsPurgeTarget` over this boot's executor, with the app's
+   * `jobs.doneRetentionMs` / `jobs.deadRetentionMs`. REQUIRED for `events`' reason: until it
+   * existed `x_jobs` and `x_job_steps` kept every job ever run, `mail.send`'s recipient and body
+   * among them.
+   */
+  readonly jobs: PurgeTarget;
+  /** Published outbox rows — `outboxPurgeTarget` at `jobs.outboxRetentionMs`. REQUIRED, as `jobs`. */
+  readonly outbox: PurgeTarget;
   /**
    * The app's own answer to "how long is an unread message kept", read off the config
    * `startServices` loads (`inboxRetentionOf`). Absent windows mean the inbox is never swept,
@@ -143,6 +152,11 @@ function retentionTargets(stores: RetentionStores): readonly PurgeTarget[] {
     ...notifyTargets(stores.inboxRetention ?? NO_INBOX_RETENTION),
     // Its own clock, and so no `nowMs`: every instant in that table is the database's.
     eventsPurgeTarget(stores.events),
+    // Last: the two widest tables, batched, so every small sweep above has run before they spend
+    // the pass. Either order is safe — the outbox sweep takes only PUBLISHED rows, and the `x_jobs`
+    // sweep vetoes only a job whose outbox row is still unpublished.
+    stores.outbox,
+    stores.jobs,
   ];
 }
 

package/src/runtime-queue.ts

@@ -35,7 +35,7 @@ import {
   setJobDriver,
   setJobsFacade,
 } from '@ultimat3/jobs';
-import { applyLockedSchema, verifySchema } from './framework-schema-apply';
+import { applyLockedSchema, ensureSchema, verifySchema } from './framework-schema-apply';
 import { appExtensions } from './migration-extensions';
 import type { DevServices } from './runtime-bindings';
 import type { RuntimeOverrides } from './runtime-overrides';
@@ -124,15 +124,20 @@ export function startDb(services: DevServices, env: ReplicaEnv): StartedDb {
  * schema — the same reason `SQL_JOBS_TABLE` is applied by the boot.
  */
 /**
- * `'apply'` — `ROLE=migrate`, `x dev` and every CLI command — runs the DDL under the migration
- * lock with a bounded `lock_timeout` and stamps the build. `'verify'` — a serving role on an
- * external database — runs none: an `alter table` queued behind one open transaction stalled every
- * enqueue in the fleet on each pod start, and racing `create table`s were a first-deploy `23505`.
+ * `'apply'` — `ROLE=migrate`, `x db migrate`, `x dev` and `x mcp serve` — runs the DDL under the
+ * migration lock with a bounded `lock_timeout` and stamps the build. `'verify'` — a serving role on
+ * an external database — runs none: an `alter table` queued behind one open transaction stalled
+ * every enqueue in the fleet on each pod start, and racing `create table`s were a first-deploy
+ * `23505`. `'ensure'` — every other CLI command, `x db backfill` and `x jobs` among them — runs
+ * none when this build's stamp is present and applies otherwise: those commands run as the
+ * runtime role in production, which owns no framework table and so cannot re-run even
+ * `if not exists` DDL (`framework-schema-apply.ts`).
  */
-export type SchemaMode = 'apply' | 'verify';
+export type SchemaMode = 'apply' | 'ensure' | 'verify';
 
 async function applySchema(client: DevDbClient, mode: SchemaMode): Promise<void> {
   if (mode === 'verify') await verifySchema(client);
+  else if (mode === 'ensure') await ensureSchema(client);
   else await applyLockedSchema(client);
 }
 /**
@@ -248,7 +253,11 @@ export async function startQueue(
    * this file reads it: see `startDb` for what two readers of one question cost.
    */
   env: ReplicaEnv = process.env,
-  schema: SchemaMode = 'apply',
+  /**
+   * `'ensure'` unless the caller IS the step that applies: a command that only reads or enqueues
+   * must not need the privileges of the migrate role (`SchemaMode`).
+   */
+  schema: SchemaMode = 'ensure',
 ): Promise<RunningQueue> {
   const { client: db, replica } = startDb(services, env);
   try {

package/src/runtime-services.ts

@@ -37,6 +37,7 @@ import type { DevServices, Env } from './runtime-bindings';
 import { selectBus } from './runtime-bus';
 import { cacheTiersOf, startCacheTiers } from './runtime-cache';
 import { type WorkerConfig, workerConfigOf } from './runtime-jobs';
+import { jobsRetentionTargets } from './runtime-jobs-retention';
 import { selectAppMailDriver } from './runtime-mail';
 import { inboxRetentionOf } from './runtime-notify-retention';
 import type { RuntimeOverrides } from './runtime-overrides';
@@ -403,6 +404,7 @@ export async function startServices(
         idempotency: queue.idempotency,
         rateLimit: rateLimitStore,
         events,
+        ...jobsRetentionTargets(executor, config),
         inboxRetention: inboxRetentionOf(config),
       }),
     );

package/src/schema-errors.ts

@@ -18,10 +18,13 @@ export class FrameworkSchemaFailedError extends UltimateError {
     super({
       code: 'X_FRAMEWORK_SCHEMA_FAILED',
       cause: `${input.pkg} could not create ${input.tables.join(', ')}: ${renderThrowable(input.cause)}`,
-      // An EDIT plus the command that CONFIRMS it, which is the house shape for a repair the gate
-      // cannot perform: the two real causes are a role without `create`, and a relation of that
-      // name already present with an incompatible shape.
-      fix: `grant create on schema public to the role in DATABASE_URL, then re-run: x db migrate`,
+      // The migrate step, as the role that owns the schema. The common cause is `42501`: a runtime
+      // role with DML only — no `create` on `public`, and not the owner of an existing framework
+      // table, which Postgres demands even of `create index if not exists`. Granting that role
+      // `create` (this line's old advice) cures the first and never the second, and widens the
+      // runtime role besides. The rarer cause, a relation of that name with another shape, is
+      // named in `cause` and reported again by the same command.
+      fix: 'x db migrate   # with DATABASE_URL naming the role that owns schema public, as ROLE=migrate runs',
       meta: { pkg: input.pkg, tables: [...input.tables] },
     });
   }

package/src/serve.ts

@@ -48,10 +48,13 @@ export type { MigratedApp, ServedApp, ServeOptions, StartedApp } from './serve-t
  * is what fails on it.
  */
 export async function runMigrations(options: ServeOptions): Promise<MigratedApp> {
+  // `'apply'`, never the stamp-first default: this IS the step that applies, and an operator
+  // re-running it is how a dropped framework table comes back.
   const queue = await startQueue(
     resolveServices(options.root, options.env),
     options.runtime,
     options.env,
+    'apply',
   );
   try {
     const migrations = await readMigrations(options.root);

package/src/templates/github/ci.yml.ts

@@ -11,6 +11,7 @@
 
 import { REQUIRED_BUN } from '../../app-root';
 import type { GeneratedFile, NameSet } from '../naming';
+import { TEST_COMPOSE_PATH, TEST_ENV_PATH } from '../scaffold-test-services';
 import { IMAGE_WORKFLOW_PATH, imageWorkflow } from './image.yml';
 import { RUNS_ON } from './runner';
 
@@ -39,9 +40,9 @@ const ci = (app: NameSet): string => `name: ${CI_WORKFLOW_NAME}
 # \`.x/build-stats.json\`, so a run with no build reports X_BUDGET_UNMEASURED. Run \`x verify\` here
 # instead and this workflow is red for a reason that has nothing to do with the commit.
 #
-# No \`services:\` block, deliberately — \`bun run setup\` brings up embedded Postgres in-process,
-# so there is nothing to provision and nothing to wait on. Add one the day this app needs a real
-# server, beside the \`DATABASE_URL\` that selects it.
+# \`bun run setup\` brings up embedded Postgres in-process; the TEST services come from
+# \`docker/docker-compose.test.yml\` — Postgres 18, Redis, NATS and S3 in RAM, durability and logs
+# off — started by the same two lines a laptop runs, never a \`services:\` block that restates them.
 
 # EVERY push, not just the default branch's. \`x new\` runs a plain \`git init\` and takes whatever
 # \`init.defaultBranch\` this machine already agreed on, so a \`branches: [main]\` filter silently
@@ -90,6 +91,9 @@ jobs:
           restore-keys: bun-\${{ runner.os }}-
       # Two steps rather than one \`&&\`, so the log names which half failed and times each.
       - run: bun run setup
+      # \`--wait\` returns once every healthcheck passes; the env file points the suites at them.
+      - run: docker compose -f ${TEST_COMPOSE_PATH} up -d --wait
+      - run: cat ${TEST_ENV_PATH} >> "$GITHUB_ENV"
       - run: bun run check
 `;
 

package/src/templates/scaffold-container-compose.ts

@@ -81,11 +81,13 @@ x-run-once-probe: &run-once-probe
 
 services:
   db:
-    image: postgres:17-alpine
+    # Postgres 18 images keep the cluster under /var/lib/postgresql/18/docker, so the volume is the
+    # parent — mounted at .../data, as for 17, the image refuses to start.
+    image: postgres:18-alpine
     environment:
       POSTGRES_PASSWORD: \${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD}
       POSTGRES_DB: ${app.kebab}
-    volumes: ['pgdata:/var/lib/postgresql/data']
+    volumes: ['pgdata:/var/lib/postgresql']
     healthcheck:
       test: ['CMD-SHELL', 'pg_isready -U postgres']
       interval: 5s

package/src/templates/scaffold-docs.ts

@@ -11,6 +11,7 @@ import type { GeneratedFile, NameSet } from './naming';
 import { binFiles } from './scaffold-bin';
 import { claudeFiles } from './scaffold-claude';
 import { containerFiles } from './scaffold-container';
+import { testServicesFiles } from './scaffold-test-services';
 
 const agents = (app: NameSet): string => `# AGENTS.md
 
@@ -67,7 +68,14 @@ branch whose test cannot fail is worse than an uncovered one, because it reads a
 browser-only mount, a container entry point — carries its \`why\`, and is printed by \`x doctor\`.
 
 Every test the scaffold and the generators write is the idiom to copy. All of them run on the
-\`unit\` step — in process, on the in-memory driver, with no database, browser or server:
+\`unit\` step — in process, on the in-memory driver, with no database, browser or server.
+
+A suite that needs a real server reads it from \`docker/test-services.env\`: Postgres 18, Redis, NATS
+and S3 from \`docker/docker-compose.test.yml\`, in RAM, with durability and logging off — CI starts
+it before \`bun run check\`, a laptop with \`docker compose -f docker/docker-compose.test.yml up -d
+--wait\` then \`set -a; . docker/test-services.env; set +a\`. A resource a test opens is held with
+\`await using\` — \`await using db = await acquireWorkerDatabase()\` drops its database however
+the test ends:
 
 | What | How its test runs it | Copy from |
 |---|---|---|
@@ -136,6 +144,14 @@ bun run dev     # all roles in one process, embedded Postgres, /_x mounted
 bun run check   # the gate: a static build, then typecheck, lint, boundaries, tests, drift, budgets
 \`\`\`
 
+Tests against real servers — Postgres 18, Redis, NATS, S3, in RAM and silent, as CI runs them:
+
+\`\`\`sh
+docker compose -f docker/docker-compose.test.yml up -d --wait
+set -a; . docker/test-services.env; set +a
+bun run check
+\`\`\`
+
 \`packages/db/migrations\` starts empty and \`x db gen\` is its only writer — \`bun run setup\` runs
 \`x db gen "initial"\` for you on a fresh clone. Until it has, \`x verify\`'s \`drift\` step is red
 with \`X_DB_DRIFT\`, and that is the fix it names.
@@ -173,7 +189,7 @@ const composeDev = (
 # production topology is docker-compose.prod.yml, and it is a different file for a reason.
 services:
   db:
-    image: postgres:17-alpine
+    image: postgres:18-alpine
     environment:
       POSTGRES_PASSWORD: ${app.kebab}
       POSTGRES_DB: ${app.kebab}
@@ -214,6 +230,8 @@ export function docsFiles(app: NameSet): readonly GeneratedFile[] {
     // The scripts `bun run setup` and `bun run check` run — TypeScript, so one file per OS.
     ...binFiles(),
     { path: 'docker/docker-compose.dev.yml', contents: composeDev(app) },
+    // What the suites run against — in RAM, silent; ci.yml starts it before the gate.
+    ...testServicesFiles(app),
     // The harness half of the same job AGENTS.md does. It lands in the app's own repo rather than
     // in a global config, so it is visible in the scaffold's diff and deletable in one line.
     ...claudeFiles(app),

package/src/templates/scaffold-test-services.ts

@@ -0,0 +1,137 @@
+// The backing services a new app's suites run against: `docker/docker-compose.test.yml` and the
+// one env file that points the suites at it. In RAM, durability off, silent — the framework's own
+// `docker/docker-compose.test.yml`, cut to what an app has. Ports sit apart from the dev stack's
+// so both can run at once; `ci.yml` starts this file before `bun run check`.
+
+import type { GeneratedFile, NameSet } from './naming';
+
+export const TEST_COMPOSE_PATH = 'docker/docker-compose.test.yml';
+export const TEST_ENV_PATH = 'docker/test-services.env';
+
+/** Host ports, one place: the compose file publishes them and the env file names them. */
+export const TEST_PORTS = { postgres: 55432, redis: 56379, nats: 54222, s3: 59000 } as const;
+
+const composeTest = (
+  app: NameSet,
+): string => `# The services the test suites run against — in RAM, and silent.
+#
+#   docker compose -f ${TEST_COMPOSE_PATH} up -d --wait
+#   set -a; . ${TEST_ENV_PATH}; set +a
+#   bun run check
+#
+# \`--wait\` returns once every healthcheck passes. CI runs the same lines (.github/workflows/ci.yml).
+# None of it is required: with no TEST_* variable set the unit suite runs on the in-memory driver,
+# and a suite that needs a real server skips. Ports are the dev stack's plus 50000, so both run.
+#
+# Three rules, on every service:
+#
+#   * RAM. Data lives on a tmpfs: no block device written, nothing left behind by \`down\`.
+#   * NO DURABILITY. fsync, synchronous_commit and full_page_writes off, WAL minimal and no WAL
+#     senders; Redis keeps no RDB snapshot and no AOF. A test database that survives a crash is a
+#     database nobody asked for.
+#   * SILENT. Each server logs as little as it can be told to, and the log driver is \`none\`. To
+#     watch one that will not start, run it attached: \`docker compose -f ${TEST_COMPOSE_PATH} up postgres\`.
+#
+# Every port binds 127.0.0.1: the credentials are in this file.
+name: ${app.kebab}-test
+
+x-silent: &silent
+  logging: { driver: none }
+
+services:
+  # Postgres 18, the production major. \`initdb -c\` writes each setting into the new cluster;
+  # \`--no-sync\` is initdb's own fsync of a cluster on a tmpfs. Postgres 18 images keep the cluster
+  # under /var/lib/postgresql/18, so the tmpfs is the parent.
+  postgres:
+    <<: *silent
+    image: postgres:18-alpine
+    environment:
+      POSTGRES_USER: ${app.kebab}
+      POSTGRES_PASSWORD: ${app.kebab}-test
+      POSTGRES_DB: postgres
+      POSTGRES_INITDB_ARGS: >-
+        --no-sync
+        -c fsync=off
+        -c synchronous_commit=off
+        -c full_page_writes=off
+        -c wal_level=minimal
+        -c max_wal_senders=0
+        -c logging_collector=off
+        -c log_min_messages=fatal
+        -c log_statement=none
+        -c log_checkpoints=off
+        -c log_connections=off
+        -c log_disconnections=off
+    tmpfs: [/var/lib/postgresql]
+    ports: ['127.0.0.1:${TEST_PORTS.postgres}:5432']
+    healthcheck:
+      # TCP, never the socket: the entrypoint's temporary init server listens on the socket only.
+      test: ['CMD-SHELL', 'pg_isready -U ${app.kebab} -h 127.0.0.1 -p 5432']
+      interval: 1s
+      timeout: 3s
+      retries: 60
+
+  redis:
+    <<: *silent
+    image: redis:8-alpine
+    command: ['redis-server', '--save', '', '--appendonly', 'no', '--loglevel', 'warning']
+    tmpfs: [/data]
+    ports: ['127.0.0.1:${TEST_PORTS.redis}:6379']
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+  # JetStream's store on the tmpfs; \`-l /dev/null\` is the only quiet a nats-server has.
+  nats:
+    <<: *silent
+    image: nats:2-alpine
+    command: ['-js', '-sd', '/data', '-m', '8222', '-l', '/dev/null']
+    tmpfs: [/data]
+    ports: ['127.0.0.1:${TEST_PORTS.nats}:4222']
+    healthcheck:
+      test: ['CMD', 'wget', '-q', '--spider', 'http://127.0.0.1:8222/healthz']
+      interval: 1s
+      timeout: 3s
+      retries: 30
+
+  # A directory under /data is a bucket, so the tmpfs mounted AT the bucket's path is the bootstrap.
+  s3:
+    <<: *silent
+    image: versity/versitygw:v1.8.0@sha256:30292fc2eeacc67a36993b01f7a7a5e3361a19cced0e80c1d71cfa2a4b0a2499
+    environment:
+      VGW_BACKEND: posix
+      VGW_BACKEND_ARG: /data
+      VGW_PORT: ':9000'
+      VGW_QUIET: 'true'
+      VGW_REGION: auto
+      ROOT_ACCESS_KEY: ${app.kebab}
+      ROOT_SECRET_KEY: ${app.kebab}-test
+    tmpfs: [/data/${app.kebab}-test]
+    ports: ['127.0.0.1:${TEST_PORTS.s3}:9000']
+    healthcheck:
+      test: ['CMD', 'nc', '-z', '127.0.0.1', '9000']
+      interval: 1s
+      timeout: 3s
+      retries: 30
+`;
+
+const testEnv = (
+  app: NameSet,
+): string => `# Where the suites find the services ${TEST_COMPOSE_PATH} starts — the one place these URLs are
+# written. A laptop sources it (\`set -a; . ${TEST_ENV_PATH}; set +a\`); CI appends it to $GITHUB_ENV.
+# TEST_DATABASE_URL is what @ultimat3/testing clones one database per test worker from.
+TEST_DATABASE_URL=postgres://${app.kebab}:${app.kebab}-test@127.0.0.1:${TEST_PORTS.postgres}/postgres
+TEST_REDIS_URL=redis://127.0.0.1:${TEST_PORTS.redis}
+TEST_NATS_URL=nats://127.0.0.1:${TEST_PORTS.nats}
+TEST_S3_URL=http://${app.kebab}:${app.kebab}-test@127.0.0.1:${TEST_PORTS.s3}/${app.kebab}-test
+`;
+
+/** The test stack and its env file. */
+export function testServicesFiles(app: NameSet): readonly GeneratedFile[] {
+  return [
+    { path: TEST_COMPOSE_PATH, contents: composeTest(app) },
+    { path: TEST_ENV_PATH, contents: testEnv(app) },
+  ];
+}