@ultimat3/cli 1.1.0 → 2.0.0

package/src/dev-roles-fixture.ts

@@ -0,0 +1,67 @@
+// The one `RunningServices` every `dev-roles` test file boots roles against, and the one reset
+// between them. Shared rather than copied, for the reason `policy-fixture.ts` gives: three files
+// start the same roles, and a second copy of the runtime drifts while each file keeps passing.
+//
+// Its own module rather than a `.test.ts` neighbours import, because `tsconfig.json` excludes
+// `*.test.ts` — a fixture written there is one `tsc` never reads.
+
+import { noopPurgeDriver } from '@ultimat3/cache';
+import { resetLifecycle } from '@ultimat3/core';
+import {
+  createMemoryDriver,
+  createMemoryEventBus,
+  createMemoryOutboxStore,
+  resetJobs,
+  resetJobsFacade,
+  resetTasks,
+} from '@ultimat3/jobs';
+import { createMemoryDriver as createMemoryMailDriver } from '@ultimat3/mail';
+import { DEFAULT_PRESENCE_TTL_MS, InProcessTransport } from '@ultimat3/realtime';
+import { defineStorage, localDriver } from '@ultimat3/storage';
+import type { RunningServices } from './dev-runtime';
+import { resolveServices } from './dev-services';
+
+/**
+ * Every service a role touches, embedded but real — no PGlite boot for a role-wiring test.
+ *
+ * `root` is a parameter and not a constant: each test file owns its own directory and deletes it,
+ * so two files sharing one on-disk storage root cannot leave the other's fixture half-removed.
+ */
+export function fixtureRuntime(root: string): RunningServices {
+  const services = resolveServices(root, {});
+  const transport = new InProcessTransport();
+  return {
+    services,
+    db: { async ping() {}, async close() {} } as unknown as RunningServices['db'],
+    jobs: createMemoryDriver(),
+    // A real store, not a stub: the `worker` role starts the outbox relay against it, and a relay
+    // whose `claim()` rejects on the first 200ms tick is an unhandled rejection in whichever test
+    // happens to still be running.
+    outbox: createMemoryOutboxStore(),
+    events: createMemoryEventBus(),
+    transport,
+    transportDetail: 'in-process fanout',
+    // The sync role reads this to build its `PresenceRegistry`; the default is what a boot with no
+    // `NATS_URL` resolves to, so the fixture is the real number rather than a rounder one.
+    presenceTtlMs: DEFAULT_PRESENCE_TTL_MS,
+    storage: defineStorage({ disks: { local: localDriver({ root: `${root}/storage` }) } }),
+    mail: createMemoryMailDriver(),
+    mailDetail: 'embedded',
+    purge: noopPurgeDriver(),
+    purgeDetail: 'none',
+    stop: async () => transport.close(),
+  };
+}
+
+/**
+ * The process-global state one started role leaves behind. `resetLifecycle` is the load-bearing
+ * one: core's lifecycle is process-wide and a stopped server leaves it drained, so without it the
+ * SECOND web role in a file answers every request `X_DRAINING` — a suite that only passes when its
+ * tests are run one at a time. `@ultimat3/http`'s own server suite resets it for the same reason.
+ */
+export function resetDevRolesState(): void {
+  resetJobs();
+  resetJobsFacade();
+  resetTasks();
+  resetLifecycle();
+}

package/src/dev-roles.ts

@@ -8,27 +8,28 @@
 
 import type { Role } from '@ultimat3/core';
 import { createContext, isRole, logger, ROLES } from '@ultimat3/core';
-import type { Route, ServerHandle } from '@ultimat3/http';
-import { createServer, defineHttpConfig } from '@ultimat3/http';
-import type { Scheduler, Worker } from '@ultimat3/jobs';
-import { createScheduler, createWorker } from '@ultimat3/jobs';
-import { listQueries } from '@ultimat3/query';
+import type { Route, ServerHandle, ServerHooks } from '@ultimat3/http';
+import { configuredAuthenticator, createServer, defineHttpConfig } from '@ultimat3/http';
+import type { OutboxRelay, Scheduler, Worker } from '@ultimat3/jobs';
 import {
-  ChannelHub,
-  createSyncNode,
-  LiveQueryRegistry,
-  listenSyncNode,
-  liveQueryDefinition,
-  PresenceRegistry,
-  RingChangeBuffer,
-  SocketRegistry,
-} from '@ultimat3/realtime';
+  createOutboxRelay,
+  createPgLeaseLeader,
+  createScheduler,
+  createWorker,
+  jobDriver,
+  pgSchedulerState,
+} from '@ultimat3/jobs';
 import { devHooks } from './dev-hooks';
+import { pgExecutorFor } from './dev-queue';
 import type { RunningReplicator } from './dev-replicator';
 import { startReplicator } from './dev-replicator';
 import type { RunningServices } from './dev-runtime';
 import type { Env } from './dev-services';
-import { BadFlagError } from './errors';
+import { startSync } from './dev-sync';
+import { BadFlagError, PortInvalidError, RuntimeDriverSplitError } from './errors';
+import { DEFAULT_METRICS_PORT, startMetricsEndpoint } from './metrics-endpoint';
+import type { RuntimeOverrides } from './runtime-overrides';
+import { inlineStyleSources } from './style-csp';
 
 /** The roles `x dev` starts when `--role` names none, in boot order. */
 export const DEV_ROLES: readonly Role[] = ['web', 'sync', 'worker', 'scheduler'];
@@ -57,6 +58,36 @@ export interface StartRolesOptions {
    * probe, which is the same failure in four costumes.
    */
   readonly http?: WebBinding;
+  /**
+   * The app's `auth.signInPath`. Threaded rather than read from the config here because
+   * `startRoles` takes plain values — a test starts a web role with no `app.config.ts` at all.
+   */
+  readonly signInPath?: string | null;
+  /**
+   * Inline `<style>` bodies this process serves that the app's own surfaces do not account for —
+   * `/_x`'s shell. The surfaces themselves are read from the stylesheet registry here rather than
+   * passed, so no caller of `startRoles` can ship a web server whose CSP blocks the pages it
+   * serves: that policy is what rendered every deployed app completely unstyled.
+   */
+  readonly inlineStyles?: readonly string[];
+  /**
+   * Non-fatal findings the browser overlay shows next to an error, for the request being answered.
+   * Only `x dev` supplies one — `serve.ts` boots through this same function and omits it, so a
+   * production process never has a diagnostic to call (axiom 6).
+   */
+  readonly devNotices?: ServerHooks['devNotices'];
+  /**
+   * Where the scrape listener binds. Defaults to `DEFAULT_METRICS_PORT`, except when `port` is 0
+   * — a caller asking the kernel for an ephemeral HTTP port is a test, and a test that grabbed
+   * 9090 would fail the next one to run beside it.
+   */
+  readonly metricsPort?: number;
+  /**
+   * What the host substituted for a boot decision. Read here for the three seams that are not
+   * services — the rate-limit store, the middleware chain and the sync authenticator — while the
+   * drivers themselves arrive already resolved on `runtime`.
+   */
+  readonly overrides?: RuntimeOverrides;
 }
 
 export interface WebBinding {
@@ -73,6 +104,8 @@ export interface RunningRoles {
   readonly url: string | null;
   /** Where the sync role accepts websockets; null when it was not selected. */
   readonly syncUrl: string | null;
+  /** `http://…` — the scrape base. Never null: every role publishes a signal worth scaling on. */
+  readonly metricsUrl: string;
   readonly server: ServerHandle | null;
   readonly worker: Worker | null;
   readonly scheduler: Scheduler | null;
@@ -114,95 +147,133 @@ export function selectRoles(flag: string | undefined): readonly Role[] {
   return SELECTABLE_ROLES.filter((role) => selected.includes(role));
 }
 
+/**
+ * A server whose route table demands an identity it has no way to resolve.
+ *
+ * `hooks.authenticate` is the only place an actor can come from, and `devHooks()` spreads nothing
+ * when the app never called `configureAuthenticator()`. So a process in that state boots clean,
+ * reports healthy, and refuses every valid session on every `auth: 'required'` route — which is
+ * exactly what the demo app did: sign-in issued a real cookie and the next page still said 401,
+ * while four unit tests over the app's own resolver stayed green because each installed a viewer
+ * by hand.
+ *
+ * A warning and not a throw, deliberately: `x new` scaffolds guarded routes before it scaffolds an
+ * authenticator, so an app in the minutes between the two is incomplete, not broken. It is loud,
+ * it names the code, and it prints the call that fixes it.
+ */
+function warnIfUnauthenticatable(routes: readonly Route[]): void {
+  if (configuredAuthenticator() !== undefined) return;
+  const guarded = routes.filter((route) => route.meta.auth === 'required');
+  if (guarded.length === 0) return;
+  logger.warn(
+    `X_CONFIG_INVALID: ${guarded.length} route(s) declare auth: 'required' and no authenticator is configured, so every request is anonymous and each of them refuses every session — fix: call configureAuthenticator() at module scope in a file under apps/*/, e.g. configureAuthenticator((request) => viewerFor(request.header('cookie')))`,
+  );
+}
+
+/**
+ * How many proxies append to `x-forwarded-for` between the client and this process, or `null`
+ * when nothing in front of it is trusted.
+ *
+ * Read from the environment and not from `app.config.ts`, for the reason `PORT` and `ROLE` are: it
+ * is a fact about the DEPLOYMENT — one image runs behind an ingress in one cluster and behind
+ * nothing on a laptop — and an app that hardcoded it would be wrong in one of the two. `x dev`
+ * sets neither and gets `trustProxy: false`, which is correct: there is no proxy.
+ *
+ * Without this seam a container behind an ingress reads `ctx.ip` as the ingress's own socket
+ * address on every request, so the rate limiter keys the entire fleet's anonymous traffic into ONE
+ * bucket and a single scanner 429s every real signup.
+ */
+export function trustedHopsFromEnv(env: Env): number | null {
+  const raw = env['TRUSTED_PROXY_HOPS']?.trim();
+  if (raw === undefined || raw === '') return null;
+  const hops = Number(raw);
+  // A malformed count is refused rather than defaulted: reading the header at the wrong index is
+  // trusting a value the client typed, which is the failure trusting a proxy exists to avoid.
+  if (!Number.isInteger(hops) || hops < 1 || hops > 16) {
+    throw new PortInvalidError({ value: raw, name: 'TRUSTED_PROXY_HOPS' });
+  }
+  return hops;
+}
+
 function startWeb(options: StartRolesOptions): ServerHandle {
+  warnIfUnauthenticatable(options.routes);
   const binding = options.http ?? DEV_BINDING;
+  const hops = trustedHopsFromEnv(options.env);
+  const store = options.overrides?.rateLimitStore;
   return createServer({
     routes: options.routes,
     role: 'web',
-    hooks: devHooks(),
+    hooks: devHooks(options.devNotices === undefined ? {} : { devNotices: options.devNotices }),
+    // Both seams `createServer` already had and `startRoles` passed neither of, so an app's own
+    // middleware could not reach the pipeline any process the framework boots actually runs.
+    ...(options.overrides?.middleware === undefined
+      ? {}
+      : { middleware: options.overrides.middleware }),
+    ...(store === undefined ? {} : { rateLimitStore: store }),
     config: defineHttpConfig({
       port: options.port,
       dev: binding.dev,
       buildId: options.buildId,
       hostname: binding.hostname,
+      signInPath: options.signInPath ?? null,
+      // One declaration, never half of one: `defineHttpConfig` refuses `trustProxy` without hops.
+      ...(hops === null ? {} : { trustProxy: true, trustedProxyHops: hops }),
+      // `scope` is mandatory since @ultimat3/http made an undeclared limiter a boot error: the
+      // old `'process'` default meant the shipped chart's three `web` replicas enforced
+      // `login: { limit: 5 }` as fifteen attempts, with `x verify` green. It is DERIVED from the
+      // store rather than hardcoded — a deployment that hands `runtime.rateLimitStore` a shared
+      // store is declaring the fleet-wide numbers, and `assertRateLimitScope` then holds the two
+      // halves together instead of a literal here quietly contradicting the store beside it.
+      rateLimit: { scope: store?.scope ?? 'process' },
+      // Hashes, never `'unsafe-inline'`: a `render: 'static'` page is a file on disk, so
+      // nothing can stamp a per-response nonce into it, but its body is fixed and a hash is a
+      // function of that body. Read after `loadApp` — importing the app IS what registered them.
+      security: {
+        csp: { extend: { 'style-src': inlineStyleSources(options.inlineStyles ?? []) } },
+      },
     }),
   }).start();
 }
 
 /**
- * Every read the app declared `live: true` becomes a subscribable query on this node, through
- * `@ultimat3/realtime`'s own bridge. A registry with nothing in it answers every live `subscribe`
- * with "no live query registered", which is a working socket serving no reads — and it is what
- * kept the row gate that decides per subscriber from ever running outside a unit test.
+ * The one moment the boot can still see both answers.
  *
- * The context is the node's, and it carries no actor: it supplies the services and the clock the
- * shared read needs, never an authority. Who may subscribe, and which rows they see, is decided
- * per socket at subscribe time and again for every row of every delivery.
- */
-function registerLiveQueries(options: StartRolesOptions): LiveQueryRegistry {
-  const registry = new LiveQueryRegistry({
-    source: new RingChangeBuffer(),
-    // A withheld row is a metric, never a frame and never an error: telling a client "there is a
-    // row you may not see" is the leak the gate exists to prevent.
-    onRowDenied: (event) => logger.debug('live.rows_denied', { ...event }),
-  });
-  const ctx = createContext({ role: 'sync', buildId: options.buildId });
-  for (const target of listQueries()) {
-    if (target.isLive) registry.register(liveQueryDefinition(target, { ctx }));
-  }
-  return registry;
-}
-
-/**
- * The sync role owns its own socket: websockets and the request pipeline drain differently.
+ * `startServices` captures the drivers it built; `loadApp` imports the app's modules after it, and
+ * a module calling `setJobDriver()` at import time moves the ambient slot and leaves the capture
+ * alone. From here on the two are indistinguishable at every call site: `handle.enqueue()` reads
+ * the ambient one, `createWorker` claims from the captured one, and `/_x` reads the ambient one —
+ * so the dashboard agrees with the enqueue side and disagrees with reality.
  *
- * Port 0 is passed straight through rather than incremented — `+ 1` would ask the kernel for
- * port 1 instead of an ephemeral one — and the reported url is the listener's own bound address,
- * never a string built from the port that was requested.
+ * Refused, not reconciled. Reading through the accessor would make the split invisible instead of
+ * impossible, and the app would still have installed a driver the boot never saw — no outbox store
+ * bound to it, no relay draining it. The fix line names the field that does work.
  */
-async function startSync(
-  options: StartRolesOptions,
-): Promise<{ url: string; stop: () => Promise<void> }> {
-  const sockets = new SocketRegistry();
-  const hub = new ChannelHub({ transport: options.runtime.transport, sockets });
-  const node = createSyncNode({
-    hub,
-    registry: registerLiveQueries(options),
-    transport: options.runtime.transport,
-    buildId: options.buildId,
-    sockets,
-    // Tier 1 is presence, and without a registry the node answers a topic subscribe with no member
-    // list at all — the KV bucket the transport just created would hold nothing and every `sync`
-    // container would run a presence-less protocol. It reads and writes `transport.shared`, so it
-    // is exactly as multi-node as the transport behind it: in-process here, the bucket under NATS.
-    presence: new PresenceRegistry({
-      transport: options.runtime.transport,
-      hub,
-      ttlMs: options.runtime.presenceTtlMs,
-    }),
+function assertOneJobDriver(runtime: RunningServices): void {
+  const ambient = jobDriver();
+  if (ambient === undefined || ambient === runtime.jobs) return;
+  throw new RuntimeDriverSplitError({
+    driver: 'jobs',
+    ambient: ambient.name,
+    captured: runtime.jobs.name,
   });
-  await node.start();
-  try {
-    const listener = listenSyncNode(node, { port: options.port === 0 ? 0 : options.port + 1 });
-    return {
-      url: listener.url,
-      stop: async () => {
-        listener.stop();
-        await node.stop();
-      },
-    };
-  } catch (error) {
-    await node.stop();
-    throw error;
-  }
 }
 
 export async function startRoles(options: StartRolesOptions): Promise<RunningRoles> {
   const selected = options.roles;
+  assertOneJobDriver(options.runtime);
   // Roles bind sockets in order, so a role that fails to start has to release the ones before it.
   // Without this a failed `sync` leaves the web server bound and unreachable by any caller.
   const started: (() => Promise<void>)[] = [];
   try {
+    // First, and for every role rather than only the two that open an HTTP socket: `worker` and
+    // `sync` are precisely the roles whose HPAs read a series the process itself has to publish,
+    // and a `worker` container with no listener is an HPA pinned at `<unknown>` forever.
+    const metrics = startMetricsEndpoint({
+      port: options.metricsPort ?? (options.port === 0 ? 0 : DEFAULT_METRICS_PORT),
+      ...(options.http === undefined ? {} : { hostname: options.http.hostname }),
+    });
+    started.push(async () => metrics.stop());
+
     const server = selected.includes('web') ? startWeb(options) : null;
     if (server !== null) started.push(() => server.stop());
 
@@ -218,8 +289,37 @@ export async function startRoles(options: StartRolesOptions): Promise<RunningRol
     worker?.start();
     if (worker !== null) started.push(() => worker.stop('x dev stopped'));
 
+    // The half of the transactional outbox that makes a staged row a running job. Without it
+    // `handle.enqueue()` inside a transaction writes to `x_outbox` and nothing ever reads it back
+    // — every enqueue in a request handler silently becomes a job that never runs.
+    //
+    // On `worker`, and on `worker` alone: it is the role that exists wherever jobs run at all, and
+    // a relay is safe to duplicate (publish-then-mark is at-least-once and the idempotency key
+    // collapses the repeat) but pointless to spread. A deployment with no `worker` has no one to
+    // run the jobs either way.
+    const relay: OutboxRelay | null = selected.includes('worker')
+      ? createOutboxRelay({ store: options.runtime.outbox, driver: options.runtime.jobs })
+      : null;
+    relay?.start();
+    // Returned, not called-and-discarded: `stop()` waits out the pass in flight, and an unawaited
+    // one hands the failure rollback the same window a dropped `await` gives the teardown below.
+    if (relay !== null) started.push(() => relay.stop());
+
+    // `state` and `leader`, not the defaults. `createMemorySchedulerState` forgets every watermark
+    // on restart, so a rolling deploy re-fires or skips whatever was due across it, and
+    // `soleLeader()` makes every replica the leader — three `scheduler` pods, three of every task.
+    //
+    // `createPgLeaseLeader` and NOT `createPgLeader`: the latter's `pg_try_advisory_lock` is
+    // SESSION-scoped, and the session ends the moment the connection goes back to the pool, so
+    // every node reads itself as leader anyway. An expiring row is correct on the executor this
+    // package is actually handed.
+    const executor = pgExecutorFor(options.runtime.db);
     const scheduler = selected.includes('scheduler')
-      ? createScheduler({ driver: options.runtime.jobs })
+      ? createScheduler({
+          driver: options.runtime.jobs,
+          state: pgSchedulerState(executor),
+          leader: createPgLeaseLeader({ executor }),
+        })
       : null;
     scheduler?.start();
     if (scheduler !== null) started.push(() => scheduler.stop());
@@ -240,6 +340,7 @@ export async function startRoles(options: StartRolesOptions): Promise<RunningRol
       roles: selected,
       url: server === null ? null : server.url(),
       syncUrl: sync?.url ?? null,
+      metricsUrl: metrics.url,
       server,
       worker,
       scheduler,
@@ -248,9 +349,16 @@ export async function startRoles(options: StartRolesOptions): Promise<RunningRol
         // Reverse boot order, so the slot is released before the bus it published to closes.
         await replicator?.stop();
         await scheduler?.stop();
+        // Before the worker, so nothing publishes into a queue whose consumer has already gone —
+        // and AWAITED, because a pass is a `driver.enqueue` followed by a `markPublished`. Dropped,
+        // this returns between the two and the lines below close the pool under the row it was
+        // about to mark: re-published next boot at best, a rejection against a closed pool at worst.
+        await relay?.stop();
         await worker?.stop('x dev stopped');
         await sync?.stop();
         await server?.stop();
+        // Last: a scrape taken while the roles above drain is the one that explains the drain.
+        metrics.stop();
       },
     };
   } catch (error) {

package/src/dev-runtime.ts

@@ -4,32 +4,36 @@
 // boot installs, only backed by embedded drivers.
 
 import { mkdirSync } from 'node:fs';
-import { join } from 'node:path';
 import type { PurgeDriver } from '@ultimat3/cache';
-import {
-  createCdnTier,
-  isNoopPurgeDriver,
-  registerTier,
-  resetTiers,
-  selectPurgeDriver,
-} from '@ultimat3/cache';
-import type { EventBus, JobDriver } from '@ultimat3/jobs';
-import { createMemoryEventBus, setEventBus } from '@ultimat3/jobs';
+import { isNoopPurgeDriver, selectPurgeDriver } from '@ultimat3/cache';
+import { isLocal, resolveEnvironment } from '@ultimat3/core';
+import type { EventBus, JobDriver, OutboxStore } from '@ultimat3/jobs';
 import type { MailDriver } from '@ultimat3/mail';
-import { isMemoryDriver, resetMailDriver, selectMailDriver, setMailDriver } from '@ultimat3/mail';
+import {
+  isMemoryDriver,
+  isUnconfiguredDriver,
+  resetMailDriver,
+  selectMailDriver,
+  setMailDriver,
+} from '@ultimat3/mail';
 import type { Transport, TransportSelection } from '@ultimat3/realtime';
 import { selectTransport } from '@ultimat3/realtime';
 import type { Storage } from '@ultimat3/storage';
-import { defineStorage, localDriver } from '@ultimat3/storage';
+import { defineStorage, localDriver, s3Driver, usesDevStorageSecret } from '@ultimat3/storage';
+import { startCacheTiers } from './dev-cache';
 import type { DevDbClient } from './dev-queue';
 import { startQueue } from './dev-queue';
 import type { DevServices, Env } from './dev-services';
+import { LocalDiskUnsafeError, StorageUnwritableError } from './errors';
 import { msg } from './messages';
+import type { RuntimeOverrides } from './runtime-overrides';
 
 export interface RunningServices {
   readonly services: DevServices;
   readonly db: DevDbClient;
   readonly jobs: JobDriver;
+  /** The `x_outbox` store behind `handle.enqueue()`. A role starts the relay that drains it. */
+  readonly outbox: OutboxStore;
   readonly events: EventBus;
   readonly transport: Transport;
   readonly storage: Storage;
@@ -62,6 +66,10 @@ export interface RunningServices {
  * the human half.
  */
 export function describeMail(runtime: RunningServices): string {
+  // Three arms, not two. A deployment with no credential outside development installs a driver that
+  // REFUSES every send, and reporting that as `external` would name it as a transport that delivers
+  // — the same lie the memory driver told when it answered `accepted` for mail nobody received.
+  if (isUnconfiguredDriver(runtime.mail)) return `mail=refused(${runtime.mailDetail})`;
   return isMemoryDriver(runtime.mail)
     ? 'mail=embedded'
     : `mail=external(${runtime.mail.name} via ${runtime.mailDetail})`;
@@ -84,6 +92,8 @@ export function describeCdn(runtime: RunningServices): string {
  * value stays where `--json` can depend on it.
  */
 export function mailLabel(runtime: RunningServices): string {
+  if (isUnconfiguredDriver(runtime.mail))
+    return msg('cli.dev.mail.refused', { detail: runtime.mailDetail });
   return isMemoryDriver(runtime.mail)
     ? msg('cli.dev.mail.embedded')
     : msg('cli.dev.mail.external', { driver: runtime.mail.name, detail: runtime.mailDetail });
@@ -98,13 +108,71 @@ export function cdnLabel(runtime: RunningServices): string {
 
 const FILE_SCHEME = 'file://';
 
-function startStorage(services: DevServices): Storage {
+/**
+ * The storage disk this process will use.
+ *
+ * An `external` binding now reaches `s3Driver`. It used to fall through to a LOCAL directory —
+ * `S3_ENDPOINT` selected a different root and nothing else, so every deployment that configured
+ * object storage silently wrote to a container-local disk, and every upload was destroyed by the
+ * next restart while the configured bucket stayed empty. Nothing failed; the files just went
+ * nowhere.
+ *
+ * The embedded branch is also where a hardened container died. `mkdirSync` on a
+ * `readOnlyRootFilesystem` throws a bare `EROFS` from inside Bun's fs, with no code, no fix and no
+ * mention of storage — the demo CrashLooped 22 times on it. A read-only filesystem is a normal way
+ * to run a container, so this reports what to do instead of what went wrong.
+ */
+export function startStorage(services: DevServices, env: Env, override?: Storage): Storage {
+  // First arm, not a branch beside the two below: a host that handed the boot a `Storage` has
+  // already answered "which disk", and re-deriving one from `S3_ENDPOINT` would be a second answer.
+  if (override !== undefined) return override;
   const binding = services.storage;
-  const root =
-    binding.mode === 'embedded'
-      ? binding.url.slice(FILE_SCHEME.length)
-      : join(services.stateDir, 'storage');
-  mkdirSync(root, { recursive: true });
+
+  if (binding.mode === 'external') {
+    const bucket = env['S3_BUCKET'] ?? '';
+    if (bucket === '') {
+      throw new StorageUnwritableError(
+        `S3_ENDPOINT is set to ${binding.url} but S3_BUCKET is empty, so there is no disk to write to`,
+        'set S3_BUCKET to the bucket name, or unset S3_ENDPOINT to use the embedded disk',
+      );
+    }
+    return defineStorage({
+      disks: {
+        object: s3Driver({
+          bucket,
+          endpoint: binding.url,
+          ...(env['S3_REGION'] === undefined ? {} : { region: env['S3_REGION'] }),
+          // MinIO needs path style; R2 and AWS do not. Declared, never sniffed from the endpoint.
+          forcePathStyle: env['S3_FORCE_PATH_STYLE'] === '1',
+        }),
+      },
+      default: 'object',
+    });
+  }
+
+  const root = binding.url.slice(FILE_SCHEME.length);
+  // The embedded disk is what this branch falls back to whenever `S3_ENDPOINT`/`S3_BUCKET` are
+  // unset — including in `production` and `staging`, where an unset `STORAGE_SIGNING_SECRET` means
+  // every signed upload grant is minted with the string published in this repo, and
+  // `acceptSignedUpload` trusts a signed constraint over the app's own `uploadPolicy`.
+  //
+  // `localDriver` refuses that at construction on its own (`X_ENV_MISSING`). Pre-empted here so
+  // the message names the STORAGE CHOICE and both ways out, rather than arriving as "a secret is
+  // missing" from a helper the operator never configured. Not an outright ban on the local disk in
+  // production: a single-node Compose deploy on a mounted volume WITH a real secret is a rung on
+  // the scale ladder, and refusing it would be a deploy-shape decision, not a security fix.
+  if (!isLocal() && usesDevStorageSecret()) {
+    throw new LocalDiskUnsafeError({ environment: resolveEnvironment(), root });
+  }
+  try {
+    mkdirSync(root, { recursive: true });
+  } catch (cause) {
+    const detail = cause instanceof Error ? cause.message : String(cause);
+    throw new StorageUnwritableError(
+      `the embedded storage disk needs ${root} and it could not be created: ${detail}`,
+      `mount a writable volume at ${root}, or set S3_ENDPOINT and S3_BUCKET to use object storage instead`,
+    );
+  }
   return defineStorage({ disks: { local: localDriver({ root }) }, default: 'local' });
 }
 
@@ -126,7 +194,11 @@ async function release(steps: readonly (() => void | Promise<void>)[]): Promise<
   return failures;
 }
 
-export async function startServices(services: DevServices, env: Env): Promise<RunningServices> {
+export async function startServices(
+  services: DevServices,
+  env: Env,
+  overrides?: RuntimeOverrides,
+): Promise<RunningServices> {
   // Before the queue: selection is pure — it parses `SMTP_URL` and builds a transport, it does
   // not dial. A typo'd credential must fail on the spot rather than after PGlite has started and
   // been unwound again, and it must fail at boot rather than on the first mail nobody receives.
@@ -140,49 +212,54 @@ export async function startServices(services: DevServices, env: Env): Promise<Ru
   // `@ultimat3/realtime`'s decision, and it is the same call a `ROLE=sync` container makes, so this
   // process cannot resolve the bus differently from the container it stands in for.
   const bus: TransportSelection = selectTransport(env);
-  const queue = await startQueue(services);
-  const { db, jobs } = queue;
+  const queue = await startQueue(services, overrides);
+  const { db, jobs, outbox, events } = queue;
   // Boot is a sequence of external resources, and every step after the first can reject — the
   // queue is already up, so from here an unwind must release it exactly like everything after it.
   const started: (() => void | Promise<void>)[] = [() => queue.stop()];
   try {
-    const events = createMemoryEventBus();
-    setEventBus(events);
     // Dialled here rather than at selection: an unreachable bus must fail at `x dev`, not on the
     // first change nobody receives, and the socket is a resource the unwind below has to release.
-    await bus.connect();
-    started.push(() => bus.transport.close());
-    const storage = startStorage(services);
+    // A supplied transport is already connected and is NOT closed here: whoever built it owns its
+    // socket, which is why the override skips both halves rather than only the dial.
+    let transport = overrides?.transport;
+    if (transport === undefined) {
+      await bus.connect();
+      started.push(() => bus.transport.close());
+      transport = bus.transport;
+    }
+    const storage = startStorage(services, env, overrides?.storage);
     // With no credential this is the memory driver: caught, not sent, so the `/_x` mail panel can
     // show what a template renders in every locale without a mailbox or a message escaping to a
     // real address. `SMTP_URL` or `RESEND_API_KEY` makes it a real transport instead — the same
     // "an unset variable means the embedded default" law the other three bindings follow.
-    const mail = selection.driver;
+    const mail = overrides?.mail ?? selection.driver;
     setMailDriver(mail);
     started.push(() => resetMailDriver());
-    // Registered only when a credential named a real edge. A noop tier would put a `cdn` line in
-    // every invalidation report claiming keys an edge that does not exist had accepted — and the
-    // `/_x` cache panel renders those reports, so the lie would be the thing an agent reads.
-    // Released with `resetTiers()`, which drops the whole registry: this boot is the only thing
-    // that registers one, and a tier left behind would purge for a process that has stopped.
-    const purging = !isNoopPurgeDriver(cdn.driver);
-    if (purging) {
-      registerTier(createCdnTier({ purge: cdn.driver }));
-      started.push(() => resetTiers());
-    }
+    const purge = overrides?.purge ?? cdn.driver;
+    // Every tier this process reads through, plus the cross-instance invalidation hop, in one
+    // call. The CDN tier used to be the only one registered here — so `createRedisTier`,
+    // `createLruTier` and `createMemoTier` shipped with no caller at all and every replica
+    // recomputed every cached read. Released with `resetTiers()`, which drops the whole registry:
+    // this boot is the only thing that registers one, and a tier left behind would purge for a
+    // process that has stopped.
+    started.push(startCacheTiers({ env, purge, transport }));
 
     return {
       services,
       db,
       jobs,
+      outbox,
       events,
-      transport: bus.transport,
+      transport,
       storage,
       mail,
       mailDetail: selection.detail,
-      transportDetail: bus.detail,
+      // The env key that selected the bus — or the honest answer that no env key did, because a
+      // boot line reading `NATS_URL` over a transport the host handed in is a lie a script parses.
+      transportDetail: overrides?.transport === undefined ? bus.detail : 'runtime override',
       presenceTtlMs: bus.presenceTtlMs,
-      purge: cdn.driver,
+      purge,
       purgeDetail: cdn.detail,
       // The same list the boot unwind uses, in the same reverse order, so a service added to the
       // boot is released by both paths — a second copy of these steps is how one of them came to