@everystack/cli 0.4.98 → 0.4.99

package/src/cli/commands/runbook.ts

@@ -13,6 +13,7 @@
  *   everystack runbook --force    overwrite hand-edits inside generated blocks
  */
 
+import { linkModels, ModelLinkError } from '@everystack/model';
 import fs from 'node:fs';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
@@ -46,8 +47,10 @@ async function loadDataModel(root: string, models: { dir: string | null; hasInde
     if (!Array.isArray(models)) {
       return { dataModel: null, warning: `models/index.ts exports no \`models\` array — data-model chapter skipped.` };
     }
+    linkModels(models);
     return { dataModel: renderDataModelSection(models), warning: null };
   } catch (err: any) {
+    if (err instanceof ModelLinkError) throw err;
     return {
       dataModel: null,
       warning: `Could not load models from ${barrel} (${err.message}) — data-model chapter skipped.`,

package/src/cli/connection-fence.ts

@@ -0,0 +1,351 @@
+/**
+ * The connection fence: a single-connection postgres.js client that fails CLOSED when its
+ * backend dies.
+ *
+ * WHY. A mutation holds a session advisory lock (the mutation lease) and often an open
+ * transaction on one backend. When that backend dies (failover, `pg_terminate_backend`, a
+ * network drop), the lock and the transaction die with it. Two driver behaviours then turn a
+ * clean failure into something worse (postgres.js 3.4.9, the pinned and latest version):
+ *
+ * 1. Inside `sql.begin` or `sql.reserve`, the next statement is handed to the CLOSED connection.
+ *    The driver schedules a socket write on a null socket, which throws `Cannot read properties
+ *    of null (reading 'write')` outside any promise (an uncaught exception), and the statement
+ *    never settles. `sql.begin` does this even when our code issues nothing more: after the
+ *    body throws it sends its own ROLLBACK down the same path. So no check inside a
+ *    `sql.begin` body can prevent it; this fence never uses `sql.begin` or `sql.reserve`.
+ * 2. At the pool level, the next statement silently RECONNECTS onto a new backend. That
+ *    backend holds no lease and is not in the transaction, so the rest of a "transaction"
+ *    would autocommit, unleased.
+ *
+ * HOW. Every statement goes through the pool (`sql.unsafe`) on a client created with
+ * `max: 1`, so there is one connection and the pool path never writes to a closed socket.
+ * The driver's public `onclose` option trips the fence the moment the connection closes; from
+ * then on nothing is handed to the driver again, so the reconnect never happens. Statements
+ * are dispatched one at a time, and each waiting caller is also released by the loss itself,
+ * so a statement the driver never settles still ends in a named `ConnectionLostError`.
+ * Transactions are plain `BEGIN` / `COMMIT` / `ROLLBACK` statements through the same fence.
+ *
+ * Three more rules close what `onclose` alone leaves open:
+ * - A rejection that is not a PostgreSQL statement error (no `severity`: ECONNRESET, EPIPE, the
+ *   driver's CONNECTION_* errors) or is a FATAL / PANIC trips the fence at once. On a TCP reset
+ *   the driver can reject the statement before its onclose runs, and nothing may be sent in
+ *   between. A failure of the connect phase (wrong database, role or password, refused port)
+ *   trips it too, but surfaces the driver's own error: nothing ran. Every other loss message
+ *   ends with the driver error that revealed it, `(cause: CODE: message)`.
+ * - What the loss says depends on WHEN it happened. Between statements: nothing was in flight,
+ *   an open transaction rolled back. With a statement in flight: it MAY HAVE COMMITTED (an
+ *   autocommit write, a hand-issued COMMIT), so `outcomeUnknown` is true. Inside `transaction()`
+ *   before its COMMIT was sent: rolled back. Neither message claims the lease is already free:
+ *   the server frees it when it notices the connection is gone.
+ * - A COMMIT that PostgreSQL answers with the ROLLBACK command tag (the transaction had already
+ *   failed) is a `TransactionRolledBackError`, never a success.
+ *
+ * THE WINDOW, stated. The fence is checked, then the driver dispatches the statement one
+ * microtask later (postgres.js runs a query's handler after `await 1`). The driver trips the
+ * fence synchronously inside its own `closed()`, the same function that nulls the socket, and
+ * `closed()` runs from a socket 'close' event, which cannot run inside a microtask gap. So a
+ * statement that passes the check reaches a connection that is still open; if it dies an
+ * instant later, `closed()` cancels the pending write and rejects the statement. Only our own
+ * synchronous code could close the socket inside the gap, and nothing here does.
+ */
+
+import type { QueryRunner } from './authz-contract';
+import { borrowedSessionRunner, type SessionOptions, type SessionRunner } from './session';
+
+/** The backend behind a fenced connection is gone; nothing more ran on it. */
+export class ConnectionLostError extends Error {
+  readonly code = 'CONNECTION_LOST';
+  /**
+   * True when a statement was IN FLIGHT as the connection died and it could have committed on
+   * its own: an autocommit write, or a COMMIT. The server may or may not have applied it. False
+   * when nothing was in flight, or when the lost statement sat inside a transaction whose
+   * COMMIT was never sent: that transaction rolled back with the backend.
+   */
+  readonly outcomeUnknown: boolean;
+  constructor(message: string, options: { outcomeUnknown?: boolean; cause?: unknown } = {}) {
+    // The CLI prints the message, never `.cause`: the driver's own words ride in the text.
+    const driverError = rootCause(options.cause);
+    super(driverError === undefined ? message : `${message} (cause: ${describeCause(driverError)})`);
+    this.name = 'ConnectionLostError';
+    this.outcomeUnknown = options.outcomeUnknown === true;
+    if (options.cause !== undefined) (this as { cause?: unknown }).cause = options.cause;
+  }
+}
+
+/** Follow `.cause` through wrapping ConnectionLostErrors to the driver's own error, if any. */
+function rootCause(cause: unknown): unknown {
+  let at = cause;
+  while (at instanceof ConnectionLostError) at = (at as { cause?: unknown }).cause;
+  return at;
+}
+
+/** `CODE: message` for a driver error, or its message alone when it has no code. */
+function describeCause(err: unknown): string {
+  if (!err || typeof err !== 'object') return String(err);
+  const { code, message } = err as { code?: unknown; message?: unknown };
+  const text = typeof message === 'string' ? message : String(err);
+  return typeof code === 'string' && code.length > 0 ? `${code}: ${text}` : text;
+}
+
+/**
+ * PostgreSQL answered COMMIT with ROLLBACK: the transaction had already failed (a statement
+ * error the body caught and carried on past), so nothing in it committed. PostgreSQL reports
+ * this with a command tag, not an error; without this check it reads as success.
+ */
+export class TransactionRolledBackError extends Error {
+  constructor(query: string) {
+    super(
+      `PostgreSQL rolled back the transaction instead of committing it (${query.trim().split(/\s+/)[0]} ` +
+      'returned ROLLBACK): an earlier statement in it failed, so NOTHING in the transaction committed.',
+    );
+    this.name = 'TransactionRolledBackError';
+  }
+}
+
+const LOST = 'database connection lost (the backend was terminated or the network dropped)';
+const LEASE_NOTE =
+  'The server releases the mutation lease when it notices the connection is gone; if a re-run ' +
+  'reports the lease is still held, that session is still alive: wait for it or terminate it.';
+
+/** Nothing was in flight: the statement was refused before it reached the driver. */
+const LOST_BETWEEN_MESSAGE =
+  `${LOST} between statements. Nothing more ran on it: any transaction still open rolled back ` +
+  `with the backend; statements that finished before the loss stand. ${LEASE_NOTE} Re-run the command.`;
+
+/** A statement was in flight and could have committed on its own. */
+const LOST_IN_FLIGHT_MESSAGE =
+  `${LOST} while a statement was in flight. That statement MAY HAVE COMMITTED: read the target ` +
+  `before re-running. Nothing more ran on the connection. ${LEASE_NOTE}`;
+
+/** Inside transaction(), before its COMMIT was sent: nothing in it can have committed. */
+const LOST_IN_TRANSACTION_MESSAGE =
+  `${LOST} inside a transaction, before its COMMIT was sent: the transaction rolled back with ` +
+  `the backend. ${LEASE_NOTE} Re-run the command.`;
+
+/** transaction()'s own COMMIT was in flight. */
+const LOST_AT_COMMIT_MESSAGE =
+  `${LOST} while COMMIT was in flight: the transaction MAY HAVE COMMITTED or may not. Read the ` +
+  `target before re-running. ${LEASE_NOTE}`;
+
+/**
+ * Does this rejection mean the CONNECTION is gone, not just the statement? A PostgreSQL error
+ * carries the server's `severity`; anything without one (ECONNRESET, EPIPE, ETIMEDOUT, the
+ * driver's CONNECTION_CLOSED / CONNECTION_DESTROYED / CONNECTION_ENDED / CONNECT_TIMEOUT) is the
+ * transport. A FATAL or PANIC is the server ending the session. A false positive fails closed,
+ * which is acceptable.
+ */
+function isConnectionFailure(err: unknown): boolean {
+  if (!err || typeof err !== 'object') return true;
+  const { severity } = err as { severity?: unknown };
+  if (typeof severity !== 'string') return true;
+  return severity === 'FATAL' || severity === 'PANIC';
+}
+
+/**
+ * Failures of the CONNECT phase: the connection never opened, so no statement can have run.
+ * The server FATALs a login can end in (no such database 3D000, no such role 28000, bad
+ * password 28P01, too many connections 53300, starting up 57P03), and the transport errors of
+ * reaching it at all. These still trip the fence, but the caller gets the driver's ORIGINAL
+ * error: "may have committed" would be false, and the server's text names what is wrong.
+ */
+const CONNECT_PHASE_CODES = new Set([
+  '3D000', '28000', '28P01', '53300', '57P03',
+  'ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'CONNECT_TIMEOUT',
+]);
+
+function isConnectPhaseFailure(err: unknown): boolean {
+  const code = (err as { code?: unknown } | null)?.code;
+  return typeof code === 'string' && CONNECT_PHASE_CODES.has(code);
+}
+
+/** COMMIT or its synonym END: the statements PostgreSQL may answer with a ROLLBACK tag. */
+const COMMIT_RE = /^\s*(COMMIT|END)\b/i;
+
+/** The loss signal, tripped by the driver's `onclose`. */
+export interface ConnectionFence {
+  /** The loss, once it happened; null while the connection is alive. */
+  readonly lost: ConnectionLostError | null;
+  /**
+   * Trip the fence. Idempotent; a no-op once the owner has ended the connection. `cause` is
+   * the driver error that revealed the loss, when one did; later refusals name it.
+   */
+  lose(cause?: unknown): void;
+  /**
+   * The value for postgres.js's `onclose` option. Chains `next` (a caller's own onclose), so
+   * installing the fence never silences another listener.
+   */
+  onclose(next?: (connectionId: number) => void): (connectionId: number) => void;
+}
+
+interface FenceInternals extends ConnectionFence {
+  ended: boolean;
+  /** Callers waiting on an in-flight statement; the loss releases each. */
+  listeners: Set<(cause?: unknown) => void>;
+}
+
+export function createConnectionFence(): ConnectionFence {
+  const fence: FenceInternals = {
+    lost: null,
+    ended: false,
+    listeners: new Set(),
+    lose(cause?: unknown) {
+      if (fence.lost || fence.ended) return;
+      // What every LATER statement is refused with: nothing of it was in flight.
+      (fence as { lost: ConnectionLostError | null }).lost = new ConnectionLostError(LOST_BETWEEN_MESSAGE, { cause });
+      const waiting = [...fence.listeners];
+      fence.listeners.clear();
+      for (const release of waiting) release();
+    },
+    onclose(next) {
+      return (connectionId: number) => {
+        fence.lose();
+        next?.(connectionId);
+      };
+    },
+  };
+  return fence;
+}
+
+/** The slice of a postgres.js client the fence drives. */
+export interface FenceableClient {
+  unsafe: (query: string) => PromiseLike<unknown>;
+  end: (options?: { timeout?: number }) => Promise<void>;
+}
+
+export interface FencedConnection {
+  /** One statement (autocommit unless the caller opened a transaction with `BEGIN`). */
+  runner: QueryRunner;
+  /** N statements in ONE transaction on this connection — see `SessionRunner`. */
+  session: SessionRunner;
+  /** `BEGIN`, the body, `COMMIT` — or `ROLLBACK` on a body error while the connection lives. */
+  transaction: <T>(fn: (run: QueryRunner) => Promise<T>) => Promise<T>;
+  /** Close the client. Statements after this are refused. */
+  end: () => Promise<void>;
+}
+
+const ISOLATION_SQL: Record<string, string> = {
+  'read uncommitted': 'READ UNCOMMITTED',
+  'read committed': 'READ COMMITTED',
+  'repeatable read': 'REPEATABLE READ',
+  serializable: 'SERIALIZABLE',
+};
+
+/** `SET TRANSACTION …` for the session options, or null when there is nothing to set. */
+function setTransactionSql(opts: SessionOptions | undefined): string | null {
+  const modes: string[] = [];
+  if (opts?.isolation !== undefined) {
+    const level = ISOLATION_SQL[String(opts.isolation).toLowerCase().trim()];
+    if (!level) {
+      throw new Error(`session isolation ${JSON.stringify(opts.isolation)} is not a PostgreSQL isolation level`);
+    }
+    modes.push(`ISOLATION LEVEL ${level}`);
+  }
+  if (opts?.readOnly) modes.push('READ ONLY');
+  return modes.length ? `SET TRANSACTION ${modes.join(', ')}` : null;
+}
+
+/**
+ * Wrap a single-connection postgres.js client (`max: 1`, created with `onclose:
+ * fence.onclose(...)`) so it fails closed on connection loss. See the file comment.
+ */
+export function fencedConnection(sql: FenceableClient, fenceIn: ConnectionFence): FencedConnection {
+  const fence = fenceIn as FenceInternals;
+  let tail: Promise<unknown> = Promise.resolve();
+
+  /** Check the fence, then hand the statement to the driver; the loss also releases the caller. */
+  const dispatch = (query: string): Promise<any[]> => {
+    if (fence.ended) return Promise.reject(new Error('this database connection has already ended'));
+    if (fence.lost) return Promise.reject(fence.lost);
+    return new Promise<any[]>((resolve, reject) => {
+      // The statement was handed to the driver, so the server may have run it: a loss from here
+      // on is IN FLIGHT, and its outcome is unknown.
+      const inFlightLoss = (cause?: unknown): ConnectionLostError =>
+        new ConnectionLostError(LOST_IN_FLIGHT_MESSAGE, { outcomeUnknown: true, cause });
+      const release = (cause?: unknown): void => reject(inFlightLoss(cause));
+      fence.listeners.add(release);
+      Promise.resolve(sql.unsafe(query)).then(
+        (rows) => {
+          fence.listeners.delete(release);
+          // Read the command tag before Array.from copies the rows and drops it.
+          const command = (rows as { command?: unknown } | null)?.command;
+          if (command === 'ROLLBACK' && COMMIT_RE.test(query)) {
+            reject(new TransactionRolledBackError(query));
+            return;
+          }
+          resolve(Array.from(rows as Iterable<any>));
+        },
+        (err) => {
+          fence.listeners.delete(release);
+          if (fence.lost || isConnectionFailure(err)) {
+            // Trip the fence NOW, before anything else can be sent. On a TCP reset Node emits
+            // 'error' before 'close', so the driver can reject this statement before its onclose
+            // runs; in that interval a catch-block ROLLBACK, a failure bookkeeping row or the
+            // lease release would otherwise reach a driver whose pool reconnects onto a NEW backend.
+            fence.lose(err);
+            // A connection that never opened ran nothing: keep the driver's own error.
+            reject(isConnectPhaseFailure(err) ? err : inFlightLoss(err));
+            return;
+          }
+          reject(err);
+        },
+      );
+    });
+  };
+
+  // One statement at a time: the next is not even checked until the previous settled, so two
+  // callers can never have a statement queued inside the driver when the connection dies.
+  const runner: QueryRunner = (query: string) => {
+    const turn = tail.then(() => dispatch(query), () => dispatch(query));
+    tail = turn.catch(() => undefined);
+    return turn;
+  };
+
+  const transaction = async <T>(fn: (run: QueryRunner) => Promise<T>): Promise<T> => {
+    await runner('BEGIN');
+    let result: T;
+    try {
+      result = await fn(runner);
+    } catch (err) {
+      // After a loss nothing is sent: the transaction already rolled back with the backend.
+      if (!fence.lost) await runner('ROLLBACK').catch(() => undefined);
+      // A body statement lost in flight cannot have committed: its COMMIT was never sent.
+      if (err instanceof ConnectionLostError) {
+        throw new ConnectionLostError(LOST_IN_TRANSACTION_MESSAGE, { cause: err });
+      }
+      throw err;
+    }
+    if (fence.lost) throw new ConnectionLostError(LOST_IN_TRANSACTION_MESSAGE, { cause: fence.lost });
+    try {
+      await runner('COMMIT');
+    } catch (err) {
+      if (err instanceof ConnectionLostError) {
+        throw err.outcomeUnknown
+          ? new ConnectionLostError(LOST_AT_COMMIT_MESSAGE, { outcomeUnknown: true, cause: err })
+          : new ConnectionLostError(LOST_IN_TRANSACTION_MESSAGE, { cause: err });
+      }
+      throw err;
+    }
+    return result;
+  };
+
+  const session: SessionRunner = async (statements, opts) => {
+    const setTransaction = setTransactionSql(opts);
+    return transaction(async (run) => {
+      if (setTransaction) await run(setTransaction);
+      // The borrowed runner pins search_path inside a savepoint and takes each allowFailure
+      // statement in its own savepoint: hand-issued, which is sound here because no driver
+      // transaction scope is keeping its own error bookkeeping.
+      return borrowedSessionRunner(run)(
+        statements,
+        opts?.searchPath !== undefined ? { searchPath: opts.searchPath } : undefined,
+      );
+    });
+  };
+
+  const end = async (): Promise<void> => {
+    fence.ended = true;
+    await sql.end({ timeout: 5 });
+  };
+
+  return { runner, session, transaction, end };
+}

package/src/cli/db-source.ts

@@ -26,6 +26,7 @@
 
 import type { QueryRunner } from './authz-contract';
 import { buildSearchPathSql, type SessionRunner, type SessionResult } from './session';
+import { createConnectionFence, fencedConnection } from './connection-fence';
 
 export type DbSource =
   | { kind: 'url'; url: string; from: 'flag' | 'env' | 'admin-env' | 'operator' }
@@ -152,13 +153,16 @@ export async function createUrlRunner(
   // minutes, resolving the in-flight query and THEN killing the session. Under reconcile's
   // BEGIN-across-calls transaction that is a silent session swap mid-batch (the reconcile
   // apply path also pins the backend pid and refuses bookkeeping if it moved).
-  const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => {}, ...sslDefaults(url) });
-  return {
-    runner: async (query: string) => Array.from(await sql.unsafe(query)),
-    session: sessionRunnerOver(sql),
-    transaction: async (fn) => sql.begin((tx: any) => fn(async (query: string) => Array.from(await tx.unsafe(query)))),
-    end: () => sql.end({ timeout: 5 }),
-  };
+  //
+  // Fenced (connection-fence.ts): every direct-lane mutation lease is taken on this connection,
+  // so a dead backend must fail CLOSED — a named ConnectionLostError, never a driver reconnect
+  // onto a new, unleased backend and never the driver's uncaught write on a closed socket.
+  // That is also why session and transaction are plain statements here, not `sql.begin`.
+  const fence = createConnectionFence();
+  const sql = postgres(url, {
+    max: 1, max_lifetime: null, onnotice: () => {}, ...sslDefaults(url), onclose: fence.onclose(),
+  });
+  return fencedConnection(sql, fence);
 }
 
 export interface UrlPipelineRunner {

package/src/cli/declared-derived.ts

@@ -12,7 +12,7 @@ import path from 'node:path';
 import fs from 'node:fs/promises';
 import { pathToFileURL } from 'node:url';
 import type { Module, ModelDescriptor, SequenceDescriptor, DerivedDescriptor } from '@everystack/model';
-import { moduleGovernedRoles } from '@everystack/model';
+import { linkModels, ModelLinkError, moduleGovernedRoles } from '@everystack/model';
 import { compileDerived } from './derived-compile';
 import { compileTableRenames, compileTableMoves } from './schema-compile';
 import type { SourceObject } from './derived-source';
@@ -69,19 +69,23 @@ export async function loadModulesFrom(modelsPath: string): Promise<Module[]> {
     try {
       mod = await import(pathToFileURL(abs).href);
     } catch (err: any) {
+      if (err instanceof ModelLinkError) throw err;
       throw new Error(`Could not load modules from ${modelsPath}: ${err.message}`);
     }
     if (Array.isArray(mod.modules)) {
       assertNoHoles(modelsPath, 'modules', mod.modules);
+      linkModels(mod.modules.flatMap((m: Module) => m.models));
       return mod.modules;
     }
     const models = mod.models ?? mod.default;
     if (Array.isArray(models)) {
       assertNoHoles(modelsPath, 'models', models);
+      linkModels(models);
       return [{ models, extensions: [], sequences: [], derived: [], governedRoles: [], functions: [] }];
     }
     throw new Error(`${modelsPath} must export a \`modules\` (or \`models\`) array.`);
   } catch (err) {
+    if (err instanceof ModelLinkError) throw err;
     throw asModelComposeError(modelsPath, err);
   }
 }
@@ -191,6 +195,7 @@ export function composeDeclaredDerived(modules: Module[], modelsPath: string): D
       extensions: [...new Set(modules.flatMap((m) => m.extensions ?? []))].sort(),
     };
   } catch (err) {
+    if (err instanceof ModelLinkError) throw err;
     throw asModelComposeError(modelsPath, err);
   }
 }

package/src/cli/derived-apply.ts

@@ -249,9 +249,26 @@ export function renderSetSearchPath(schemas: string[], local: boolean): string |
  */
 export function renderEnsureObjectSchemas(objects: SourceObject[]): string[] {
   const statements: string[] = [];
+  const usage = declaredObjectSchemaUsage(objects);
+  for (const [schema, roles] of Object.entries(usage)) {
+    statements.push(`CREATE SCHEMA IF NOT EXISTS ${quoteIdent(schema)}`);
+    if (roles.length > 0) {
+      statements.push(`GRANT USAGE ON SCHEMA ${quoteIdent(schema)} TO ${roles.join(', ')}`);
+    }
+  }
+  return statements;
+}
+
+/**
+ * The schema USAGE the declared objects imply, as data: every non-public schema an object lives
+ * in (sorted) → the sorted roles its objects grant anything to. The rule renderEnsureObjectSchemas
+ * renders into SQL (see A13 there for why PUBLIC is never included); exposed so a recorder — the
+ * backup's authz manifest — reads the SAME answer instead of re-deriving it.
+ */
+export function declaredObjectSchemaUsage(objects: SourceObject[]): Record<string, string[]> {
+  const usage: Record<string, string[]> = {};
   const schemas = [...new Set(objects.map((o) => o.schema))].filter((s) => s !== 'public').sort();
   for (const schema of schemas) {
-    statements.push(`CREATE SCHEMA IF NOT EXISTS ${quoteIdent(schema)}`);
     const roles = new Set<string>();
     for (const o of objects) {
       if (o.schema !== schema) continue;
@@ -262,11 +279,9 @@ export function renderEnsureObjectSchemas(objects: SourceObject[]): string[] {
         if (privileges.length > 0 && role.toUpperCase() !== 'PUBLIC') roles.add(role);
       }
     }
-    if (roles.size > 0) {
-      statements.push(`GRANT USAGE ON SCHEMA ${quoteIdent(schema)} TO ${[...roles].sort().join(', ')}`);
-    }
+    usage[schema] = [...roles].sort();
   }
-  return statements;
+  return usage;
 }
 
 // ---------------------------------------------------------------------------

package/src/cli/index.ts

@@ -317,7 +317,7 @@ Usage:
   everystack db:provision      --rotate --stage <name> [--slot api|ops|both] [--grace 60]  Rotate credentials live (A/B role swap, no deploy, no downtime)
   everystack db:snapshot       [--stage <name>] [--instance <id>]  Take a physical RDS snapshot (instant DR point; RDS only — use db:backup for portable logical backups)
   everystack db:snapshots      [--stage <name>] [--instance <id>]  List manual RDS snapshots for the instance
-  everystack db:backup         [--stage <name>]  Logical pg_dump of the DB → private S3 backups bucket (prints the backup id). Add --database-url <local> --stage <target> to dump a LOCAL database and upload it into the target stage's bucket as a restorable backup — the credential-free break-glass local→stage full replace (then: db:restore --from <id> --stage <target>)
+  everystack db:backup         [--stage <name>]  Logical pg_dump of the DB → private S3 backups bucket (prints the backup id). Add --database-url <local> --stage <target> to dump a LOCAL database and upload it into the target stage's bucket. Such a backup records no origin commit, so db:restore refuses it: restoring a local backup into a stage is closed until local backups can record a verified origin
   everystack db:backups        [--stage <name>]  List logical backups (id, size, created)
   everystack db:restore        --from <id> [--stage <name>] --confirm  Restore a backup INTO the stage's DB (DESTRUCTIVE)
   everystack db:backup:download <id> [--stage <name>]  Presigned URL to download a backup's dump (valid 1h)

package/src/cli/mutation-lease.ts

@@ -179,6 +179,11 @@ export function leaseSession(): string {
  * write of the verb. A verb that opens a SECOND connection mid-body has a writer the lease
  * does not cover.
  *
+ * That connection must fail CLOSED when its backend dies: hand this a fenced runner
+ * (connection-fence.ts — `createUrlRunner` and the server's mutation connection both are).
+ * An unfenced postgres.js pool reconnects onto a new backend after a loss, so the rest of the
+ * body would run without the lease; the lease's own release below would too.
+ *
  * HONEST LIMITATION (do not overstate): this is CONVENTIONALLY enforced, not runtime-enforced.
  * A PostgreSQL connection carries no read/write intent the connection layer can inspect, so
  * "reject a second connection" is unrealizable in-process. The counterexample is already in the

package/src/cli/plain-output.ts

@@ -77,6 +77,16 @@ function head(statement: string): string {
   return body === '' ? statement.replace(/\s+/g, ' ').trim() : body;
 }
 
+/**
+ * Match report marks by the whole statement with ONLY outer whitespace removed, on both sides.
+ * `head` is a description, not an identity: collapsing whitespace can change literals and
+ * function bodies. Internal formatting, comments, case and terminators must still match exactly.
+ * This key never replaces the original SQL passed to descriptions or returned to callers.
+ */
+function statementKey(statement: string): string {
+  return statement.trim();
+}
+
 /** The whole statement as one line, comments stripped and truncated — for the fallback description. */
 function oneLine(statement: string): string {
   const body = head(statement).replace(/;$/, '');
@@ -339,7 +349,8 @@ function objectKey(s: string): { verb: 'drop' | 'create'; key: string } | null {
  * definition edit. So the drop is removed and its create is marked `rebuilt`, which renders as
  * the update line it always was.
  *
- * Statements this does not recognise pass through untouched, in order.
+ * Statements this does not recognise pass through untouched, in order. Returned statements keep
+ * their original text; `rebuilt` holds statementKey values (outer whitespace removed).
  */
 export function collapseRebuilds(statements: readonly string[]): {
   statements: string[];
@@ -355,7 +366,7 @@ export function collapseRebuilds(statements: readonly string[]): {
   for (const s of statements) {
     const k = objectKey(head(s));
     if (k?.verb === 'drop' && created.has(k.key)) {
-      rebuilt.add(created.get(k.key)!);
+      rebuilt.add(statementKey(created.get(k.key)!));
       continue;
     }
     kept.push(s);
@@ -378,8 +389,9 @@ const plural = (n: number, one: string, many: string): string => (n === 1 ? one
 
 /** One change line, indented, already carrying its mark. */
 function changeLine(statement: string, destructive: ReadonlySet<string>, rebuilt?: ReadonlySet<string>): string {
-  if (destructive.has(statement)) return `  ${DESTRUCTIVE_MARK} — ${describeDestructive(statement)}`;
-  const { symbol, text } = describeChange(statement, { rebuilt: rebuilt?.has(statement) === true });
+  const key = statementKey(statement);
+  if (destructive.has(key)) return `  ${DESTRUCTIVE_MARK} — ${describeDestructive(statement)}`;
+  const { symbol, text } = describeChange(statement, { rebuilt: rebuilt?.has(key) === true });
   return `  ${symbol} ${text}`;
 }
 
@@ -398,7 +410,7 @@ function changeLines(
 export interface PlanReportInput {
   /** The real DDL the plan will run — notices and held drops are not changes. */
   executable: readonly string[];
-  /** The destructive subset, as the apply gate classifies it. Must be members of `executable`. */
+  /** The apply gate's destructive subset. Must match `executable` apart from outer whitespace. */
   destructive: readonly string[];
   /** What the plan is aimed at, in the reader's words: a stage name, or `local`. */
   target: string;
@@ -415,7 +427,7 @@ export function renderPlanReport(input: PlanReportInput): string[] {
   const { executable, target, stage, planPath, verbose = false } = input;
   if (executable.length === 0) return ['Up to date. Nothing to plan.'];
 
-  const destructive = new Set(input.destructive);
+  const destructive = new Set(input.destructive.map(statementKey));
   const lines: string[] = [
     `Plan: ${executable.length} ${plural(executable.length, 'change', 'changes')} (${target})`,
     '',
@@ -440,6 +452,7 @@ export function renderPlanReport(input: PlanReportInput): string[] {
 
 export interface ApplyReceiptInput {
   executable: readonly string[];
+  /** The apply gate's destructive subset. Must match `executable` apart from outer whitespace. */
   destructive: readonly string[];
   /** The target was already where the plan ends — an idempotent re-apply. */
   alreadyApplied?: boolean;
@@ -461,7 +474,7 @@ export function renderApplyReceipt(input: ApplyReceiptInput): string[] {
   const n = executable.length;
   const tally = `Applied ${n} ${plural(n, 'change', 'changes')}.`;
   if (n === 0 || n > MAX_RECEIPT_CHANGES) return [tally];
-  return [...changeLines(executable, new Set(input.destructive), MAX_RECEIPT_CHANGES, rebuilt), '', tally];
+  return [...changeLines(executable, new Set(input.destructive.map(statementKey)), MAX_RECEIPT_CHANGES, rebuilt), '', tally];
 }
 
 export interface CatchUpFailureInput {
@@ -529,7 +542,16 @@ function sentence(text: string): string {
 }
 
 /** db:check's default output: one line, always. */
-export function renderCheckVerdict(pending: number): string[] {
-  if (pending === 0) return ['Up to date.'];
-  return [`${pending} ${plural(pending, 'change', 'changes')} pending. Run db:plan to review.`];
+export function renderCheckVerdict(pending: number, warnings = 0): string[] {
+  if (warnings === 0) {
+    if (pending === 0) return ['Up to date.'];
+    return [`${pending} ${plural(pending, 'change', 'changes')} pending. Run db:plan to review.`];
+  }
+  // Warnings never fail the gate, but a clean-looking line would hide them.
+  const warned = `${warnings} ${plural(warnings, 'warning', 'warnings')}`;
+  const readThem = `db:check --verbose to read ${warnings === 1 ? 'it' : 'them'}`;
+  if (pending === 0) return [`Up to date, ${warned}. Run ${readThem}.`];
+  return [
+    `${pending} ${plural(pending, 'change', 'changes')} pending, ${warned}. Run db:plan to review, db:check --verbose to read the ${plural(warnings, 'warning', 'warnings')}.`,
+  ];
 }

package/src/cli/runbook/model-section.ts

@@ -7,6 +7,7 @@
  * have — and an ability change shows up as runbook drift (`runbook --check`).
  */
 
+import { ModelLinkError } from '@everystack/model';
 import type { Ability, FieldSpec, ModelDescriptor, Relation } from '@everystack/model';
 
 function typeLabel(spec: FieldSpec): string {
@@ -35,7 +36,8 @@ function referenceTarget(spec: FieldSpec): string | null {
   try {
     const target = spec.references() as { table?: string } | undefined;
     return target?.table ?? 'unknown';
-  } catch {
+  } catch (err) {
+    if (err instanceof ModelLinkError) throw err;
     return 'unknown';
   }
 }
@@ -73,7 +75,8 @@ function relationLine(name: string, rel: Relation): string {
   let target = 'unknown';
   try {
     target = rel.target().table;
-  } catch {
+  } catch (err) {
+    if (err instanceof ModelLinkError) throw err;
     /* forward reference that can't resolve here — label stays unknown */
   }
   const shape = rel.kind === 'belongsTo' ? 'belongs to' : 'has many';