@everystack/cli 0.4.98 → 0.4.100

package/src/cli/commands/db-check.ts

@@ -44,6 +44,7 @@ import { loadDeclaredDerived, retiredSqlDirAnywhere, retiredSqlDirFlagRefusal, t
 import type { SequenceDescriptor } from '@everystack/model';
 import type { SourceObject } from '../derived-source';
 import { findReadAuthzGaps, findNakedGrants } from '../authz-lint';
+import { findSelectGrantGaps } from '../select-grant-advisory';
 import { parseBaseline, renderBaseline, BASELINE_FILE } from '../authz-baseline';
 import {
   findDerivedReadGaps, findSecdefExecuteGaps, findMatviewSnapshotWarnings, findPublicExecutableSecdef,
@@ -69,6 +70,14 @@ export interface CheckFinding {
   message: string;
 }
 
+/** The quiet default's one line: failing findings are "pending", warnings ride along and never fail the gate. */
+export function checkVerdictLines(findings: readonly CheckFinding[], ephemeralFailed: boolean): string[] {
+  return renderCheckVerdict(
+    findings.filter((f) => f.level === 'fail').length + (ephemeralFailed ? 1 : 0),
+    findings.filter((f) => f.level === 'warn').length,
+  );
+}
+
 export interface StaticCheckInput {
   modelsPath: string;
   /** Barrel siblings and what each declares — the "on disk but ungoverned" scan. */
@@ -197,6 +206,12 @@ export function runStaticChecks(input: StaticCheckInput): CheckFinding[] {
     findings.push({ level: 'warn', area: 'authz', message: naked.message });
   }
 
+  // A write grant with no SELECT beside it: every UPDATE/DELETE ... WHERE (and any RETURNING) by
+  // that role is "permission denied". WARN: the grant set is legal, only its use is broken.
+  for (const gap of findSelectGrantGaps(input.models)) {
+    findings.push({ level: 'warn', area: 'authz', message: gap.message });
+  }
+
   // The adoption baseline, ARTIFACT ONLY.
   //
   // This check cannot see the threat and must not pretend to. The foreign grantees it
@@ -494,9 +509,7 @@ export async function dbCheckCommand(flags: Record<string, string>): Promise<voi
     // every failing finding, plus the from-scratch build when it did not land where the models
     // say it should. `db:plan` is the verb that shows each one against a real target, so it is
     // the only command the line names. Re-run with --verbose to read the findings themselves.
-    for (const text of renderCheckVerdict(
-      findings.filter((f) => f.level === 'fail').length + (ephemeralFailed ? 1 : 0),
-    )) printLine(text);
+    for (const text of checkVerdictLines(findings, ephemeralFailed)) printLine(text);
   }
 
   if (failed) process.exit(1);

package/src/cli/commands/db-fork.ts

@@ -21,9 +21,10 @@
 
 import { resolveConfig, opsFunction, type CliConfig } from '../config';
 import { invokeAction, presignGet } from '../aws';
-import { keyForId, parseBackupRef, isProductionTier, crossStageGuard } from '../backup';
+import { keyForId, metaKey, parseBackupRef, isProductionTier, crossStageGuard } from '../backup';
 import { pollTaskUntilStopped } from '../task-poll';
 import { runStageBackup, stageBackupDeps } from '../stage-backup';
+import { printRestoreWitness } from './db-backup';
 import { step, success, fail, info, warn } from '../output';
 
 export type ForkGuardVerdict =
@@ -109,11 +110,16 @@ export async function dbForkCommand(flags: Record<string, string>): Promise<void
     info(`Backup ${backupId}.`);
   }
 
-  // Move 2: the presigned hand-off — the operator's credentials ARE the authorization.
-  step('Presigning the dump (1 hour)...');
+  // Move 2: the presigned hand-off — the operator's credentials ARE the authorization. The meta
+  // travels beside the dump: the target's Lambda reads the backup's origin commit from it (the
+  // restore witness), since it cannot read the source stage's bucket itself.
+  step('Presigning the dump and its meta (1 hour)...');
   let url: string;
+  let metaUrl: string;
   try {
-    url = await presignGet(source.region, source.backupsBucket, keyForId(backupId)!, 3600);
+    const key = keyForId(backupId)!;
+    url = await presignGet(source.region, source.backupsBucket, key, 3600);
+    metaUrl = await presignGet(source.region, source.backupsBucket, metaKey(key), 3600);
   } catch (err: any) {
     fail(`Presign failed: ${err.message}`);
     process.exit(1);
@@ -131,6 +137,7 @@ export async function dbForkCommand(flags: Record<string, string>): Promise<void
   step(`Restoring into ${target} (pg_restore in the Task; replaces its database)...`);
   const restored: any = await invokeAction(targetConfig.region, opsFunction(targetConfig), 'db:restore', {
     url,
+    metaUrl,
     confirm: true,
     actor: process.env.USER ?? null,
   });
@@ -138,6 +145,7 @@ export async function dbForkCommand(flags: Record<string, string>): Promise<void
     fail(`Restore failed: ${restored.error}`);
     process.exit(1);
   }
+  printRestoreWitness(restored, target);
   const rPoll = await pollTaskUntilStopped(targetConfig.region, opsFunction(targetConfig), { runId: restored.runId, taskArn: restored.taskArn });
   if (rPoll.outcome !== 'stopped' || rPoll.status.exitCode !== 0) {
     const why = rPoll.outcome === 'stopped' ? `exit ${rPoll.status.exitCode}${rPoll.status.stoppedReason ? ` — ${rPoll.status.stoppedReason}` : ''}` : rPoll.outcome;

package/src/cli/commands/db-generate.ts

@@ -23,6 +23,7 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
 import type { ModelDescriptor, Module } from '@everystack/model';
+import { linkModels } from '@everystack/model';
 import { introspectSchema } from '../schema-introspect';
 import { compileDrizzleSource } from '../schema-source';
 import { compileModuleMigration } from '../migration-compile';
@@ -60,7 +61,7 @@ export async function loadModels(modelsPath: string): Promise<ModelDescriptor[]>
     const models = mod.models ?? mod.default;
     if (!Array.isArray(models)) throw new Error(`${modelsPath} must export a \`models\` array (got ${typeof models}).`);
     assertNoHoles(modelsPath, 'models', models);
-    return models;
+    return linkModels(models);
   } catch (err) {
     throw asModelComposeError(modelsPath, err);
   }
@@ -80,9 +81,12 @@ export async function loadModules(modelsPath: string): Promise<Module[]> {
   } catch (err: any) {
     throw new Error(`Could not load modules from ${modelsPath}: ${err.message}`);
   }
-  if (Array.isArray(mod.modules)) return mod.modules;
+  if (Array.isArray(mod.modules)) {
+    linkModels(mod.modules.flatMap((module: Module) => module.models));
+    return mod.modules;
+  }
   const models = mod.models ?? mod.default;
-  if (Array.isArray(models)) return [{ models, extensions: [], sequences: [], derived: [], governedRoles: [], functions: [] }];
+  if (Array.isArray(models)) return [{ models: linkModels(models), extensions: [], sequences: [], derived: [], governedRoles: [], functions: [] }];
   throw new Error(`${modelsPath} must export a \`modules\` (or \`models\`) array.`);
 }
 

package/src/cli/commands/db.ts

@@ -1344,7 +1344,7 @@ export async function dbProvisionCommand(flags: Record<string, string>): Promise
 }
 
 /** Render a doctor report — identical output whichever venue produced it. */
-function printDoctorReport(report: any): void {
+export function printDoctorReport(report: any): void {
   console.log('');
   info(`api connection (DATABASE_URL):    role=${report.api.role}  superuser=${report.api.isSuperuser}  bypassrls=${report.api.bypassRls}`);
   info(`operator connection (ADMIN_URL):  role=${report.admin.role}  superuser=${report.admin.isSuperuser}  bypassrls=${report.admin.bypassRls}`);
@@ -1361,7 +1361,8 @@ function printDoctorReport(report: any): void {
   if (report.ok) {
     success('db:doctor — database is least-privilege and RLS-subject');
   } else {
-    fail('db:doctor — the api connection is NOT correctly least-privilege (see failures above)');
+    const failed = (report.checks as Array<{ status: string }>).filter((c) => c.status === 'fail').length;
+    fail(`db:doctor — ${failed} check(s) failed (see failures above)`);
     // Probing a local database as your own superuser makes these fail by construction:
     // the checks describe the credential the API serves with, not the schema. A developer
     // reading them as defects in their app will go hunting for a bug that is not there.

package/src/cli/commands/runbook.ts

@@ -13,6 +13,7 @@
  *   everystack runbook --force    overwrite hand-edits inside generated blocks
  */
 
+import { linkModels, ModelLinkError } from '@everystack/model';
 import fs from 'node:fs';
 import path from 'node:path';
 import { pathToFileURL } from 'node:url';
@@ -46,8 +47,10 @@ async function loadDataModel(root: string, models: { dir: string | null; hasInde
     if (!Array.isArray(models)) {
       return { dataModel: null, warning: `models/index.ts exports no \`models\` array — data-model chapter skipped.` };
     }
+    linkModels(models);
     return { dataModel: renderDataModelSection(models), warning: null };
   } catch (err: any) {
+    if (err instanceof ModelLinkError) throw err;
     return {
       dataModel: null,
       warning: `Could not load models from ${barrel} (${err.message}) — data-model chapter skipped.`,

package/src/cli/connection-fence.ts

@@ -0,0 +1,351 @@
+/**
+ * The connection fence: a single-connection postgres.js client that fails CLOSED when its
+ * backend dies.
+ *
+ * WHY. A mutation holds a session advisory lock (the mutation lease) and often an open
+ * transaction on one backend. When that backend dies (failover, `pg_terminate_backend`, a
+ * network drop), the lock and the transaction die with it. Two driver behaviours then turn a
+ * clean failure into something worse (postgres.js 3.4.9, the pinned and latest version):
+ *
+ * 1. Inside `sql.begin` or `sql.reserve`, the next statement is handed to the CLOSED connection.
+ *    The driver schedules a socket write on a null socket, which throws `Cannot read properties
+ *    of null (reading 'write')` outside any promise (an uncaught exception), and the statement
+ *    never settles. `sql.begin` does this even when our code issues nothing more: after the
+ *    body throws it sends its own ROLLBACK down the same path. So no check inside a
+ *    `sql.begin` body can prevent it; this fence never uses `sql.begin` or `sql.reserve`.
+ * 2. At the pool level, the next statement silently RECONNECTS onto a new backend. That
+ *    backend holds no lease and is not in the transaction, so the rest of a "transaction"
+ *    would autocommit, unleased.
+ *
+ * HOW. Every statement goes through the pool (`sql.unsafe`) on a client created with
+ * `max: 1`, so there is one connection and the pool path never writes to a closed socket.
+ * The driver's public `onclose` option trips the fence the moment the connection closes; from
+ * then on nothing is handed to the driver again, so the reconnect never happens. Statements
+ * are dispatched one at a time, and each waiting caller is also released by the loss itself,
+ * so a statement the driver never settles still ends in a named `ConnectionLostError`.
+ * Transactions are plain `BEGIN` / `COMMIT` / `ROLLBACK` statements through the same fence.
+ *
+ * Three more rules close what `onclose` alone leaves open:
+ * - A rejection that is not a PostgreSQL statement error (no `severity`: ECONNRESET, EPIPE, the
+ *   driver's CONNECTION_* errors) or is a FATAL / PANIC trips the fence at once. On a TCP reset
+ *   the driver can reject the statement before its onclose runs, and nothing may be sent in
+ *   between. A failure of the connect phase (wrong database, role or password, refused port)
+ *   trips it too, but surfaces the driver's own error: nothing ran. Every other loss message
+ *   ends with the driver error that revealed it, `(cause: CODE: message)`.
+ * - What the loss says depends on WHEN it happened. Between statements: nothing was in flight,
+ *   an open transaction rolled back. With a statement in flight: it MAY HAVE COMMITTED (an
+ *   autocommit write, a hand-issued COMMIT), so `outcomeUnknown` is true. Inside `transaction()`
+ *   before its COMMIT was sent: rolled back. Neither message claims the lease is already free:
+ *   the server frees it when it notices the connection is gone.
+ * - A COMMIT that PostgreSQL answers with the ROLLBACK command tag (the transaction had already
+ *   failed) is a `TransactionRolledBackError`, never a success.
+ *
+ * THE WINDOW, stated. The fence is checked, then the driver dispatches the statement one
+ * microtask later (postgres.js runs a query's handler after `await 1`). The driver trips the
+ * fence synchronously inside its own `closed()`, the same function that nulls the socket, and
+ * `closed()` runs from a socket 'close' event, which cannot run inside a microtask gap. So a
+ * statement that passes the check reaches a connection that is still open; if it dies an
+ * instant later, `closed()` cancels the pending write and rejects the statement. Only our own
+ * synchronous code could close the socket inside the gap, and nothing here does.
+ */
+
+import type { QueryRunner } from './authz-contract';
+import { borrowedSessionRunner, type SessionOptions, type SessionRunner } from './session';
+
+/** The backend behind a fenced connection is gone; nothing more ran on it. */
+export class ConnectionLostError extends Error {
+  readonly code = 'CONNECTION_LOST';
+  /**
+   * True when a statement was IN FLIGHT as the connection died and it could have committed on
+   * its own: an autocommit write, or a COMMIT. The server may or may not have applied it. False
+   * when nothing was in flight, or when the lost statement sat inside a transaction whose
+   * COMMIT was never sent: that transaction rolled back with the backend.
+   */
+  readonly outcomeUnknown: boolean;
+  constructor(message: string, options: { outcomeUnknown?: boolean; cause?: unknown } = {}) {
+    // The CLI prints the message, never `.cause`: the driver's own words ride in the text.
+    const driverError = rootCause(options.cause);
+    super(driverError === undefined ? message : `${message} (cause: ${describeCause(driverError)})`);
+    this.name = 'ConnectionLostError';
+    this.outcomeUnknown = options.outcomeUnknown === true;
+    if (options.cause !== undefined) (this as { cause?: unknown }).cause = options.cause;
+  }
+}
+
+/** Follow `.cause` through wrapping ConnectionLostErrors to the driver's own error, if any. */
+function rootCause(cause: unknown): unknown {
+  let at = cause;
+  while (at instanceof ConnectionLostError) at = (at as { cause?: unknown }).cause;
+  return at;
+}
+
+/** `CODE: message` for a driver error, or its message alone when it has no code. */
+function describeCause(err: unknown): string {
+  if (!err || typeof err !== 'object') return String(err);
+  const { code, message } = err as { code?: unknown; message?: unknown };
+  const text = typeof message === 'string' ? message : String(err);
+  return typeof code === 'string' && code.length > 0 ? `${code}: ${text}` : text;
+}
+
+/**
+ * PostgreSQL answered COMMIT with ROLLBACK: the transaction had already failed (a statement
+ * error the body caught and carried on past), so nothing in it committed. PostgreSQL reports
+ * this with a command tag, not an error; without this check it reads as success.
+ */
+export class TransactionRolledBackError extends Error {
+  constructor(query: string) {
+    super(
+      `PostgreSQL rolled back the transaction instead of committing it (${query.trim().split(/\s+/)[0]} ` +
+      'returned ROLLBACK): an earlier statement in it failed, so NOTHING in the transaction committed.',
+    );
+    this.name = 'TransactionRolledBackError';
+  }
+}
+
+const LOST = 'database connection lost (the backend was terminated or the network dropped)';
+const LEASE_NOTE =
+  'The server releases the mutation lease when it notices the connection is gone; if a re-run ' +
+  'reports the lease is still held, that session is still alive: wait for it or terminate it.';
+
+/** Nothing was in flight: the statement was refused before it reached the driver. */
+const LOST_BETWEEN_MESSAGE =
+  `${LOST} between statements. Nothing more ran on it: any transaction still open rolled back ` +
+  `with the backend; statements that finished before the loss stand. ${LEASE_NOTE} Re-run the command.`;
+
+/** A statement was in flight and could have committed on its own. */
+const LOST_IN_FLIGHT_MESSAGE =
+  `${LOST} while a statement was in flight. That statement MAY HAVE COMMITTED: read the target ` +
+  `before re-running. Nothing more ran on the connection. ${LEASE_NOTE}`;
+
+/** Inside transaction(), before its COMMIT was sent: nothing in it can have committed. */
+const LOST_IN_TRANSACTION_MESSAGE =
+  `${LOST} inside a transaction, before its COMMIT was sent: the transaction rolled back with ` +
+  `the backend. ${LEASE_NOTE} Re-run the command.`;
+
+/** transaction()'s own COMMIT was in flight. */
+const LOST_AT_COMMIT_MESSAGE =
+  `${LOST} while COMMIT was in flight: the transaction MAY HAVE COMMITTED or may not. Read the ` +
+  `target before re-running. ${LEASE_NOTE}`;
+
+/**
+ * Does this rejection mean the CONNECTION is gone, not just the statement? A PostgreSQL error
+ * carries the server's `severity`; anything without one (ECONNRESET, EPIPE, ETIMEDOUT, the
+ * driver's CONNECTION_CLOSED / CONNECTION_DESTROYED / CONNECTION_ENDED / CONNECT_TIMEOUT) is the
+ * transport. A FATAL or PANIC is the server ending the session. A false positive fails closed,
+ * which is acceptable.
+ */
+function isConnectionFailure(err: unknown): boolean {
+  if (!err || typeof err !== 'object') return true;
+  const { severity } = err as { severity?: unknown };
+  if (typeof severity !== 'string') return true;
+  return severity === 'FATAL' || severity === 'PANIC';
+}
+
+/**
+ * Failures of the CONNECT phase: the connection never opened, so no statement can have run.
+ * The server FATALs a login can end in (no such database 3D000, no such role 28000, bad
+ * password 28P01, too many connections 53300, starting up 57P03), and the transport errors of
+ * reaching it at all. These still trip the fence, but the caller gets the driver's ORIGINAL
+ * error: "may have committed" would be false, and the server's text names what is wrong.
+ */
+const CONNECT_PHASE_CODES = new Set([
+  '3D000', '28000', '28P01', '53300', '57P03',
+  'ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'CONNECT_TIMEOUT',
+]);
+
+function isConnectPhaseFailure(err: unknown): boolean {
+  const code = (err as { code?: unknown } | null)?.code;
+  return typeof code === 'string' && CONNECT_PHASE_CODES.has(code);
+}
+
+/** COMMIT or its synonym END: the statements PostgreSQL may answer with a ROLLBACK tag. */
+const COMMIT_RE = /^\s*(COMMIT|END)\b/i;
+
+/** The loss signal, tripped by the driver's `onclose`. */
+export interface ConnectionFence {
+  /** The loss, once it happened; null while the connection is alive. */
+  readonly lost: ConnectionLostError | null;
+  /**
+   * Trip the fence. Idempotent; a no-op once the owner has ended the connection. `cause` is
+   * the driver error that revealed the loss, when one did; later refusals name it.
+   */
+  lose(cause?: unknown): void;
+  /**
+   * The value for postgres.js's `onclose` option. Chains `next` (a caller's own onclose), so
+   * installing the fence never silences another listener.
+   */
+  onclose(next?: (connectionId: number) => void): (connectionId: number) => void;
+}
+
+interface FenceInternals extends ConnectionFence {
+  ended: boolean;
+  /** Callers waiting on an in-flight statement; the loss releases each. */
+  listeners: Set<(cause?: unknown) => void>;
+}
+
+export function createConnectionFence(): ConnectionFence {
+  const fence: FenceInternals = {
+    lost: null,
+    ended: false,
+    listeners: new Set(),
+    lose(cause?: unknown) {
+      if (fence.lost || fence.ended) return;
+      // What every LATER statement is refused with: nothing of it was in flight.
+      (fence as { lost: ConnectionLostError | null }).lost = new ConnectionLostError(LOST_BETWEEN_MESSAGE, { cause });
+      const waiting = [...fence.listeners];
+      fence.listeners.clear();
+      for (const release of waiting) release();
+    },
+    onclose(next) {
+      return (connectionId: number) => {
+        fence.lose();
+        next?.(connectionId);
+      };
+    },
+  };
+  return fence;
+}
+
+/** The slice of a postgres.js client the fence drives. */
+export interface FenceableClient {
+  unsafe: (query: string) => PromiseLike<unknown>;
+  end: (options?: { timeout?: number }) => Promise<void>;
+}
+
+export interface FencedConnection {
+  /** One statement (autocommit unless the caller opened a transaction with `BEGIN`). */
+  runner: QueryRunner;
+  /** N statements in ONE transaction on this connection — see `SessionRunner`. */
+  session: SessionRunner;
+  /** `BEGIN`, the body, `COMMIT` — or `ROLLBACK` on a body error while the connection lives. */
+  transaction: <T>(fn: (run: QueryRunner) => Promise<T>) => Promise<T>;
+  /** Close the client. Statements after this are refused. */
+  end: () => Promise<void>;
+}
+
+const ISOLATION_SQL: Record<string, string> = {
+  'read uncommitted': 'READ UNCOMMITTED',
+  'read committed': 'READ COMMITTED',
+  'repeatable read': 'REPEATABLE READ',
+  serializable: 'SERIALIZABLE',
+};
+
+/** `SET TRANSACTION …` for the session options, or null when there is nothing to set. */
+function setTransactionSql(opts: SessionOptions | undefined): string | null {
+  const modes: string[] = [];
+  if (opts?.isolation !== undefined) {
+    const level = ISOLATION_SQL[String(opts.isolation).toLowerCase().trim()];
+    if (!level) {
+      throw new Error(`session isolation ${JSON.stringify(opts.isolation)} is not a PostgreSQL isolation level`);
+    }
+    modes.push(`ISOLATION LEVEL ${level}`);
+  }
+  if (opts?.readOnly) modes.push('READ ONLY');
+  return modes.length ? `SET TRANSACTION ${modes.join(', ')}` : null;
+}
+
+/**
+ * Wrap a single-connection postgres.js client (`max: 1`, created with `onclose:
+ * fence.onclose(...)`) so it fails closed on connection loss. See the file comment.
+ */
+export function fencedConnection(sql: FenceableClient, fenceIn: ConnectionFence): FencedConnection {
+  const fence = fenceIn as FenceInternals;
+  let tail: Promise<unknown> = Promise.resolve();
+
+  /** Check the fence, then hand the statement to the driver; the loss also releases the caller. */
+  const dispatch = (query: string): Promise<any[]> => {
+    if (fence.ended) return Promise.reject(new Error('this database connection has already ended'));
+    if (fence.lost) return Promise.reject(fence.lost);
+    return new Promise<any[]>((resolve, reject) => {
+      // The statement was handed to the driver, so the server may have run it: a loss from here
+      // on is IN FLIGHT, and its outcome is unknown.
+      const inFlightLoss = (cause?: unknown): ConnectionLostError =>
+        new ConnectionLostError(LOST_IN_FLIGHT_MESSAGE, { outcomeUnknown: true, cause });
+      const release = (cause?: unknown): void => reject(inFlightLoss(cause));
+      fence.listeners.add(release);
+      Promise.resolve(sql.unsafe(query)).then(
+        (rows) => {
+          fence.listeners.delete(release);
+          // Read the command tag before Array.from copies the rows and drops it.
+          const command = (rows as { command?: unknown } | null)?.command;
+          if (command === 'ROLLBACK' && COMMIT_RE.test(query)) {
+            reject(new TransactionRolledBackError(query));
+            return;
+          }
+          resolve(Array.from(rows as Iterable<any>));
+        },
+        (err) => {
+          fence.listeners.delete(release);
+          if (fence.lost || isConnectionFailure(err)) {
+            // Trip the fence NOW, before anything else can be sent. On a TCP reset Node emits
+            // 'error' before 'close', so the driver can reject this statement before its onclose
+            // runs; in that interval a catch-block ROLLBACK, a failure bookkeeping row or the
+            // lease release would otherwise reach a driver whose pool reconnects onto a NEW backend.
+            fence.lose(err);
+            // A connection that never opened ran nothing: keep the driver's own error.
+            reject(isConnectPhaseFailure(err) ? err : inFlightLoss(err));
+            return;
+          }
+          reject(err);
+        },
+      );
+    });
+  };
+
+  // One statement at a time: the next is not even checked until the previous settled, so two
+  // callers can never have a statement queued inside the driver when the connection dies.
+  const runner: QueryRunner = (query: string) => {
+    const turn = tail.then(() => dispatch(query), () => dispatch(query));
+    tail = turn.catch(() => undefined);
+    return turn;
+  };
+
+  const transaction = async <T>(fn: (run: QueryRunner) => Promise<T>): Promise<T> => {
+    await runner('BEGIN');
+    let result: T;
+    try {
+      result = await fn(runner);
+    } catch (err) {
+      // After a loss nothing is sent: the transaction already rolled back with the backend.
+      if (!fence.lost) await runner('ROLLBACK').catch(() => undefined);
+      // A body statement lost in flight cannot have committed: its COMMIT was never sent.
+      if (err instanceof ConnectionLostError) {
+        throw new ConnectionLostError(LOST_IN_TRANSACTION_MESSAGE, { cause: err });
+      }
+      throw err;
+    }
+    if (fence.lost) throw new ConnectionLostError(LOST_IN_TRANSACTION_MESSAGE, { cause: fence.lost });
+    try {
+      await runner('COMMIT');
+    } catch (err) {
+      if (err instanceof ConnectionLostError) {
+        throw err.outcomeUnknown
+          ? new ConnectionLostError(LOST_AT_COMMIT_MESSAGE, { outcomeUnknown: true, cause: err })
+          : new ConnectionLostError(LOST_IN_TRANSACTION_MESSAGE, { cause: err });
+      }
+      throw err;
+    }
+    return result;
+  };
+
+  const session: SessionRunner = async (statements, opts) => {
+    const setTransaction = setTransactionSql(opts);
+    return transaction(async (run) => {
+      if (setTransaction) await run(setTransaction);
+      // The borrowed runner pins search_path inside a savepoint and takes each allowFailure
+      // statement in its own savepoint: hand-issued, which is sound here because no driver
+      // transaction scope is keeping its own error bookkeeping.
+      return borrowedSessionRunner(run)(
+        statements,
+        opts?.searchPath !== undefined ? { searchPath: opts.searchPath } : undefined,
+      );
+    });
+  };
+
+  const end = async (): Promise<void> => {
+    fence.ended = true;
+    await sql.end({ timeout: 5 });
+  };
+
+  return { runner, session, transaction, end };
+}

package/src/cli/db-source.ts

@@ -26,6 +26,7 @@
 
 import type { QueryRunner } from './authz-contract';
 import { buildSearchPathSql, type SessionRunner, type SessionResult } from './session';
+import { createConnectionFence, fencedConnection } from './connection-fence';
 
 export type DbSource =
   | { kind: 'url'; url: string; from: 'flag' | 'env' | 'admin-env' | 'operator' }
@@ -152,13 +153,16 @@ export async function createUrlRunner(
   // minutes, resolving the in-flight query and THEN killing the session. Under reconcile's
   // BEGIN-across-calls transaction that is a silent session swap mid-batch (the reconcile
   // apply path also pins the backend pid and refuses bookkeeping if it moved).
-  const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => {}, ...sslDefaults(url) });
-  return {
-    runner: async (query: string) => Array.from(await sql.unsafe(query)),
-    session: sessionRunnerOver(sql),
-    transaction: async (fn) => sql.begin((tx: any) => fn(async (query: string) => Array.from(await tx.unsafe(query)))),
-    end: () => sql.end({ timeout: 5 }),
-  };
+  //
+  // Fenced (connection-fence.ts): every direct-lane mutation lease is taken on this connection,
+  // so a dead backend must fail CLOSED — a named ConnectionLostError, never a driver reconnect
+  // onto a new, unleased backend and never the driver's uncaught write on a closed socket.
+  // That is also why session and transaction are plain statements here, not `sql.begin`.
+  const fence = createConnectionFence();
+  const sql = postgres(url, {
+    max: 1, max_lifetime: null, onnotice: () => {}, ...sslDefaults(url), onclose: fence.onclose(),
+  });
+  return fencedConnection(sql, fence);
 }
 
 export interface UrlPipelineRunner {

package/src/cli/declared-derived.ts

@@ -12,7 +12,7 @@ import path from 'node:path';
 import fs from 'node:fs/promises';
 import { pathToFileURL } from 'node:url';
 import type { Module, ModelDescriptor, SequenceDescriptor, DerivedDescriptor } from '@everystack/model';
-import { moduleGovernedRoles } from '@everystack/model';
+import { linkModels, ModelLinkError, moduleGovernedRoles } from '@everystack/model';
 import { compileDerived } from './derived-compile';
 import { compileTableRenames, compileTableMoves } from './schema-compile';
 import type { SourceObject } from './derived-source';
@@ -69,19 +69,23 @@ export async function loadModulesFrom(modelsPath: string): Promise<Module[]> {
     try {
       mod = await import(pathToFileURL(abs).href);
     } catch (err: any) {
+      if (err instanceof ModelLinkError) throw err;
       throw new Error(`Could not load modules from ${modelsPath}: ${err.message}`);
     }
     if (Array.isArray(mod.modules)) {
       assertNoHoles(modelsPath, 'modules', mod.modules);
+      linkModels(mod.modules.flatMap((m: Module) => m.models));
       return mod.modules;
     }
     const models = mod.models ?? mod.default;
     if (Array.isArray(models)) {
       assertNoHoles(modelsPath, 'models', models);
+      linkModels(models);
       return [{ models, extensions: [], sequences: [], derived: [], governedRoles: [], functions: [] }];
     }
     throw new Error(`${modelsPath} must export a \`modules\` (or \`models\`) array.`);
   } catch (err) {
+    if (err instanceof ModelLinkError) throw err;
     throw asModelComposeError(modelsPath, err);
   }
 }
@@ -191,6 +195,7 @@ export function composeDeclaredDerived(modules: Module[], modelsPath: string): D
       extensions: [...new Set(modules.flatMap((m) => m.extensions ?? []))].sort(),
     };
   } catch (err) {
+    if (err instanceof ModelLinkError) throw err;
     throw asModelComposeError(modelsPath, err);
   }
 }

package/src/cli/derived-apply.ts

@@ -249,9 +249,26 @@ export function renderSetSearchPath(schemas: string[], local: boolean): string |
  */
 export function renderEnsureObjectSchemas(objects: SourceObject[]): string[] {
   const statements: string[] = [];
+  const usage = declaredObjectSchemaUsage(objects);
+  for (const [schema, roles] of Object.entries(usage)) {
+    statements.push(`CREATE SCHEMA IF NOT EXISTS ${quoteIdent(schema)}`);
+    if (roles.length > 0) {
+      statements.push(`GRANT USAGE ON SCHEMA ${quoteIdent(schema)} TO ${roles.join(', ')}`);
+    }
+  }
+  return statements;
+}
+
+/**
+ * The schema USAGE the declared objects imply, as data: every non-public schema an object lives
+ * in (sorted) → the sorted roles its objects grant anything to. The rule renderEnsureObjectSchemas
+ * renders into SQL (see A13 there for why PUBLIC is never included); exposed so a recorder — the
+ * backup's authz manifest — reads the SAME answer instead of re-deriving it.
+ */
+export function declaredObjectSchemaUsage(objects: SourceObject[]): Record<string, string[]> {
+  const usage: Record<string, string[]> = {};
   const schemas = [...new Set(objects.map((o) => o.schema))].filter((s) => s !== 'public').sort();
   for (const schema of schemas) {
-    statements.push(`CREATE SCHEMA IF NOT EXISTS ${quoteIdent(schema)}`);
     const roles = new Set<string>();
     for (const o of objects) {
       if (o.schema !== schema) continue;
@@ -262,11 +279,9 @@ export function renderEnsureObjectSchemas(objects: SourceObject[]): string[] {
         if (privileges.length > 0 && role.toUpperCase() !== 'PUBLIC') roles.add(role);
       }
     }
-    if (roles.size > 0) {
-      statements.push(`GRANT USAGE ON SCHEMA ${quoteIdent(schema)} TO ${[...roles].sort().join(', ')}`);
-    }
+    usage[schema] = [...roles].sort();
   }
-  return statements;
+  return usage;
 }
 
 // ---------------------------------------------------------------------------

package/src/cli/index.ts

@@ -317,7 +317,7 @@ Usage:
   everystack db:provision      --rotate --stage <name> [--slot api|ops|both] [--grace 60]  Rotate credentials live (A/B role swap, no deploy, no downtime)
   everystack db:snapshot       [--stage <name>] [--instance <id>]  Take a physical RDS snapshot (instant DR point; RDS only — use db:backup for portable logical backups)
   everystack db:snapshots      [--stage <name>] [--instance <id>]  List manual RDS snapshots for the instance
-  everystack db:backup         [--stage <name>]  Logical pg_dump of the DB → private S3 backups bucket (prints the backup id). Add --database-url <local> --stage <target> to dump a LOCAL database and upload it into the target stage's bucket as a restorable backup — the credential-free break-glass local→stage full replace (then: db:restore --from <id> --stage <target>)
+  everystack db:backup         [--stage <name>]  Logical pg_dump of the DB → private S3 backups bucket (prints the backup id). Add --database-url <local> --stage <target> to dump a LOCAL database and upload it into the target stage's bucket. Such a backup records no origin commit, so db:restore refuses it: restoring a local backup into a stage is closed until local backups can record a verified origin
   everystack db:backups        [--stage <name>]  List logical backups (id, size, created)
   everystack db:restore        --from <id> [--stage <name>] --confirm  Restore a backup INTO the stage's DB (DESTRUCTIVE)
   everystack db:backup:download <id> [--stage <name>]  Presigned URL to download a backup's dump (valid 1h)