@everystack/cli 0.4.34 → 0.4.36

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.34",
+  "version": "0.4.36",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/commands/db-backup.ts

@@ -25,7 +25,7 @@ import {
   parseBackupRef, keyForId, crossStageGuard, restoreTargetGuard,
   backupKey, backupId, metaKey, utcStamp,
 } from '../backup.js';
-import { pgEnvFromUrl } from './db.js';
+import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { pollTaskUntilStopped } from '../task-poll.js';
 import { step, success, fail, info, warn } from '../output.js';
 
@@ -57,7 +57,9 @@ export function resolveBackupVenue(flags: Record<string, string>): BackupVenue {
  * restoring it would rewind that history AND makes pg_restore --clean fail on DROP SCHEMA everystack).
  */
 export function pgDumpFullArgs(): string[] {
-  return ['-Fc', '--no-owner', '--no-privileges', '--no-comments', '--exclude-schema=everystack'];
+  // -d carries keepalives only; the credential still rides the PG* env (see db.ts). A full dump of
+  // a large database goes quiet for long stretches, which is exactly when an unkept connection dies.
+  return ['-d', pgKeepaliveConninfo(), '-Fc', '--no-owner', '--no-privileges', '--no-comments', '--exclude-schema=everystack'];
 }
 
 /** The .meta.json sidecar — byte-shape identical to the server's runBackup meta (backup-run.ts:156),

package/src/cli/commands/db-export.ts

@@ -26,7 +26,7 @@ import { fingerprintModels } from '../schema-fingerprint.js';
 import { resolveModelsPath } from '../models-path.js';
 import { loadModels } from './db-generate.js';
 import { loadDeclaredDerived } from '../declared-derived.js';
-import { pgEnvFromUrl } from './db.js';
+import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { utcStamp } from '../backup.js';
 import { pollTaskUntilStopped } from '../task-poll.js';
 import { step, success, fail, info } from '../output.js';
@@ -99,7 +99,8 @@ export function localArtifactMeta(opts: {
 
 /** pg_dump args for the local venue — same dump shape as the ops venue, written straight to a file. */
 export function pgDumpLocalArgs(schema: string, dumpPath: string): string[] {
-  return ['-Fc', '--no-owner', '--no-privileges', '--no-comments', `--schema=${schema}`, '-f', dumpPath];
+  // -d carries keepalives only; the credential still rides the PG* env (see db.ts).
+  return ['-d', pgKeepaliveConninfo(), '-Fc', '--no-owner', '--no-privileges', '--no-comments', `--schema=${schema}`, '-f', dumpPath];
 }
 
 /**

package/src/cli/commands/db-swap.ts

@@ -26,15 +26,21 @@ import { fingerprintModels } from '../schema-fingerprint.js';
 import { resolveModelsPath } from '../models-path.js';
 import { loadModels } from './db-generate.js';
 import { loadDeclaredDerived } from '../declared-derived.js';
+import type { SourceObject } from '../derived-source.js';
+import { pairedDerivedSchemas, renderPairedDerivedBuild, renderSwapSchemaUsage, swapSchemaRoles, expectedIncomingObjects, renderPairedProvenance } from '../swap-pair.js';
+import { introspectDerived } from '../derived-introspect.js';
 import { createUrlRunner } from '../db-source.js';
+import type { QueryRunner } from '../authz-contract.js';
 import { executeSwap, type SwapVerdict } from '../swap-execute.js';
+import { startHeartbeat, humanElapsed } from '../swap-heartbeat.js';
+import { formatBytes } from '../bundle-weight.js';
 import { rewriteStatementLine, opensCopyData, closesCopyData } from '../schema-rewrite.js';
 import { resolveOperatorUrlViaStage } from '../direct-venue.js';
 import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
 import { resolveConfig, opsFunction } from '../config.js';
 import { invokeAction, presignGet } from '../aws.js';
 import { keyForArtifactId, metaKey } from '../backup.js';
-import { pgEnvFromUrl } from './db.js';
+import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { step, success, fail, warn, info } from '../output.js';
 
 /** A COPY-aware line transform that rewrites the schema token on statement lines only. */
@@ -70,37 +76,254 @@ function schemaRewriteStream(from: string, to: string): Transform {
 }
 
 /**
- * Restore a `-Fc` artifact into `<schema>_incoming`: pg_restore -f - (archive → SQL) → the schema
- * rewrite → psql. The archive names `<schema>`; the rewrite lands it as `<incoming>`, COPY-data-safe.
+ * Consecutive polls reporting NO server-side backend before the watchdog kills psql. Three at the
+ * 10s cadence is 30s of agreement, which is well past any single-poll blip and far short of the
+ * eight minutes the unguarded version cost.
  */
-async function restoreIntoIncoming(url: string, artifactPath: string, schema: string, incoming: string): Promise<void> {
-  // Connect via PG* env, not `-d <url>`. libpq VALIDATES URI query params against its keyword
-  // list and REJECTS non-keywords like `search_path` ("invalid URI query parameter") — and the
-  // operator URL db:operator-url mints bakes search_path in (fine for postgres.js, fatal for a
-  // libpq client). pgEnvFromUrl extracts only libpq keywords (dropping search_path, routing
-  // sslmode→PGSSLMODE) and puts the password in PGPASSWORD, off the process argv. pg_restore -f -
-  // just reads the archive file to SQL on stdout (no connection).
-  const restore = spawn('pg_restore', ['-f', '-', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
-  const psql = spawn('psql', ['-v', 'ON_ERROR_STOP=1'], {
-    stdio: ['pipe', 'ignore', 'pipe'],
-    env: { ...process.env, ...pgEnvFromUrl(url) },
-  });
-  let rErr = '', pErr = '';
-  restore.stderr.on('data', (d) => { rErr += d.toString(); });
-  psql.stderr.on('data', (d) => { pErr += d.toString(); });
-  const psqlExit = new Promise<void>((res, rej) => {
-    psql.on('error', rej);
-    psql.on('close', (c) => c === 0 ? res() : rej(new Error(`psql exited ${c}: ${pErr.trim()}`)));
-  });
-  const restoreExit = new Promise<void>((res, rej) => {
-    restore.on('error', rej);
-    restore.on('close', (c) => c === 0 ? res() : rej(new Error(`pg_restore exited ${c}: ${rErr.trim()}`)));
+const DEAD_BACKEND_POLLS = 3;
+
+/**
+ * TOC entry types that a DERIVED object can appear as. Order matters: the longer types must be
+ * tested first, or `MATERIALIZED VIEW DATA` parses as `MATERIALIZED VIEW` with a mangled schema.
+ */
+const DERIVED_TOC_TYPES = [
+  'MATERIALIZED VIEW DATA',
+  'MATERIALIZED VIEW',
+  'PROCEDURE',
+  'FUNCTION',
+  'TRIGGER',
+  'VIEW',
+] as const;
+
+/**
+ * Strip the DECLARED DERIVED objects out of a pg_restore TOC listing.
+ *
+ * An artifact should carry BASE STATE — tables, data, indexes, constraints, sequences. It should
+ * not carry the derived layer, because db:reconcile owns that and rebuilds it from the descriptors.
+ * `pg_dump --schema=<s>` cannot make that distinction: it dumps everything in the schema, derived
+ * objects included.
+ *
+ * Shipping them is not merely redundant, it DEADLOCKS the swap. A derived object inside the base
+ * schema may reference the derived schema built on top of it (`stats.draft_value` returns
+ * `SETOF stats_view.draft_value_row`). The restore then cannot run until the derived layer exists,
+ * while the derived layer cannot be built until the new base tables land. Neither can go first.
+ * Measured on a real artifact: the only entries referencing the derived schema were the 16
+ * reconcile-managed functions; every other entry was state.
+ *
+ * Filtering by TOC entry rather than by SQL text is deliberate: `pg_restore -L` is the supported
+ * mechanism for restoring a subset, and the alternative is pattern-matching a six-million-line SQL
+ * file. Entries are COMMENTED OUT rather than deleted so the listing stays diffable.
+ */
+export function filterDerivedFromToc(
+  toc: string,
+  declaredIdentities: Iterable<string>,
+): { listing: string; skipped: string[] } {
+  const declared = new Set(declaredIdentities);
+  const skipped: string[] = [];
+  const listing = toc.split('\n').map((line) => {
+    // `<dumpId>; <catalogOid> <oid> <TYPE> <schema> <name(args)> <owner>`
+    const m = /^(\d+;\s+\d+\s+\d+)\s+(.+)$/.exec(line);
+    if (!m) return line; // header/comment/blank — pass through untouched
+    const rest = m[2];
+    const type = DERIVED_TOC_TYPES.find((t) => rest.startsWith(`${t} `));
+    if (!type) return line; // not a derived-capable entry (TABLE, INDEX, POLICY, ...)
+    const after = rest.slice(type.length + 1);
+    const parts = after.split(/\s+/);
+    if (parts.length < 2) return line;
+    const schema = parts[0];
+    // The owner is the last token; everything between it and the schema is the name (+args).
+    const nameWithArgs = parts.slice(1, -1).join(' ');
+    const name = nameWithArgs.replace(/\(.*$/, ''); // drop the argument list
+    if (!declared.has(`${schema}.${name}`)) return line;
+    skipped.push(`${schema}.${name} (${type.toLowerCase()})`);
+    return `;${line}`;
+  }).join('\n');
+  return { listing, skipped };
+}
+
+/** A pass-through that counts the bytes crossing it — Phase A's progress signal. */
+function countingTap(onBytes: (total: number) => void): Transform {
+  let total = 0;
+  return new Transform({
+    transform(chunk, _enc, cb) {
+      total += chunk.length;
+      onBytes(total);
+      cb(null, chunk);
+    },
   });
-  await Promise.all([
-    pipeline(restore.stdout!, schemaRewriteStream(schema, incoming), psql.stdin!),
-    restoreExit,
-    psqlExit,
-  ]);
+}
+
+/** The observation sinks the restore reports through. Injected so the phases stay testable. */
+interface RestoreIO {
+  /** The operator connection — IDLE for the whole restore, so the Phase B heartbeat reuses it. */
+  runner: QueryRunner;
+  log: (msg: string) => void;
+  warn: (msg: string) => void;
+  /**
+   * `schema.name` of every DECLARED derived object. These are stripped from the restore: the
+   * artifact carries base state, db:reconcile owns the derived layer. See filterDerivedFromToc.
+   */
+  declaredIdentities: string[];
+}
+
+/**
+ * Restore a `-Fc` artifact into `<schema>_incoming`, in TWO phases split by a local temp file:
+ *
+ *   Phase A (local, no network): pg_restore -f - (archive → SQL) → the COPY-safe schema rewrite →
+ *     a local `.sql` file. The archive names `<schema>`; the rewrite lands it as `<incoming>`.
+ *   Phase B (network): psql -f <file> streams that file to the target at psql's own pace.
+ *
+ * Why the temp file and not a live `pg_restore | rewrite | psql` pipe: decoupling the producer
+ * (local, fast) from the consumer removes all cross-process backpressure, and it makes Phase A
+ * measurable on its own. Costs one temp file (~the uncompressed dump), cleaned up in `finally`.
+ *
+ * A CORRECTION, because the original note here sent two debugging sessions down the wrong path.
+ * It claimed the three-way pipe DEADLOCKED at the DDL→data boundary and that the temp file was the
+ * cure. The temp file went in, and the restore STILL died at the same place. The pipe was never the
+ * root cause; the `write EPIPE` it produced was a downstream symptom.
+ *
+ * The real cause (proven 2026-07-27, see db.ts's keepalive note): the connection dies during the
+ * long quiet stretch of the index/constraint phase, the server terminates the backend, and the
+ * client — which never sees a FIN or RST — blocks forever on a socket its kernel still calls
+ * ESTABLISHED. The fix is TCP keepalives on psql's connection, plus the watchdog below, NOT the
+ * process topology. The temp file is kept because it is genuinely better instrumented, not because
+ * it fixes a deadlock.
+ *
+ * Both phases are INSTRUMENTED. This ran silent once — a multi-GB push with no output at all — and
+ * a run that died unattended was indistinguishable from one still working. Now: a byte counter on
+ * Phase A, a server-side heartbeat on Phase B (see swap-heartbeat.ts), and both child processes'
+ * stderr streamed as it arrives instead of being withheld until exit.
+ */
+async function restoreIntoIncoming(
+  url: string,
+  artifactPath: string,
+  schema: string,
+  incoming: string,
+  io: RestoreIO,
+): Promise<void> {
+  const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'everystack-restore-'));
+  const sqlPath = path.join(tmpDir, `${incoming}.sql`);
+  const t0 = Date.now();
+  try {
+    // Phase A0: strip the declared derived layer from the restore. The artifact is base STATE;
+    // db:reconcile owns derived. Landing them here is redundant at best and deadlocks the restore
+    // when a derived object in the base schema references the derived schema above it.
+    const tocPath = path.join(tmpDir, 'toc.list');
+    const toc = await new Promise<string>((res, rej) => {
+      const p = spawn('pg_restore', ['-l', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
+      let out = '', err = '';
+      p.stdout.on('data', (d) => { out += d.toString(); });
+      p.stderr.on('data', (d) => { err += d.toString(); });
+      p.on('error', rej);
+      p.on('close', (c) => c === 0 ? res(out) : rej(new Error(`pg_restore -l exited ${c}: ${err.trim()}`)));
+    });
+    const { listing, skipped } = filterDerivedFromToc(toc, io.declaredIdentities);
+    await fs.promises.writeFile(tocPath, listing, 'utf8');
+    io.log(
+      skipped.length
+        ? `restore: skipping ${skipped.length} declared derived object(s) — db:reconcile owns them (${skipped.slice(0, 6).join(', ')}${skipped.length > 6 ? `, +${skipped.length - 6} more` : ''}).`
+        : 'restore: the artifact carries no declared derived objects.',
+    );
+
+    // Phase A: pg_restore → rewrite → local file. Only local processes; nothing can stall here.
+    io.log(`restore phase A: pg_restore → schema rewrite (${schema} → ${incoming}) → ${sqlPath}`);
+    const restore = spawn('pg_restore', ['-L', tocPath, '-f', '-', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
+    let rErr = '';
+    restore.stderr.on('data', (d) => {
+      const s = d.toString();
+      rErr += s;
+      // Surface as it happens — a warning withheld until exit is a warning nobody can act on.
+      for (const line of s.split('\n').map((l: string) => l.trim()).filter(Boolean)) io.log(`pg_restore: ${line}`);
+    });
+    const restoreExit = new Promise<void>((res, rej) => {
+      restore.on('error', rej);
+      restore.on('close', (c) => c === 0 ? res() : rej(new Error(`pg_restore exited ${c}: ${rErr.trim()}`)));
+    });
+    let written = 0;
+    const phaseATimer = setInterval(() => {
+      io.log(`restore phase A: ${formatBytes(written)} of SQL written (t+${humanElapsed(Date.now() - t0)})...`);
+    }, 5_000);
+    (phaseATimer as any).unref?.();
+    try {
+      await Promise.all([
+        pipeline(
+          restore.stdout!,
+          countingTap((n) => { written = n; }),
+          schemaRewriteStream(schema, incoming),
+          fs.createWriteStream(sqlPath),
+        ),
+        restoreExit,
+      ]);
+    } finally {
+      clearInterval(phaseATimer);
+    }
+    const aMs = Date.now() - t0;
+    io.log(`restore phase A done: ${formatBytes(written)} of SQL in ${humanElapsed(aMs)}.`);
+
+    // Phase B: psql reads the local file and streams to the target. The credential rides PG* env,
+    // never argv (libpq also REJECTS non-keyword URI params like the `search_path` the operator URL
+    // bakes in — fine for postgres.js, fatal for a libpq URI). `-d` carries ONLY keepalives, which
+    // have no PG* env equivalent and are what keep this connection from dying in the index phase.
+    io.log(`restore phase B: psql streaming ${formatBytes(written)} to the target — heartbeat every 10s.`);
+    const psql = spawn('psql', ['-d', pgKeepaliveConninfo(), '-v', 'ON_ERROR_STOP=1', '-f', sqlPath], {
+      stdio: ['ignore', 'ignore', 'pipe'],
+      env: { ...process.env, ...pgEnvFromUrl(url) },
+    });
+    let pErr = '';
+    psql.stderr.on('data', (d) => {
+      const s = d.toString();
+      pErr += s;
+      for (const line of s.split('\n').map((l: string) => l.trim()).filter(Boolean)) io.warn(`psql: ${line}`);
+    });
+
+    // The watchdog. Keepalives should prevent the dead-backend hang, but if it happens anyway the
+    // heartbeat SEES it — pg_stat_activity, read over our own live connection, reports no psql
+    // backend while the psql process sits there forever. Detection without action is what cost an
+    // eight-minute wait: turn it into a kill and a named failure.
+    let deadBackendPolls = 0;
+    const stopHeartbeat = startHeartbeat(io.runner, {
+      incoming,
+      log: io.log,
+      warn: io.warn,
+      onSample: (sample) => {
+        if (sample.state === null) deadBackendPolls += 1;
+        else deadBackendPolls = 0;
+        if (deadBackendPolls === DEAD_BACKEND_POLLS && psql.exitCode === null) {
+          io.warn(
+            `restore: psql is still running but the server reports NO backend for it after ${deadBackendPolls} consecutive polls. `
+            + 'The connection died and psql will never notice (no FIN/RST reaches it). Killing it rather than hanging.',
+          );
+          psql.kill('SIGTERM');
+          // SIGTERM will not land if psql is parked in a blocking read on a dead socket.
+          const hardKill = setTimeout(() => { if (psql.exitCode === null) psql.kill('SIGKILL'); }, 5_000);
+          (hardKill as any).unref?.();
+        }
+      },
+    });
+    const bStart = Date.now();
+    try {
+      await new Promise<void>((res, rej) => {
+        psql.on('error', rej);
+        psql.on('close', (c) => {
+          if (c === 0) return res();
+          if (deadBackendPolls >= DEAD_BACKEND_POLLS) {
+            return rej(new Error(
+              'the restore connection died mid-load and psql hung on it (the server had no backend for it). '
+              + 'psql was killed by the watchdog; NOTHING was swapped and live is untouched. '
+              + 'This is the network path dropping a connection that goes quiet during the index/constraint phase — '
+              + 'keepalives are now set, so if you are seeing this the path is dropping the flow faster than a 30s probe interval.',
+            ));
+          }
+          rej(new Error(`psql exited ${c}: ${pErr.trim()}`));
+        });
+      });
+    } finally {
+      await stopHeartbeat();
+    }
+    io.log(`restore phase B done in ${humanElapsed(Date.now() - bStart)} (restore total ${humanElapsed(Date.now() - t0)}).`);
+  } finally {
+    await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
+  }
 }
 
 /**
@@ -193,17 +416,40 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
   if (!from) { fail('db:swap needs --from <artifact.dump | artifact-id> (the schema-scoped -Fc archive to land).'); process.exit(1); }
 
   // Resolve the venue.
-  //  - --database-url (or DATABASE_URL): a local/direct operator connection.
+  //  - --database-url: a local/direct operator connection. EXPLICIT FLAG ONLY.
   //  - --stage --direct: resolve the stage's OPERATOR connection from its ops Lambda and execute
   //    CLI-side with an unbounded clock (a multi-GB restore blows the 900s Lambda ceiling). The
   //    operator never holds a URL; the swap snapshots the stage via db:backup before it lands.
   //  - --stage alone: refuse, naming --direct — the ops-Lambda venue can't hold the restore clock.
-  let url = flags['database-url'] || process.env.DATABASE_URL;
+  //
+  // `process.env.DATABASE_URL` is NOT a venue here, and used to be.
+  //
+  // It was read first, and the stage branch was guarded by `if (!url && stage)` — so an exported
+  // DATABASE_URL SILENTLY OVERRODE `--stage`. An operator asking for dev got whatever the
+  // environment named, the stage's snapshot was skipped (the branch that sets snapshotViaStage
+  // never ran, hence the "direct v1" warning), and the swap reported success against a database
+  // nobody had asked for. A consumer hit exactly this: two runs differing only by an unrelated
+  // diagnostic flag went to different databases, because one shell had the variable exported and
+  // the other did not. Their dev derived layer was untouched because dev was never the target.
+  //
+  // On a DESTRUCTIVE verb an ambient variable must never choose the target, and db:export already
+  // states the rule: --database-url is explicit-flag-only, never the env — the venue must be
+  // deliberate. This is that rule, applied where it mattered most and was missing.
+  const urlFlag = flags['database-url'];
+  if (urlFlag && stage) {
+    fail(`db:swap got BOTH --database-url and --stage ${stage} — that is two different targets and the wrong one is destructive. Pass exactly one.`);
+    process.exit(1);
+  }
+  if (!urlFlag && !stage && process.env.DATABASE_URL) {
+    fail('db:swap will not take its target from the DATABASE_URL environment variable — a destructive swap must name its target explicitly. Pass --database-url <url> (local/direct) or --stage <name> --direct.');
+    process.exit(1);
+  }
+  let url = urlFlag;
   let snapshotViaStage = false;
   let region: string | undefined;
   let opsFn: string | undefined;
 
-  if (!url && stage) {
+  if (stage) {
     if (!direct) {
       fail('db:swap --stage needs --direct: a schema restore can exceed the ops-Lambda 900-second clock, so the swap runs CLI-side with an unbounded clock (credential-free — the operator never holds a URL). Re-run with --stage ' + stage + ' --direct.');
       process.exit(1);
@@ -229,16 +475,40 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
     process.exit(1);
   }
 
+  // --rebuild-derived carries a real outage window: the derived layer does not exist between the
+  // swap committing and db:reconcile --apply finishing. State it BEFORE the work starts — saying it
+  // only afterward tells the operator about an outage they are already in. It is now the OPT-OUT:
+  // the paired swap below is the default and has no window at all.
+  if (flags['rebuild-derived'] === 'true') {
+    warn('--rebuild-derived drops the dependent derived objects as part of the swap. They do NOT exist until db:reconcile --apply finishes — an outage window proportional to the size of the derived layer.');
+    warn('  the paired swap (the default, without this flag) rebuilds the layer over the incoming data and renames it in the same transaction — no window. Drop the flag unless you specifically want the old behaviour.');
+  }
+
   const modelsPath = resolveModelsPath(flags.models);
   let models: ModelDescriptor[];
   let declaredFingerprint: string;
+  let declaredDerivedObjects: SourceObject[] = [];
+  let paired: string[] = [];
   try {
     step(`Loading models from ${modelsPath}...`);
     models = await loadModels(modelsPath);
     const declaredDb = await loadDeclaredDerived(flags.models);
     declaredFingerprint = fingerprintModels(models, { schemas: [schema], sequences: declaredDb?.sequences }).hash;
+    // The identities db:reconcile can regenerate — what makes a dependent safe to drop.
+    declaredDerivedObjects = declaredDb?.objects ?? [];
+    // The PAIRED swap is the default whenever a declared derived schema hangs off this one: the
+    // layer is rebuilt over the incoming tables and renamed in the same transaction, so it is
+    // never absent. --rebuild-derived is the explicit opt-out (drop, swap, reconcile after), kept
+    // for the case where rebuilding twice is not worth the zero-downtime guarantee.
+    if (flags['rebuild-derived'] !== 'true' && declaredDb) {
+      paired = pairedDerivedSchemas(models, declaredDb.derived, schema);
+    }
   } catch (err: any) { fail(err.message); process.exit(1); }
 
+  if (paired.length > 0) {
+    info(`paired swap: ${paired.join(', ')} will be rebuilt over the incoming data and renamed in the SAME transaction — the derived layer is never absent.`);
+  }
+
   // Resolve --from to a local plain -Fc dump (a local file, or an S3 export id fetched down).
   let artifact: ResolvedArtifact;
   try {
@@ -263,7 +533,82 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
         models, schema,
         artifactFingerprint,
         declaredFingerprint,
-        applyIncoming: async () => { await restoreIntoIncoming(url!, artifact.dumpPath, schema, `${schema}_incoming`); },
+        // What db:reconcile can regenerate — the set a dependent must be in to be safe to drop.
+        declaredIdentities: declaredDerivedObjects.map((o) => o.identity),
+        rebuildDerived: flags['rebuild-derived'] === 'true',
+        paired,
+        // Schema-level USAGE, re-applied in the swap transaction and asserted after it commits.
+        // The incoming schemas arrive with no schema ACL, so without this the swap lands correct
+        // data behind schemas the app cannot enter — every endpoint 500s with "does not exist".
+        schemaUsage: renderSwapSchemaUsage(models, declaredDerivedObjects, schema, paired),
+        schemaUsageRoles: [...swapSchemaRoles(models, declaredDerivedObjects, [schema, ...paired])]
+          .flatMap(([s, roles]) => roles.map((role) => ({ schema: s, role }))),
+        // Every declared object the paired build must produce. Checked before the rename, so a
+        // partial layer refuses with live untouched rather than committing a silent shortfall.
+        expectedDerived: paired.length
+          ? expectedIncomingObjects(declaredDerivedObjects, schema, paired)
+          : undefined,
+        // Tell the reconciler what the build made. Without this the next db:reconcile — days
+        // later, for an unrelated edit — sees the whole layer as drift and rebuilds it under
+        // ACCESS EXCLUSIVE. The objects are correct; only the bookkeeping was missing.
+        recordProvenance: paired.length
+          ? async (r) => {
+              const live = await introspectDerived(r);
+              const prov = renderPairedProvenance(declaredDerivedObjects, live.objects, schema, paired);
+              for (const st of prov.statements) await r(st);
+              if (prov.recorded.length === 0) {
+                warn(`provenance recorded NOTHING — no declared object in ${[schema, ...paired].join(', ')} matched a live catalog entry. `
+                  + `The swap itself succeeded, but the next db:reconcile will treat this layer as drift. Run db:reconcile --check to see what it thinks.`);
+              } else {
+                info(`provenance recorded for ${prov.recorded.length} object(s) — a post-swap db:reconcile is NOT required.`);
+                if (prov.unmatched.length) {
+                  warn(`  ${prov.unmatched.length} declared object(s) had no live catalog entry and were NOT recorded: ${prov.unmatched.slice(0, 10).join(', ')}${prov.unmatched.length > 10 ? ', …' : ''}`);
+                }
+              }
+            }
+          : undefined,
+        // Build the incoming derived layer over <schema>_incoming, before the rename. Skipped
+        // entirely when nothing pairs — an all-public app's swap is byte-identical to before.
+        buildPairedDerived: paired.length
+          ? async (r) => {
+              step(`Building the incoming derived layer (${paired.join(', ')})...`);
+              const built = renderPairedDerivedBuild(declaredDerivedObjects, schema, paired);
+              for (const s of built.statements) await r(s);
+              // --dump-build: diagnostic only, no behaviour change. Writes what the build EMITTED
+              // and what the catalog HOLDS immediately afterwards, before the rename. Those two
+              // together separate "the build rendered the wrong SQL" from "the build was fine and
+              // the rename lost it" — a distinction that is otherwise only reachable by racing a
+              // second connection against the build window.
+              if (flags['dump-build']) {
+                const twins = paired.map((p) => `${p}_incoming`);
+                const rows = await r(
+                  `SELECT n.nspname || '.' || c.relname AS identity, pg_get_viewdef(c.oid) AS definition
+                     FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+                    WHERE n.nspname = ANY (ARRAY[${twins.map((t) => `'${t.replace(/'/g, "''")}'`).join(',')}])
+                      AND c.relkind IN ('v','m') ORDER BY 1`,
+                );
+                fs.writeFileSync(flags['dump-build'], JSON.stringify({
+                  schema, paired,
+                  searchPath: built.statements[0],
+                  statementCount: built.statements.length,
+                  statements: built.statements,
+                  liveInIncomingAfterBuild: rows,
+                }, null, 2));
+                info(`--dump-build: wrote ${built.statements.length} rendered statement(s) and ${rows.length} live definition(s) to ${flags['dump-build']}`);
+              }
+            }
+          : undefined,
+        log: (m) => info(m),
+        // The runner is handed in and USED: it is idle for the whole restore, so the Phase B
+        // heartbeat reads the loading backend's state over it (swap-heartbeat.ts).
+        applyIncoming: async (r) => {
+          await restoreIntoIncoming(url!, artifact.dumpPath, schema, `${schema}_incoming`, {
+            runner: r,
+            log: (m) => info(m),
+            warn: (m) => warn(m),
+            declaredIdentities: declaredDerivedObjects.map((o) => o.identity),
+          });
+        },
         snapshot: snapshotViaStage
           ? async () => {
               step('Snapshotting the stage before the swap (db:backup)...');
@@ -278,8 +623,26 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
     if (res.status === 'swapped') {
       success(`Swapped ${schema} — the artifact is live (no refresh ran).`);
       for (const w of res.warnings ?? []) warn(`verify warning: ${w.name}${w.detail ? ` — ${w.detail}` : ''}`);
+      // The derived layer was dropped with the swap. Say so LOUDLY: until reconcile runs, every
+      // reader of those objects is looking at a schema that no longer has them.
+      if (flags['rebuild-derived'] === 'true') {
+        warn(`the derived objects depending on ${schema} were dropped — they do NOT exist until you regenerate them.`);
+        warn(`  run now: everystack db:reconcile --apply --stage ${stage ?? '<stage>'} --direct`);
+      } else if (paired.length > 0) {
+        // Say it in the OUTPUT, not just the docs. The two-step "swap then reconcile" was correct
+        // advice for the unpaired swap and is baked into deploy scripts written against it. Run
+        // now, that reconcile drops and recreates the layer the swap just built correctly, holding
+        // ACCESS EXCLUSIVE — turning a zero-downtime swap into an outage. Seen in the field.
+        info(`the derived layer (${paired.join(', ')}) is live and complete — a post-swap db:reconcile is NOT required.`);
+        info(`  if your deploy script still runs one, remove it: it would drop and rebuild these objects under ACCESS EXCLUSIVE and cause the outage this swap exists to avoid.`);
+      }
     } else {
       fail(`db:swap ${res.status}: ${res.reason}`);
+      // Name every object CASCADE would have destroyed, one per line — a comma-joined list of
+      // dozens is unreadable, and this is the list the operator has to act on.
+      if (res.status === 'refused-dependents') {
+        for (const d of res.dependents ?? []) info(`  would be destroyed: ${d.schema}.${d.name} (${d.kind})`);
+      }
       process.exit(1);
     }
   } catch (err: any) {

package/src/cli/commands/db.ts

@@ -38,6 +38,51 @@ export function pgEnvFromUrl(url: string): Record<string, string> {
   return env;
 }
 
+// --- TCP keepalives: why every long-running pg binary needs them -------------------------------
+//
+// Verified failure, 2026-07-27, on a real db:swap against a deployed stage. A restore's COPY phase
+// saturates the connection; the index/constraint phase that FOLLOWS it sends one statement and then
+// goes quiet for minutes. During that quiet period a stateful middlebox on the path (NAT gateway,
+// firewall) drops the flow's state. The server's own keepalive then finds a dead peer and
+// terminates the backend. The CLIENT, on the far side of the break, never receives a FIN or RST:
+// its socket stays ESTABLISHED and it blocks on a read that will never return.
+//
+// Measured: psql alive at 8m54s with 0.05s CPU, socket ESTABLISHED and Send-Q 0, while a SECOND
+// connection to the same database confirmed via pg_stat_activity that no psql backend existed.
+//
+// Client keepalives fix both halves. Probing every 30s of quiet keeps the middlebox's state alive
+// so the flow is never dropped, and if the peer does die the client declares it dead in ~80s
+// (30 + 5 x 10) instead of hanging forever.
+//
+// These MUST travel as libpq connection parameters — there is no PG* environment variable for
+// keepalives. Passing them via `-d` keyword/value form still leaves host/user/password/sslmode to
+// the PG* env, because libpq resolves each parameter from the conninfo first and the environment
+// second. So the credential stays off argv. Mirrors @everystack/server's backup.ts.
+
+export const PG_KEEPALIVE_IDLE_S = 30;
+export const PG_KEEPALIVE_INTERVAL_S = 10;
+export const PG_KEEPALIVE_COUNT = 5;
+
+/** Escape a value for libpq keyword/value conninfo (single-quoted, backslash-escaped). */
+function conninfoValue(v: string): string {
+  return `'${v.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+}
+
+/**
+ * A libpq keyword/value conninfo carrying ONLY keepalives (plus `dbname` when given). Everything
+ * else still resolves from the PG* env — see the note above. Safe to log: it holds no credential.
+ */
+export function pgKeepaliveConninfo(dbName?: string): string {
+  const parts = [
+    'keepalives=1',
+    `keepalives_idle=${PG_KEEPALIVE_IDLE_S}`,
+    `keepalives_interval=${PG_KEEPALIVE_INTERVAL_S}`,
+    `keepalives_count=${PG_KEEPALIVE_COUNT}`,
+  ];
+  if (dbName) parts.unshift(`dbname=${conninfoValue(dbName)}`);
+  return parts.join(' ');
+}
+
 export async function dbMigrateCommand(flags: Record<string, string>): Promise<void> {
   step('Resolving deployed config...');
   let config;
@@ -160,10 +205,14 @@ export interface ProvisionSecretPlan {
 }
 
 /**
- * Decide what db:provision writes and what it announces. Both spellings of each secret
- * are written — PascalCase is the only name an `sst.Secret` component can carry (so the
- * wiring text and the store finally agree), the raw name keeps env-style consumers and
- * `secrets export` working. Pure, so the announcement contract is pinned by tests.
+ * Decide what db:provision writes and what it announces. Writes the canonical
+ * SCREAMING_SNAKE names only (DATABASE_URL / ADMIN_DATABASE_URL) — the names the
+ * reference app declares (`new sst.Secret('DATABASE_URL')`), getDatabaseUrl() /
+ * getAdminDatabaseUrl() read, and `secrets export` emits to .env. (The old both-
+ * spellings write hedged against an sst.Secret needing a PascalCase name — it does
+ * not; underscore names are valid, so the PascalCase copy had no consumer.) The
+ * READS of `existing` below still tolerate the legacy PascalCase spelling so a stage
+ * provisioned by an older CLI is still detected. Pure — the contract is pinned by tests.
  */
 export function buildProvisionSecretPlan(args: {
   result: { loginRole: string; adminRole?: string; adminVerified?: boolean | null };
@@ -174,13 +223,12 @@ export function buildProvisionSecretPlan(args: {
   const { result, authUrl, adminUrl, existing } = args;
   const updates: Record<string, string> = {
     DATABASE_URL: authUrl,
-    DatabaseUrl: authUrl,
   };
   const notes: string[] = [];
   const warnings: string[] = [];
 
   const prevAuthRole = roleOfUrl(existing.DATABASE_URL ?? existing.DatabaseUrl);
-  notes.push(`DATABASE_URL / DatabaseUrl: ${prevAuthRole ?? 'unset'} → ${result.loginRole}`);
+  notes.push(`DATABASE_URL: ${prevAuthRole ?? 'unset'} → ${result.loginRole}`);
   if (prevAuthRole && prevAuthRole !== result.loginRole) {
     warnings.push(
       `DATABASE_URL was already set (role '${prevAuthRole}') — any function linked to it connects as '${result.loginRole}' after its next cold start. Ensure grants are in place: everystack db:reconcile && everystack db:doctor.`,
@@ -189,10 +237,9 @@ export function buildProvisionSecretPlan(args: {
 
   if (result.adminRole && adminUrl) {
     updates.ADMIN_DATABASE_URL = adminUrl;
-    updates.AdminDatabaseUrl = adminUrl;
     const prevAdminRole = roleOfUrl(existing.AdminDatabaseUrl ?? existing.ADMIN_DATABASE_URL);
     const verified = result.adminVerified === true ? 'login verified' : 'login NOT verified';
-    notes.push(`AdminDatabaseUrl / ADMIN_DATABASE_URL: ${prevAdminRole ?? 'unset'} → ${result.adminRole} (${verified})`);
+    notes.push(`ADMIN_DATABASE_URL: ${prevAdminRole ?? 'unset'} → ${result.adminRole} (${verified})`);
     if (result.adminVerified !== true) {
       warnings.push(
         `The '${result.adminRole}' login could not be verified from the ops function — confirm operator connectivity before relying on it: everystack db:doctor.`,