@everystack/cli 0.4.34 → 0.4.35

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.34",
+  "version": "0.4.35",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/commands/db-backup.ts

@@ -25,7 +25,7 @@ import {
   parseBackupRef, keyForId, crossStageGuard, restoreTargetGuard,
   backupKey, backupId, metaKey, utcStamp,
 } from '../backup.js';
-import { pgEnvFromUrl } from './db.js';
+import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { pollTaskUntilStopped } from '../task-poll.js';
 import { step, success, fail, info, warn } from '../output.js';
 
@@ -57,7 +57,9 @@ export function resolveBackupVenue(flags: Record<string, string>): BackupVenue {
  * restoring it would rewind that history AND makes pg_restore --clean fail on DROP SCHEMA everystack).
  */
 export function pgDumpFullArgs(): string[] {
-  return ['-Fc', '--no-owner', '--no-privileges', '--no-comments', '--exclude-schema=everystack'];
+  // -d carries keepalives only; the credential still rides the PG* env (see db.ts). A full dump of
+  // a large database goes quiet for long stretches, which is exactly when an unkept connection dies.
+  return ['-d', pgKeepaliveConninfo(), '-Fc', '--no-owner', '--no-privileges', '--no-comments', '--exclude-schema=everystack'];
 }
 
 /** The .meta.json sidecar — byte-shape identical to the server's runBackup meta (backup-run.ts:156),

package/src/cli/commands/db-export.ts

@@ -26,7 +26,7 @@ import { fingerprintModels } from '../schema-fingerprint.js';
 import { resolveModelsPath } from '../models-path.js';
 import { loadModels } from './db-generate.js';
 import { loadDeclaredDerived } from '../declared-derived.js';
-import { pgEnvFromUrl } from './db.js';
+import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { utcStamp } from '../backup.js';
 import { pollTaskUntilStopped } from '../task-poll.js';
 import { step, success, fail, info } from '../output.js';
@@ -99,7 +99,8 @@ export function localArtifactMeta(opts: {
 
 /** pg_dump args for the local venue — same dump shape as the ops venue, written straight to a file. */
 export function pgDumpLocalArgs(schema: string, dumpPath: string): string[] {
-  return ['-Fc', '--no-owner', '--no-privileges', '--no-comments', `--schema=${schema}`, '-f', dumpPath];
+  // -d carries keepalives only; the credential still rides the PG* env (see db.ts).
+  return ['-d', pgKeepaliveConninfo(), '-Fc', '--no-owner', '--no-privileges', '--no-comments', `--schema=${schema}`, '-f', dumpPath];
 }
 
 /**

package/src/cli/commands/db-swap.ts

@@ -27,14 +27,17 @@ import { resolveModelsPath } from '../models-path.js';
 import { loadModels } from './db-generate.js';
 import { loadDeclaredDerived } from '../declared-derived.js';
 import { createUrlRunner } from '../db-source.js';
+import type { QueryRunner } from '../authz-contract.js';
 import { executeSwap, type SwapVerdict } from '../swap-execute.js';
+import { startHeartbeat, humanElapsed } from '../swap-heartbeat.js';
+import { formatBytes } from '../bundle-weight.js';
 import { rewriteStatementLine, opensCopyData, closesCopyData } from '../schema-rewrite.js';
 import { resolveOperatorUrlViaStage } from '../direct-venue.js';
 import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
 import { resolveConfig, opsFunction } from '../config.js';
 import { invokeAction, presignGet } from '../aws.js';
 import { keyForArtifactId, metaKey } from '../backup.js';
-import { pgEnvFromUrl } from './db.js';
+import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { step, success, fail, warn, info } from '../output.js';
 
 /** A COPY-aware line transform that rewrites the schema token on statement lines only. */
@@ -70,37 +73,254 @@ function schemaRewriteStream(from: string, to: string): Transform {
 }
 
 /**
- * Restore a `-Fc` artifact into `<schema>_incoming`: pg_restore -f - (archive → SQL) → the schema
- * rewrite → psql. The archive names `<schema>`; the rewrite lands it as `<incoming>`, COPY-data-safe.
+ * Consecutive polls reporting NO server-side backend before the watchdog kills psql. Three at the
+ * 10s cadence is 30s of agreement, which is well past any single-poll blip and far short of the
+ * eight minutes the unguarded version cost.
  */
-async function restoreIntoIncoming(url: string, artifactPath: string, schema: string, incoming: string): Promise<void> {
-  // Connect via PG* env, not `-d <url>`. libpq VALIDATES URI query params against its keyword
-  // list and REJECTS non-keywords like `search_path` ("invalid URI query parameter") — and the
-  // operator URL db:operator-url mints bakes search_path in (fine for postgres.js, fatal for a
-  // libpq client). pgEnvFromUrl extracts only libpq keywords (dropping search_path, routing
-  // sslmode→PGSSLMODE) and puts the password in PGPASSWORD, off the process argv. pg_restore -f -
-  // just reads the archive file to SQL on stdout (no connection).
-  const restore = spawn('pg_restore', ['-f', '-', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
-  const psql = spawn('psql', ['-v', 'ON_ERROR_STOP=1'], {
-    stdio: ['pipe', 'ignore', 'pipe'],
-    env: { ...process.env, ...pgEnvFromUrl(url) },
-  });
-  let rErr = '', pErr = '';
-  restore.stderr.on('data', (d) => { rErr += d.toString(); });
-  psql.stderr.on('data', (d) => { pErr += d.toString(); });
-  const psqlExit = new Promise<void>((res, rej) => {
-    psql.on('error', rej);
-    psql.on('close', (c) => c === 0 ? res() : rej(new Error(`psql exited ${c}: ${pErr.trim()}`)));
-  });
-  const restoreExit = new Promise<void>((res, rej) => {
-    restore.on('error', rej);
-    restore.on('close', (c) => c === 0 ? res() : rej(new Error(`pg_restore exited ${c}: ${rErr.trim()}`)));
+const DEAD_BACKEND_POLLS = 3;
+
+/**
+ * TOC entry types that a DERIVED object can appear as. Order matters: the longer types must be
+ * tested first, or `MATERIALIZED VIEW DATA` parses as `MATERIALIZED VIEW` with a mangled schema.
+ */
+const DERIVED_TOC_TYPES = [
+  'MATERIALIZED VIEW DATA',
+  'MATERIALIZED VIEW',
+  'PROCEDURE',
+  'FUNCTION',
+  'TRIGGER',
+  'VIEW',
+] as const;
+
+/**
+ * Strip the DECLARED DERIVED objects out of a pg_restore TOC listing.
+ *
+ * An artifact should carry BASE STATE — tables, data, indexes, constraints, sequences. It should
+ * not carry the derived layer, because db:reconcile owns that and rebuilds it from the descriptors.
+ * `pg_dump --schema=<s>` cannot make that distinction: it dumps everything in the schema, derived
+ * objects included.
+ *
+ * Shipping them is not merely redundant, it DEADLOCKS the swap. A derived object inside the base
+ * schema may reference the derived schema built on top of it (`stats.draft_value` returns
+ * `SETOF stats_view.draft_value_row`). The restore then cannot run until the derived layer exists,
+ * while the derived layer cannot be built until the new base tables land. Neither can go first.
+ * Measured on a real artifact: the only entries referencing the derived schema were the 16
+ * reconcile-managed functions; every other entry was state.
+ *
+ * Filtering by TOC entry rather than by SQL text is deliberate: `pg_restore -L` is the supported
+ * mechanism for restoring a subset, and the alternative is pattern-matching a six-million-line SQL
+ * file. Entries are COMMENTED OUT rather than deleted so the listing stays diffable.
+ */
+export function filterDerivedFromToc(
+  toc: string,
+  declaredIdentities: Iterable<string>,
+): { listing: string; skipped: string[] } {
+  const declared = new Set(declaredIdentities);
+  const skipped: string[] = [];
+  const listing = toc.split('\n').map((line) => {
+    // `<dumpId>; <catalogOid> <oid> <TYPE> <schema> <name(args)> <owner>`
+    const m = /^(\d+;\s+\d+\s+\d+)\s+(.+)$/.exec(line);
+    if (!m) return line; // header/comment/blank — pass through untouched
+    const rest = m[2];
+    const type = DERIVED_TOC_TYPES.find((t) => rest.startsWith(`${t} `));
+    if (!type) return line; // not a derived-capable entry (TABLE, INDEX, POLICY, ...)
+    const after = rest.slice(type.length + 1);
+    const parts = after.split(/\s+/);
+    if (parts.length < 2) return line;
+    const schema = parts[0];
+    // The owner is the last token; everything between it and the schema is the name (+args).
+    const nameWithArgs = parts.slice(1, -1).join(' ');
+    const name = nameWithArgs.replace(/\(.*$/, ''); // drop the argument list
+    if (!declared.has(`${schema}.${name}`)) return line;
+    skipped.push(`${schema}.${name} (${type.toLowerCase()})`);
+    return `;${line}`;
+  }).join('\n');
+  return { listing, skipped };
+}
+
+/** A pass-through that counts the bytes crossing it — Phase A's progress signal. */
+function countingTap(onBytes: (total: number) => void): Transform {
+  let total = 0;
+  return new Transform({
+    transform(chunk, _enc, cb) {
+      total += chunk.length;
+      onBytes(total);
+      cb(null, chunk);
+    },
   });
-  await Promise.all([
-    pipeline(restore.stdout!, schemaRewriteStream(schema, incoming), psql.stdin!),
-    restoreExit,
-    psqlExit,
-  ]);
+}
+
+/** The observation sinks the restore reports through. Injected so the phases stay testable. */
+interface RestoreIO {
+  /** The operator connection — IDLE for the whole restore, so the Phase B heartbeat reuses it. */
+  runner: QueryRunner;
+  log: (msg: string) => void;
+  warn: (msg: string) => void;
+  /**
+   * `schema.name` of every DECLARED derived object. These are stripped from the restore: the
+   * artifact carries base state, db:reconcile owns the derived layer. See filterDerivedFromToc.
+   */
+  declaredIdentities: string[];
+}
+
+/**
+ * Restore a `-Fc` artifact into `<schema>_incoming`, in TWO phases split by a local temp file:
+ *
+ *   Phase A (local, no network): pg_restore -f - (archive → SQL) → the COPY-safe schema rewrite →
+ *     a local `.sql` file. The archive names `<schema>`; the rewrite lands it as `<incoming>`.
+ *   Phase B (network): psql -f <file> streams that file to the target at psql's own pace.
+ *
+ * Why the temp file and not a live `pg_restore | rewrite | psql` pipe: decoupling the producer
+ * (local, fast) from the consumer removes all cross-process backpressure, and it makes Phase A
+ * measurable on its own. Costs one temp file (~the uncompressed dump), cleaned up in `finally`.
+ *
+ * A CORRECTION, because the original note here sent two debugging sessions down the wrong path.
+ * It claimed the three-way pipe DEADLOCKED at the DDL→data boundary and that the temp file was the
+ * cure. The temp file went in, and the restore STILL died at the same place. The pipe was never the
+ * root cause; the `write EPIPE` it produced was a downstream symptom.
+ *
+ * The real cause (proven 2026-07-27, see db.ts's keepalive note): the connection dies during the
+ * long quiet stretch of the index/constraint phase, the server terminates the backend, and the
+ * client — which never sees a FIN or RST — blocks forever on a socket its kernel still calls
+ * ESTABLISHED. The fix is TCP keepalives on psql's connection, plus the watchdog below, NOT the
+ * process topology. The temp file is kept because it is genuinely better instrumented, not because
+ * it fixes a deadlock.
+ *
+ * Both phases are INSTRUMENTED. This ran silent once — a multi-GB push with no output at all — and
+ * a run that died unattended was indistinguishable from one still working. Now: a byte counter on
+ * Phase A, a server-side heartbeat on Phase B (see swap-heartbeat.ts), and both child processes'
+ * stderr streamed as it arrives instead of being withheld until exit.
+ */
+async function restoreIntoIncoming(
+  url: string,
+  artifactPath: string,
+  schema: string,
+  incoming: string,
+  io: RestoreIO,
+): Promise<void> {
+  const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'everystack-restore-'));
+  const sqlPath = path.join(tmpDir, `${incoming}.sql`);
+  const t0 = Date.now();
+  try {
+    // Phase A0: strip the declared derived layer from the restore. The artifact is base STATE;
+    // db:reconcile owns derived. Landing them here is redundant at best and deadlocks the restore
+    // when a derived object in the base schema references the derived schema above it.
+    const tocPath = path.join(tmpDir, 'toc.list');
+    const toc = await new Promise<string>((res, rej) => {
+      const p = spawn('pg_restore', ['-l', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
+      let out = '', err = '';
+      p.stdout.on('data', (d) => { out += d.toString(); });
+      p.stderr.on('data', (d) => { err += d.toString(); });
+      p.on('error', rej);
+      p.on('close', (c) => c === 0 ? res(out) : rej(new Error(`pg_restore -l exited ${c}: ${err.trim()}`)));
+    });
+    const { listing, skipped } = filterDerivedFromToc(toc, io.declaredIdentities);
+    await fs.promises.writeFile(tocPath, listing, 'utf8');
+    io.log(
+      skipped.length
+        ? `restore: skipping ${skipped.length} declared derived object(s) — db:reconcile owns them (${skipped.slice(0, 6).join(', ')}${skipped.length > 6 ? `, +${skipped.length - 6} more` : ''}).`
+        : 'restore: the artifact carries no declared derived objects.',
+    );
+
+    // Phase A: pg_restore → rewrite → local file. Only local processes; nothing can stall here.
+    io.log(`restore phase A: pg_restore → schema rewrite (${schema} → ${incoming}) → ${sqlPath}`);
+    const restore = spawn('pg_restore', ['-L', tocPath, '-f', '-', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
+    let rErr = '';
+    restore.stderr.on('data', (d) => {
+      const s = d.toString();
+      rErr += s;
+      // Surface as it happens — a warning withheld until exit is a warning nobody can act on.
+      for (const line of s.split('\n').map((l: string) => l.trim()).filter(Boolean)) io.log(`pg_restore: ${line}`);
+    });
+    const restoreExit = new Promise<void>((res, rej) => {
+      restore.on('error', rej);
+      restore.on('close', (c) => c === 0 ? res() : rej(new Error(`pg_restore exited ${c}: ${rErr.trim()}`)));
+    });
+    let written = 0;
+    const phaseATimer = setInterval(() => {
+      io.log(`restore phase A: ${formatBytes(written)} of SQL written (t+${humanElapsed(Date.now() - t0)})...`);
+    }, 5_000);
+    (phaseATimer as any).unref?.();
+    try {
+      await Promise.all([
+        pipeline(
+          restore.stdout!,
+          countingTap((n) => { written = n; }),
+          schemaRewriteStream(schema, incoming),
+          fs.createWriteStream(sqlPath),
+        ),
+        restoreExit,
+      ]);
+    } finally {
+      clearInterval(phaseATimer);
+    }
+    const aMs = Date.now() - t0;
+    io.log(`restore phase A done: ${formatBytes(written)} of SQL in ${humanElapsed(aMs)}.`);
+
+    // Phase B: psql reads the local file and streams to the target. The credential rides PG* env,
+    // never argv (libpq also REJECTS non-keyword URI params like the `search_path` the operator URL
+    // bakes in — fine for postgres.js, fatal for a libpq URI). `-d` carries ONLY keepalives, which
+    // have no PG* env equivalent and are what keep this connection from dying in the index phase.
+    io.log(`restore phase B: psql streaming ${formatBytes(written)} to the target — heartbeat every 10s.`);
+    const psql = spawn('psql', ['-d', pgKeepaliveConninfo(), '-v', 'ON_ERROR_STOP=1', '-f', sqlPath], {
+      stdio: ['ignore', 'ignore', 'pipe'],
+      env: { ...process.env, ...pgEnvFromUrl(url) },
+    });
+    let pErr = '';
+    psql.stderr.on('data', (d) => {
+      const s = d.toString();
+      pErr += s;
+      for (const line of s.split('\n').map((l: string) => l.trim()).filter(Boolean)) io.warn(`psql: ${line}`);
+    });
+
+    // The watchdog. Keepalives should prevent the dead-backend hang, but if it happens anyway the
+    // heartbeat SEES it — pg_stat_activity, read over our own live connection, reports no psql
+    // backend while the psql process sits there forever. Detection without action is what cost an
+    // eight-minute wait: turn it into a kill and a named failure.
+    let deadBackendPolls = 0;
+    const stopHeartbeat = startHeartbeat(io.runner, {
+      incoming,
+      log: io.log,
+      warn: io.warn,
+      onSample: (sample) => {
+        if (sample.state === null) deadBackendPolls += 1;
+        else deadBackendPolls = 0;
+        if (deadBackendPolls === DEAD_BACKEND_POLLS && psql.exitCode === null) {
+          io.warn(
+            `restore: psql is still running but the server reports NO backend for it after ${deadBackendPolls} consecutive polls. `
+            + 'The connection died and psql will never notice (no FIN/RST reaches it). Killing it rather than hanging.',
+          );
+          psql.kill('SIGTERM');
+          // SIGTERM will not land if psql is parked in a blocking read on a dead socket.
+          const hardKill = setTimeout(() => { if (psql.exitCode === null) psql.kill('SIGKILL'); }, 5_000);
+          (hardKill as any).unref?.();
+        }
+      },
+    });
+    const bStart = Date.now();
+    try {
+      await new Promise<void>((res, rej) => {
+        psql.on('error', rej);
+        psql.on('close', (c) => {
+          if (c === 0) return res();
+          if (deadBackendPolls >= DEAD_BACKEND_POLLS) {
+            return rej(new Error(
+              'the restore connection died mid-load and psql hung on it (the server had no backend for it). '
+              + 'psql was killed by the watchdog; NOTHING was swapped and live is untouched. '
+              + 'This is the network path dropping a connection that goes quiet during the index/constraint phase — '
+              + 'keepalives are now set, so if you are seeing this the path is dropping the flow faster than a 30s probe interval.',
+            ));
+          }
+          rej(new Error(`psql exited ${c}: ${pErr.trim()}`));
+        });
+      });
+    } finally {
+      await stopHeartbeat();
+    }
+    io.log(`restore phase B done in ${humanElapsed(Date.now() - bStart)} (restore total ${humanElapsed(Date.now() - t0)}).`);
+  } finally {
+    await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
+  }
 }
 
 /**
@@ -229,14 +449,24 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
     process.exit(1);
   }
 
+  // --rebuild-derived carries a real outage window: the derived layer does not exist between the
+  // swap committing and db:reconcile --apply finishing. State it BEFORE the work starts — saying it
+  // only afterward tells the operator about an outage they are already in.
+  if (flags['rebuild-derived'] === 'true') {
+    warn('--rebuild-derived drops the dependent derived objects as part of the swap. They do NOT exist until db:reconcile --apply finishes — an outage window proportional to the size of the derived layer.');
+  }
+
   const modelsPath = resolveModelsPath(flags.models);
   let models: ModelDescriptor[];
   let declaredFingerprint: string;
+  let declaredDerivedObjects: Array<{ identity: string }> = [];
   try {
     step(`Loading models from ${modelsPath}...`);
     models = await loadModels(modelsPath);
     const declaredDb = await loadDeclaredDerived(flags.models);
     declaredFingerprint = fingerprintModels(models, { schemas: [schema], sequences: declaredDb?.sequences }).hash;
+    // The identities db:reconcile can regenerate — what makes a dependent safe to drop.
+    declaredDerivedObjects = declaredDb?.objects ?? [];
   } catch (err: any) { fail(err.message); process.exit(1); }
 
   // Resolve --from to a local plain -Fc dump (a local file, or an S3 export id fetched down).
@@ -263,7 +493,20 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
         models, schema,
         artifactFingerprint,
         declaredFingerprint,
-        applyIncoming: async () => { await restoreIntoIncoming(url!, artifact.dumpPath, schema, `${schema}_incoming`); },
+        // What db:reconcile can regenerate — the set a dependent must be in to be safe to drop.
+        declaredIdentities: declaredDerivedObjects.map((o) => o.identity),
+        rebuildDerived: flags['rebuild-derived'] === 'true',
+        log: (m) => info(m),
+        // The runner is handed in and USED: it is idle for the whole restore, so the Phase B
+        // heartbeat reads the loading backend's state over it (swap-heartbeat.ts).
+        applyIncoming: async (r) => {
+          await restoreIntoIncoming(url!, artifact.dumpPath, schema, `${schema}_incoming`, {
+            runner: r,
+            log: (m) => info(m),
+            warn: (m) => warn(m),
+            declaredIdentities: declaredDerivedObjects.map((o) => o.identity),
+          });
+        },
         snapshot: snapshotViaStage
           ? async () => {
               step('Snapshotting the stage before the swap (db:backup)...');
@@ -278,8 +521,19 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
     if (res.status === 'swapped') {
       success(`Swapped ${schema} — the artifact is live (no refresh ran).`);
       for (const w of res.warnings ?? []) warn(`verify warning: ${w.name}${w.detail ? ` — ${w.detail}` : ''}`);
+      // The derived layer was dropped with the swap. Say so LOUDLY: until reconcile runs, every
+      // reader of those objects is looking at a schema that no longer has them.
+      if (flags['rebuild-derived'] === 'true') {
+        warn(`the derived objects depending on ${schema} were dropped — they do NOT exist until you regenerate them.`);
+        warn(`  run now: everystack db:reconcile --apply --stage ${stage ?? '<stage>'} --direct`);
+      }
     } else {
       fail(`db:swap ${res.status}: ${res.reason}`);
+      // Name every object CASCADE would have destroyed, one per line — a comma-joined list of
+      // dozens is unreadable, and this is the list the operator has to act on.
+      if (res.status === 'refused-dependents') {
+        for (const d of res.dependents ?? []) info(`  would be destroyed: ${d.schema}.${d.name} (${d.kind})`);
+      }
       process.exit(1);
     }
   } catch (err: any) {

package/src/cli/commands/db.ts

@@ -38,6 +38,51 @@ export function pgEnvFromUrl(url: string): Record<string, string> {
   return env;
 }
 
+// --- TCP keepalives: why every long-running pg binary needs them -------------------------------
+//
+// Verified failure, 2026-07-27, on a real db:swap against a deployed stage. A restore's COPY phase
+// saturates the connection; the index/constraint phase that FOLLOWS it sends one statement and then
+// goes quiet for minutes. During that quiet period a stateful middlebox on the path (NAT gateway,
+// firewall) drops the flow's state. The server's own keepalive then finds a dead peer and
+// terminates the backend. The CLIENT, on the far side of the break, never receives a FIN or RST:
+// its socket stays ESTABLISHED and it blocks on a read that will never return.
+//
+// Measured: psql alive at 8m54s with 0.05s CPU, socket ESTABLISHED and Send-Q 0, while a SECOND
+// connection to the same database confirmed via pg_stat_activity that no psql backend existed.
+//
+// Client keepalives fix both halves. Probing every 30s of quiet keeps the middlebox's state alive
+// so the flow is never dropped, and if the peer does die the client declares it dead in ~80s
+// (30 + 5 x 10) instead of hanging forever.
+//
+// These MUST travel as libpq connection parameters — there is no PG* environment variable for
+// keepalives. Passing them via `-d` keyword/value form still leaves host/user/password/sslmode to
+// the PG* env, because libpq resolves each parameter from the conninfo first and the environment
+// second. So the credential stays off argv. Mirrors @everystack/server's backup.ts.
+
+export const PG_KEEPALIVE_IDLE_S = 30;
+export const PG_KEEPALIVE_INTERVAL_S = 10;
+export const PG_KEEPALIVE_COUNT = 5;
+
+/** Escape a value for libpq keyword/value conninfo (single-quoted, backslash-escaped). */
+function conninfoValue(v: string): string {
+  return `'${v.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+}
+
+/**
+ * A libpq keyword/value conninfo carrying ONLY keepalives (plus `dbname` when given). Everything
+ * else still resolves from the PG* env — see the note above. Safe to log: it holds no credential.
+ */
+export function pgKeepaliveConninfo(dbName?: string): string {
+  const parts = [
+    'keepalives=1',
+    `keepalives_idle=${PG_KEEPALIVE_IDLE_S}`,
+    `keepalives_interval=${PG_KEEPALIVE_INTERVAL_S}`,
+    `keepalives_count=${PG_KEEPALIVE_COUNT}`,
+  ];
+  if (dbName) parts.unshift(`dbname=${conninfoValue(dbName)}`);
+  return parts.join(' ');
+}
+
 export async function dbMigrateCommand(flags: Record<string, string>): Promise<void> {
   step('Resolving deployed config...');
   let config;
@@ -160,10 +205,14 @@ export interface ProvisionSecretPlan {
 }
 
 /**
- * Decide what db:provision writes and what it announces. Both spellings of each secret
- * are written — PascalCase is the only name an `sst.Secret` component can carry (so the
- * wiring text and the store finally agree), the raw name keeps env-style consumers and
- * `secrets export` working. Pure, so the announcement contract is pinned by tests.
+ * Decide what db:provision writes and what it announces. Writes the canonical
+ * SCREAMING_SNAKE names only (DATABASE_URL / ADMIN_DATABASE_URL) — the names the
+ * reference app declares (`new sst.Secret('DATABASE_URL')`), getDatabaseUrl() /
+ * getAdminDatabaseUrl() read, and `secrets export` emits to .env. (The old both-
+ * spellings write hedged against an sst.Secret needing a PascalCase name — it does
+ * not; underscore names are valid, so the PascalCase copy had no consumer.) The
+ * READS of `existing` below still tolerate the legacy PascalCase spelling so a stage
+ * provisioned by an older CLI is still detected. Pure — the contract is pinned by tests.
  */
 export function buildProvisionSecretPlan(args: {
   result: { loginRole: string; adminRole?: string; adminVerified?: boolean | null };
@@ -174,13 +223,12 @@ export function buildProvisionSecretPlan(args: {
   const { result, authUrl, adminUrl, existing } = args;
   const updates: Record<string, string> = {
     DATABASE_URL: authUrl,
-    DatabaseUrl: authUrl,
   };
   const notes: string[] = [];
   const warnings: string[] = [];
 
   const prevAuthRole = roleOfUrl(existing.DATABASE_URL ?? existing.DatabaseUrl);
-  notes.push(`DATABASE_URL / DatabaseUrl: ${prevAuthRole ?? 'unset'} → ${result.loginRole}`);
+  notes.push(`DATABASE_URL: ${prevAuthRole ?? 'unset'} → ${result.loginRole}`);
   if (prevAuthRole && prevAuthRole !== result.loginRole) {
     warnings.push(
       `DATABASE_URL was already set (role '${prevAuthRole}') — any function linked to it connects as '${result.loginRole}' after its next cold start. Ensure grants are in place: everystack db:reconcile && everystack db:doctor.`,
@@ -189,10 +237,9 @@ export function buildProvisionSecretPlan(args: {
 
   if (result.adminRole && adminUrl) {
     updates.ADMIN_DATABASE_URL = adminUrl;
-    updates.AdminDatabaseUrl = adminUrl;
     const prevAdminRole = roleOfUrl(existing.AdminDatabaseUrl ?? existing.ADMIN_DATABASE_URL);
     const verified = result.adminVerified === true ? 'login verified' : 'login NOT verified';
-    notes.push(`AdminDatabaseUrl / ADMIN_DATABASE_URL: ${prevAdminRole ?? 'unset'} → ${result.adminRole} (${verified})`);
+    notes.push(`ADMIN_DATABASE_URL: ${prevAdminRole ?? 'unset'} → ${result.adminRole} (${verified})`);
     if (result.adminVerified !== true) {
       warnings.push(
         `The '${result.adminRole}' login could not be verified from the ops function — confirm operator connectivity before relying on it: everystack db:doctor.`,

package/src/cli/swap-execute.ts

@@ -54,11 +54,181 @@ export interface ExecuteSwapOptions {
   verify?: () => Promise<SwapVerdict>;
   /** Restore the stage to the snapshot from step 2 when verify is fatal. */
   rollbackToSnapshot?: () => Promise<void>;
+  /** Progress/instrumentation sink (per-table landed rows, the count assertion). Defaults to a no-op. */
+  log?: (msg: string) => void;
+  /**
+   * `schema.name` of every object the DECLARED derived layer can regenerate (db:reconcile's source
+   * of truth). A dependent in this set is safe to drop: reconcile rebuilds it from the descriptor.
+   * A dependent NOT in it can never be rebuilt, so it is refused unconditionally.
+   */
+  declaredIdentities?: string[];
+  /**
+   * Informed consent to DROP the dependent derived objects as part of the swap (`--rebuild-derived`).
+   *
+   * This is the intended workflow — swap the schema, then regenerate the derived layer — not a
+   * workaround. The gate exists to stop SILENT destruction, so an operator who says "drop them, I
+   * will reconcile after" has supplied exactly the consent that was missing. Undeclared dependents
+   * are still refused: consent cannot cover an object nothing knows how to rebuild.
+   */
+  rebuildDerived?: boolean;
+}
+
+/** One table's landed-vs-live count, the intrinsic post-swap assertion's unit. */
+export interface RowCount {
+  table: string;
+  incoming: number;
+  live: number;
+}
+
+/**
+ * The intrinsic count assertion: after the rename, the LIVE schema must hold exactly what the
+ * artifact landed in the incoming schema. A no-op swap (the rename silently not taking) leaves the
+ * OLD table under `<schema>.<table>` — its count differs from what was restored, and this catches it.
+ * A table present in incoming but absent live (a partial rename) is a mismatch too. Pure — the impure
+ * path reads the two count maps and hands them here, so the verdict wording is unit-tested.
+ */
+export function diffRowCounts(counts: RowCount[]): SwapCheck[] {
+  const bad: SwapCheck[] = [];
+  for (const c of counts) {
+    if (c.incoming !== c.live) {
+      bad.push({
+        name: `rowcount:${c.table}`,
+        ok: false,
+        detail: `artifact landed ${c.incoming} rows but live ${c.table} has ${c.live} after swap — the swap did not take effect (or landed partial data).`,
+        severity: 'fatal',
+      });
+    }
+  }
+  return bad;
+}
+
+/** One object OUTSIDE the swapped schema that depends on something inside it. */
+export interface CrossSchemaDependent {
+  schema: string;
+  name: string;
+  /** `view`, `materialized view`, `function`, `procedure`. */
+  kind: string;
+  /** Identity arguments for a function/procedure — DROP FUNCTION needs them when overloaded. */
+  args?: string;
+}
+
+/** `schema.name`, the join key against the declared derived layer's identities. */
+export function dependentIdentity(d: CrossSchemaDependent): string {
+  return `${d.schema}.${d.name}`;
+}
+
+/**
+ * The DROP for one dependent. CASCADE is safe here and ordering does not matter: the dependent set
+ * is a transitive CLOSURE, so anything CASCADE could reach is already in the set being dropped.
+ */
+export function dropDependentSql(d: CrossSchemaDependent): string {
+  const ref = `"${d.schema.replace(/"/g, '""')}"."${d.name.replace(/"/g, '""')}"`;
+  if (d.kind === 'materialized view') return `DROP MATERIALIZED VIEW IF EXISTS ${ref} CASCADE;`;
+  if (d.kind === 'view') return `DROP VIEW IF EXISTS ${ref} CASCADE;`;
+  // Functions can be overloaded, so the identity arguments are load-bearing.
+  const kw = d.kind === 'procedure' ? 'PROCEDURE' : 'FUNCTION';
+  return `DROP ${kw} IF EXISTS ${ref}(${d.args ?? ''}) CASCADE;`;
+}
+
+/**
+ * Objects in OTHER schemas that depend on the swapped schema — the CASCADE blast radius.
+ *
+ * Why this gate exists. The swap ends with `DROP SCHEMA <schema>_retiring CASCADE`. PostgreSQL
+ * binds a view, matview or function to its dependencies by OID, so the rename carries every such
+ * binding onto the RETIRING schema — and CASCADE then drops them. Silently. The swap already
+ * understands this for foreign keys (it drops and recreates them around the rename, see
+ * crossSchemaForeignKeys) and was blind to everything else.
+ *
+ * The walk must be TRANSITIVE, and the first version was not. Matching only DIRECT dependents,
+ * given `stats.t` ← `other.v1` ← `other.v2`, it found v1 and missed v2 entirely. v2 binds to v1 by
+ * OID, so when v1 goes so does v2. Worse: when the only direct dependent lives INSIDE the swapped
+ * schema, a direct-only query reports NOTHING while CASCADE still destroys everything downstream.
+ *
+ * So: seed the closure with every relation, type and function in the schema, then walk pg_depend
+ * forward to fixpoint. Views and matviews reach their dependencies through pg_rewrite, so an edge
+ * is normalized to the OWNING relation (`rw.ev_class`), not the rewrite rule. Types are seeded
+ * because `RETURNS SETOF <schema>.<table>` records against the table's composite TYPE rather than
+ * the table — a pg_class-only walk misses every such function.
+ *
+ * Objects INSIDE the schema are excluded (they ride along with the swap), as are the incoming and
+ * retiring schemas (transient, and a leftover `<schema>_incoming` from a failed run must not make
+ * the gate refuse forever). Foreign keys never surface here: they are pg_constraint, and the swap
+ * drops and recreates them itself.
+ *
+ * Verified against a live PostgreSQL 16 fixture: a three-deep view chain, a function returning
+ * SETOF a table, a view inside the schema, and an unrelated view in another schema.
+ */
+export function crossSchemaDependentsQuery(schema: string, incoming: string, retiring: string): string {
+  const q = (s: string) => s.replace(/'/g, "''");
+  const skip = `ARRAY['${q(incoming)}','${q(retiring)}']`;
+  const notSelf = `n.nspname <> '${q(schema)}' AND n.nspname <> ALL (${skip})`;
+  return `WITH RECURSIVE seed AS (
+  SELECT c.oid AS oid, 'pg_class'::regclass AS cls
+    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = '${q(schema)}'
+  UNION
+  SELECT t.oid, 'pg_type'::regclass
+    FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE n.nspname = '${q(schema)}'
+  UNION
+  SELECT p.oid, 'pg_proc'::regclass
+    FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname = '${q(schema)}'
+), closure AS (
+  SELECT oid, cls FROM seed
+  UNION
+  SELECT CASE WHEN d.classid = 'pg_rewrite'::regclass THEN rw.ev_class ELSE d.objid END,
+         CASE WHEN d.classid = 'pg_rewrite'::regclass THEN 'pg_class'::regclass ELSE d.classid END
+    FROM closure cl
+    JOIN pg_depend d ON d.refclassid = cl.cls AND d.refobjid = cl.oid
+    LEFT JOIN pg_rewrite rw ON rw.oid = d.objid AND d.classid = 'pg_rewrite'::regclass
+   WHERE d.classid IN ('pg_rewrite'::regclass, 'pg_class'::regclass, 'pg_proc'::regclass, 'pg_type'::regclass)
+)
+SELECT DISTINCT n.nspname AS dep_schema, c.relname AS dep_name,
+       CASE c.relkind WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' ELSE c.relkind::text END AS dep_kind,
+       ''::text AS dep_args
+  FROM closure cl
+  JOIN pg_class c     ON c.oid = cl.oid AND cl.cls = 'pg_class'::regclass
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+ WHERE c.relkind IN ('v', 'm') AND ${notSelf}
+UNION
+SELECT DISTINCT n.nspname, p.proname,
+       CASE p.prokind WHEN 'p' THEN 'procedure' ELSE 'function' END,
+       pg_get_function_identity_arguments(p.oid)
+  FROM closure cl
+  JOIN pg_proc p      ON p.oid = cl.oid AND cl.cls = 'pg_proc'::regclass
+  JOIN pg_namespace n ON n.oid = p.pronamespace
+ WHERE ${notSelf}
+ ORDER BY 1, 2`;
+}
+
+/** The refusal message — names what is at risk, and which of the two refusals this is. */
+export function crossSchemaDependentsRefusal(
+  schema: string,
+  deps: CrossSchemaDependent[],
+  opts: { rebuildRequested?: boolean; undeclared?: CrossSchemaDependent[] } = {},
+): string {
+  const name = (d: CrossSchemaDependent) => `${d.schema}.${d.name} (${d.kind})`;
+  const undeclared = opts.undeclared ?? [];
+
+  // Refusal 1: consent was given, but something in the set cannot be regenerated.
+  if (opts.rebuildRequested && undeclared.length > 0) {
+    return `${undeclared.length} of the ${deps.length} object(s) depending on ${schema} are NOT declared, so nothing can regenerate them: `
+      + `${undeclared.slice(0, 20).map(name).join(', ')}${undeclared.length > 20 ? `, and ${undeclared.length - 20} more` : ''}. `
+      + `--rebuild-derived drops dependents on the promise that db:reconcile --apply rebuilds them, and that promise does not hold for an undeclared object — dropping it would destroy it. `
+      + `Nothing was changed. Declare them in db/models (then db:reconcile --apply), or drop them yourself if they are genuinely disposable.`;
+  }
+
+  // Refusal 2: no consent given. Explain the danger and the flag.
+  const listed = deps.slice(0, 20).map(name).join(', ');
+  const more = deps.length > 20 ? `, and ${deps.length - 20} more` : '';
+  return `${deps.length} object(s) outside ${schema} depend on it, and the swap would DESTROY them: ${listed}${more}. `
+    + `PostgreSQL binds views, matviews and functions to their dependencies by OID, so renaming ${schema} carries those bindings onto ${schema}_retiring, and the swap's final DROP SCHEMA ... CASCADE drops them silently. `
+    + `Nothing was changed — this refusal happens before the snapshot. `
+    + `If regenerating them is what you want, re-run with --rebuild-derived: the swap drops them in the same transaction and db:reconcile --apply rebuilds them from the declared descriptors.`;
 }
 
 export type SwapStatus =
   | 'swapped'
   | 'refused-fingerprint'
+  | 'refused-dependents'
   | 'refused-integrity'
   | 'rolled-back-verify';
 
@@ -69,6 +239,10 @@ export interface SwapResult {
   verdict?: SwapVerdict;
   /** Warn-severity checks that did NOT trigger rollback (surfaced). */
   warnings?: SwapCheck[];
+  /** On `refused-dependents`: exactly what CASCADE would have destroyed. */
+  dependents?: CrossSchemaDependent[];
+  /** The subset nothing can regenerate — the reason --rebuild-derived was not enough. */
+  undeclaredDependents?: CrossSchemaDependent[];
 }
 
 /** Does the verdict force a rollback? Any failing check whose severity is fatal (the default). */
@@ -79,7 +253,27 @@ export function isFatalVerdict(verdict: SwapVerdict): boolean {
   return checks.some((c) => !c.ok && (c.severity ?? 'fatal') === 'fatal');
 }
 
+/** Double-quote a Postgres identifier read from the catalog (embedded quotes doubled). */
+function quoteIdent(name: string): string {
+  return `"${name.replace(/"/g, '""')}"`;
+}
+
+/** The base tables physically present in `schema` (what a restore actually landed / what live serves). */
+async function baseTablesIn(runner: QueryRunner, schema: string): Promise<string[]> {
+  const rows = await runner(
+    `SELECT table_name FROM information_schema.tables WHERE table_schema = '${schema.replace(/'/g, "''")}' AND table_type = 'BASE TABLE' ORDER BY table_name`,
+  );
+  return rows.map((r: any) => r.table_name as string);
+}
+
+/** `count(*)` for one table, as a number. */
+async function countRows(runner: QueryRunner, schema: string, table: string): Promise<number> {
+  const rows = await runner(`SELECT count(*)::bigint AS n FROM ${quoteIdent(schema)}.${quoteIdent(table)}`);
+  return Number(rows[0]?.n ?? 0);
+}
+
 export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions): Promise<SwapResult> {
+  const log = opts.log ?? (() => {});
   // 1. Fingerprint gate — declared-vs-declared: does the artifact and the target agree on the shape.
   if (opts.artifactFingerprint !== opts.declaredFingerprint) {
     return {
@@ -90,12 +284,63 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
 
   const plan = renderSchemaSwap(opts.models, { schema: opts.schema, incoming: opts.incoming, retiring: opts.retiring });
 
+  // 1b. THE CASCADE GATE. Refuse before the snapshot — before anything moves at all — if any object
+  //     outside this schema depends on it. The swap's final DROP SCHEMA ... CASCADE would destroy
+  //     them silently, and a silent destroyer is the worst thing this command could be.
+  const depRows = await runner(crossSchemaDependentsQuery(opts.schema, plan.incoming, plan.retiring));
+  const dependents: CrossSchemaDependent[] = depRows.map((r: any) => ({
+    schema: r.dep_schema, name: r.dep_name, kind: r.dep_kind, args: r.dep_args || undefined,
+  }));
+  if (dependents.length > 0) {
+    const declared = new Set(opts.declaredIdentities ?? []);
+    const undeclared = dependents.filter((d) => !declared.has(dependentIdentity(d)));
+
+    // Consent covers only what can be rebuilt. An undeclared dependent is refused even WITH
+    // --rebuild-derived: dropping it destroys it, because nothing knows how to recreate it.
+    if (!opts.rebuildDerived || undeclared.length > 0) {
+      return {
+        status: 'refused-dependents',
+        reason: crossSchemaDependentsRefusal(opts.schema, dependents, {
+          rebuildRequested: !!opts.rebuildDerived,
+          undeclared,
+        }),
+        dependents,
+        undeclaredDependents: undeclared.length ? undeclared : undefined,
+      };
+    }
+
+    // Consented, and every one is declared: drop them inside the swap transaction, before the
+    // rename. db:reconcile --apply regenerates them from the descriptors afterwards.
+    // Future tense on purpose: these DROPs ride the swap transaction, which is several steps away
+    // and may never run. Announcing them as done was misleading on a restore that failed first.
+    log(`${dependents.length} declared derived object(s) depend on ${opts.schema} and WILL BE DROPPED by the swap transaction (regenerate with db:reconcile --apply). Nothing is dropped unless the swap itself commits.`);
+    plan.statements.unshift(...dependents.map(dropDependentSql));
+  }
+
   // 2. Snapshot (the rollback point) before anything destructive.
   if (opts.snapshot) await opts.snapshot();
 
-  // 3. Land the incoming schema while live keeps serving.
+  // 3. Land the incoming schema while live keeps serving. Drop a stale incoming FIRST so a retry
+  //    after a failed restore is idempotent (a half-landed <schema>_incoming from a prior crash
+  //    would otherwise collide on the artifact's CREATE SCHEMA and fail every retry).
+  await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(plan.incoming)} CASCADE`);
   await opts.applyIncoming(runner);
 
+  // Capture what the artifact actually landed, per table — the count the live schema must match
+  // after the swap. Empty here means the restore created the schema but no tables: a silent
+  // partial that must fail, not pass.
+  const incomingTables = await baseTablesIn(runner, plan.incoming);
+  const expected = new Map<string, number>();
+  for (const t of incomingTables) expected.set(t, await countRows(runner, plan.incoming, t));
+  log(`landed ${incomingTables.length} table(s) into ${plan.incoming}: ${incomingTables.map((t) => `${t}=${expected.get(t)}`).join(', ') || '(none)'}`);
+  if (incomingTables.length === 0) {
+    try { await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(plan.incoming)} CASCADE`); } catch { /* best effort */ }
+    return {
+      status: 'refused-integrity',
+      reason: `the restore landed NO base tables into ${plan.incoming} — the artifact did not apply (a restore that reported success but wrote nothing). The swap was NOT applied; live is untouched.`,
+    };
+  }
+
   // 4. The atomic swap. A FK re-validation failure (a bad artifact) rolls the whole thing back.
   await runner('BEGIN');
   try {
@@ -111,7 +356,23 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
     };
   }
 
-  // 5. Verify (post-commit). Fatal → roll back to the snapshot; warn → surface only.
+  // 5a. The intrinsic count assertion (ALWAYS runs — this is the safety net a silent no-op needs).
+  //     After the rename, live must hold exactly what landed. A mismatch means the swap did not
+  //     take effect; roll back to the snapshot and refuse LOUD.
+  const counts: RowCount[] = [];
+  for (const t of incomingTables) counts.push({ table: t, incoming: expected.get(t)!, live: await countRows(runner, opts.schema, t) });
+  log(`post-swap live counts: ${counts.map((c) => `${c.table}=${c.live}`).join(', ')}`);
+  const countChecks = diffRowCounts(counts);
+  if (countChecks.length > 0) {
+    if (opts.rollbackToSnapshot) await opts.rollbackToSnapshot();
+    return {
+      status: 'rolled-back-verify',
+      reason: `post-swap row counts do not match the artifact — the swap did not land: ${countChecks.map((c) => c.detail).join(' ')} ${opts.rollbackToSnapshot ? 'Rolled back to the pre-swap snapshot.' : 'NO snapshot was configured to roll back to.'}`,
+      verdict: { ok: false, checks: countChecks },
+    };
+  }
+
+  // 5b. Verify (post-commit). Fatal → roll back to the snapshot; warn → surface only.
   let verdict: SwapVerdict | undefined;
   if (opts.verify) {
     verdict = await opts.verify();

package/src/cli/swap-heartbeat.ts

@@ -0,0 +1,407 @@
+/**
+ * swap-heartbeat — liveness for db:swap's restore leg, read from the OTHER side of the connection.
+ *
+ * The problem: `restoreIntoIncoming` spawns `psql -f <file>` and prints nothing until it exits.
+ * Both stderr streams are accumulated into strings and surfaced only on a non-zero exit, so for the
+ * whole multi-GB push a wedged restore and a healthy one produce the identical observation: silence.
+ *
+ * The probe: db:swap already holds an operator connection (`createUrlRunner`, max: 1) that sits
+ * COMPLETELY IDLE for the duration of the restore — `applyIncoming` ignores the runner it is handed
+ * and spawns psql on its own connection. So liveness costs no new connection and no new credential.
+ *
+ * ============================================================================================
+ * THE PROBE MUST NEVER TOUCH A RELATION. Learned the hard way, 2026-07-27.
+ * ============================================================================================
+ * The first version summed `pg_total_relation_size()` over the incoming schema. That function calls
+ * `relation_open(relid, AccessShareLock)` — it LOCKS every relation it measures, and holds those
+ * locks until the whole `sum()` completes. Against a restore doing DDL, two things follow:
+ *
+ *   1. The probe BLOCKS behind the restore's AccessExclusiveLock. Measured: it hung until killed by
+ *      `statement_timeout`; unbounded without one. A liveness probe that hangs is worse than none —
+ *      it reports "no backend connected" about a restore that is fine.
+ *   2. Worse, it blocks the RESTORE. PostgreSQL queues lock requests fairly, so once the probe's
+ *      pending AccessShare is in line, the restore's next AccessExclusive queues behind IT. Polling
+ *      every 10s injected a lock barrier across every table in the schema.
+ *
+ * Everything below reads ONLY statistics and catalog views — `pg_stat_all_tables`,
+ * `pg_stat_progress_copy`, `pg_stat_activity`, `pg_namespace`. All verified lock-free against a held
+ * AccessExclusiveLock (0.03s vs. blocking-until-killed). Do not reintroduce a relation-size call,
+ * `relpages` arithmetic aside — if you need bytes, `pg_stat_progress_copy.bytes_processed` is the
+ * lock-free source.
+ *
+ * Two signals, and the pair is what discriminates:
+ *   - rows landed (committed inserts + the in-flight COPY's tuples) — is data actually arriving
+ *   - the loader's wait state — what is it waiting ON
+ *
+ * Neither alone is enough. Over a high-latency link a HEALTHY COPY sits in `Client/ClientRead`
+ * constantly, between chunks — treating that as a stall cries wolf on every normal restore. The
+ * deadlock signature is `ClientRead` with FLAT progress across consecutive polls: the server is
+ * waiting for a client that has stopped sending. `Lock` is never healthy here and reports at once.
+ */
+
+import type { QueryRunner } from './authz-contract.js';
+
+/** One poll: what has landed, and what the loading backend is doing. */
+export interface HeartbeatSample {
+  /** ms since the restore phase started. */
+  elapsedMs: number;
+  /**
+   * Rows landed, from `n_tup_ins` ALONE. Null before the schema exists.
+   *
+   * Do NOT add the in-flight COPY's tuples to this. The first version did, on the assumption that
+   * a COPY's tuples only enter `n_tup_ins` at commit. They do not: the backend flushes stats while
+   * the COPY is still running, so the two overlap. Measured on a real run — it reported 767,883
+   * rows landed for an artifact whose entire SQL file contains at most 345,663. In-flight tuples
+   * are carried separately as `copyRows` and reported, never summed.
+   */
+  rows: number | null;
+  /** Tuples processed by in-flight COPYs. Separate from `rows` — see above. */
+  copyRows: number;
+  /** Bytes processed by in-flight COPYs; 0 between them. The finer-grained signal inside one COPY. */
+  copyBytes: number;
+  /** How many COPY commands are running right now. */
+  copies: number;
+  /** Tables present in the incoming schema. */
+  tables: number;
+  /**
+   * ALL relations in the schema — tables, indexes, sequences, matviews. Its growth is the progress
+   * signal for the DDL and index phases, when no row has landed yet and nothing is COPYing.
+   */
+  relations: number;
+  /** Whether the incoming schema exists yet — keeps "empty" and "absent" distinguishable. */
+  schemaExists: boolean;
+  /** The loading backend's state, or null when no psql backend is connected. */
+  state: string | null;
+  waitEventType: string | null;
+  waitEvent: string | null;
+  /** Who holds the lock we are waiting on: `pid N [app] query...`. Null unless blocked. */
+  blockedBy: string | null;
+}
+
+export type Liveness =
+  /** Rows or COPY bytes grew since the last poll. */
+  | 'progressing'
+  /** Waiting on a lock — never healthy on this path. */
+  | 'blocked'
+  /** Flat progress AND the server is waiting on the client: the deadlock signature. */
+  | 'client-stall'
+  /** Connected and working, but nothing measurable moved this poll (DDL, index build). */
+  | 'busy'
+  /** No psql backend connected — not started yet, or already gone. */
+  | 'absent';
+
+/** Consecutive stalled polls before the heartbeat escalates from info to warn. */
+export const STALL_POLLS = 3;
+
+/**
+ * Minimum quiet time since the last observed progress before a stall is called out.
+ *
+ * STALL_POLLS alone fires at the TAIL of a healthy run: a real restore ended with three flat polls
+ * and the warning printed one second before it completed successfully. Requiring a grace window
+ * since progress was last SEEN — not merely N polls in a row — keeps the alarm for genuine stalls.
+ */
+export const STALL_GRACE_MS = 60_000;
+
+/** Default ms between polls. Long enough to be quiet, short enough to localize a stall. */
+const DEFAULT_INTERVAL_MS = 10_000;
+
+/**
+ * Client-side cap on one poll. The query is provably lock-free, but a hung CONNECTION (network,
+ * server-side termination) could still park it forever, and a probe that never returns is exactly
+ * the failure this module exists to report. Racing in JS also covers hangs no `statement_timeout`
+ * would catch, because it does not depend on the server being responsive at all.
+ */
+export const PROBE_TIMEOUT_MS = 5_000;
+
+/** Single-quote a string literal for inlining into the probe SQL. */
+function quoteLiteral(v: string): string {
+  return v.replace(/'/g, "''");
+}
+
+/**
+ * The probe, as ONE round trip. Statistics and catalog views ONLY — see the module header. Every
+ * CTE here was verified to return in ~0.03s while another session held an AccessExclusiveLock on
+ * the schema's tables.
+ */
+export function heartbeatQuery(incoming: string): string {
+  const ns = quoteLiteral(incoming);
+  return `WITH ns AS (
+  SELECT oid FROM pg_namespace WHERE nspname = '${ns}'
+), tbl AS (
+  SELECT coalesce(sum(n_tup_ins), 0)::bigint AS rows_in,
+         count(*)::int AS tables
+  FROM pg_stat_all_tables WHERE schemaname = '${ns}'
+), rel AS (
+  -- EVERY relation, not just tables: indexes, sequences and matviews are relations, so this keeps
+  -- producing a progress signal through the CREATE INDEX phase after the data has landed.
+  SELECT count(*)::int AS relations
+  FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = '${ns}'
+), cp AS (
+  -- Scoped to OUR load: COPY FROM only (never a concurrent pg_dump's COPY TO), and only into
+  -- relations in the incoming schema. Unscoped, this counted every COPY in the database — on a
+  -- real run it reported 2,269,567 rows in flight for an artifact containing 341,160, because a
+  -- concurrent dump was streaming out. Worse than noise: "COPY advancing" reads as progress, so
+  -- another session's work could mask our restore being genuinely stuck.
+  SELECT count(*)::int AS copies,
+         coalesce(sum(p.bytes_processed), 0)::bigint AS copy_bytes,
+         coalesce(sum(p.tuples_processed), 0)::bigint AS copy_tuples
+  FROM pg_stat_progress_copy p
+  JOIN pg_class c     ON c.oid = p.relid
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+  WHERE p.datname = current_database()
+    AND p.command = 'COPY FROM'
+    AND n.nspname = '${ns}'
+), be AS (
+  -- pg_blocking_pids answers the question a bare wait_event cannot: WHO is holding the lock.
+  -- Without it "Lock/relation" is unactionable and invites guessing at the culprit.
+  SELECT a.state, a.wait_event_type, a.wait_event,
+         (SELECT string_agg(
+                   format('pid %s [%s] %s', b.pid,
+                          coalesce(nullif(b.application_name, ''), '?'),
+                          left(regexp_replace(coalesce(b.query, ''), '\\s+', ' ', 'g'), 80)),
+                   '; ')
+            FROM pg_stat_activity b
+           WHERE b.pid = ANY (pg_blocking_pids(a.pid))) AS blocked_by
+  FROM pg_stat_activity a
+  WHERE a.datname = current_database()
+    AND a.pid <> pg_backend_pid()
+    AND a.application_name = 'psql'
+  ORDER BY a.backend_start DESC
+  LIMIT 1
+)
+SELECT (SELECT count(*) FROM ns) > 0 AS schema_exists,
+       tbl.rows_in, tbl.tables, rel.relations,
+       cp.copies, cp.copy_bytes, cp.copy_tuples,
+       be.state, be.wait_event_type, be.wait_event, be.blocked_by
+FROM tbl CROSS JOIN rel CROSS JOIN cp LEFT JOIN be ON true`;
+}
+
+/** Map a probe row to a sample. A missing row, or a schema that does not exist, means null rows. */
+export function toSample(row: any, elapsedMs: number): HeartbeatSample {
+  const schemaExists = row?.schema_exists === true;
+  const committed = row?.rows_in == null ? null : Number(row.rows_in);
+  return {
+    elapsedMs,
+    rows: schemaExists && committed !== null ? committed : null,
+    copyRows: Number(row?.copy_tuples ?? 0),
+    copyBytes: Number(row?.copy_bytes ?? 0),
+    copies: Number(row?.copies ?? 0),
+    tables: Number(row?.tables ?? 0),
+    relations: Number(row?.relations ?? 0),
+    schemaExists,
+    state: row?.state ?? null,
+    waitEventType: row?.wait_event_type ?? null,
+    waitEvent: row?.wait_event ?? null,
+    blockedBy: row?.blocked_by ?? null,
+  };
+}
+
+/**
+ * Did anything measurable move between two samples? Any of the three independent signals counts.
+ * `rows` is monotonic but coarse; the COPY counters are live but reset to 0 when a COPY finishes,
+ * so a drop in them is never evidence of a stall — only a rise is evidence of progress.
+ */
+function movedForward(prev: HeartbeatSample, cur: HeartbeatSample): boolean {
+  if (prev.rows !== null && cur.rows !== null && cur.rows > prev.rows) return true;
+  if (cur.copyBytes > prev.copyBytes) return true;
+  if (cur.copyRows > prev.copyRows) return true;
+  // Schema growth. Without this a restore visibly creating sixteen tables every poll read as a
+  // stall and tripped the deadlock alarm, because no ROW had landed yet — the data phase had not
+  // started. Every phase of a restore must have a signal, or the quiet ones look like hangs.
+  return cur.relations > prev.relations;
+}
+
+/**
+ * The discrimination. Order is the whole point:
+ *   1. no backend            → absent (nothing else can be said)
+ *   2. Lock                  → blocked, even mid-progress, and even with no baseline
+ *   3. no predecessor        → busy (cannot claim progress OR a stall on the first poll)
+ *   4. rows/bytes grew       → progressing, EVEN in ClientRead (the healthy high-latency case)
+ *   5. schema not created    → busy (DDL phase, nothing to measure)
+ *   6. flat + ClientRead     → client-stall (the deadlock signature)
+ */
+export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample): Liveness {
+  if (cur.state === null) return 'absent';
+  if (!prev) return 'busy';
+  // PROGRESS OUTRANKS EVERY SCARY WAIT STATE, including Lock. A restore takes relation locks
+  // constantly while building 84 indexes and 37 constraints, so a 10s sample catches it mid-wait
+  // routinely. The first version returned 'blocked' on sight of a Lock and printed "nothing
+  // landing" over a run whose row count was climbing 25k every poll. Same rule as ClientRead: the
+  // wait state only means something when nothing is moving.
+  if (movedForward(prev, cur)) return 'progressing';
+  if (cur.waitEventType === 'Lock') return 'blocked';
+  if (!cur.schemaExists) return 'busy';
+  if (cur.waitEvent === 'ClientRead') return 'client-stall';
+  return 'busy';
+}
+
+/** `12s`, `1m30s`, `2h05m01s`. */
+export function humanElapsed(ms: number): string {
+  const total = Math.max(0, Math.floor(ms / 1000));
+  const h = Math.floor(total / 3600);
+  const m = Math.floor((total % 3600) / 60);
+  const s = total % 60;
+  if (h) return `${h}h${String(m).padStart(2, '0')}m${String(s).padStart(2, '0')}s`;
+  if (m) return `${m}m${String(s).padStart(2, '0')}s`;
+  return `${s}s`;
+}
+
+/** `1,234,567` — rows read better grouped. */
+function groupNum(n: number): string {
+  return n.toLocaleString('en-US');
+}
+
+/** One operator-readable line per poll. Says what moved, what it is waiting on, and how long in. */
+export function formatSample(prev: HeartbeatSample | null, cur: HeartbeatSample, liveness: Liveness): string {
+  const at = `t+${humanElapsed(cur.elapsedMs)}`;
+  const wait = cur.waitEventType ? `${cur.waitEventType}/${cur.waitEvent ?? '?'}` : (cur.state ?? 'idle');
+
+  if (liveness === 'absent') {
+    return `restore: no psql backend connected to the target (${at}) — the loader has not started yet, or has already exited.`;
+  }
+  if (!cur.schemaExists) {
+    return `restore: schema not created yet, no tables yet — DDL phase, ${wait} (${at}).`;
+  }
+
+  const rows = cur.rows === null ? 'unknown' : groupNum(cur.rows);
+  // In-flight COPY tuples are reported alongside the landed count, never folded into it.
+  const copying = cur.copies > 0
+    ? `, ${cur.copies} COPY running (${groupNum(cur.copyRows)} rows in flight)`
+    : '';
+  const scope = `${rows} row(s) landed across ${cur.tables} table(s)${copying}`;
+
+  // Every branch below states only what it actually read. The first version hardcoded "nothing
+  // landing" into the blocked branch, which then printed over a run whose row count was climbing.
+  if (liveness === 'blocked') {
+    // Naming the holder is the difference between an actionable report and a shrug.
+    const by = cur.blockedBy ? ` HELD BY ${cur.blockedBy}` : ' (holder not visible — it may belong to another role)';
+    return `restore: waiting on ${wait} with no progress this poll${by} — ${scope} (${at}).`;
+  }
+  if (liveness === 'client-stall') {
+    return `restore: nothing landed since the last poll and the server is waiting on the client (${wait}) — ${scope} (${at}).`;
+  }
+  if (liveness === 'progressing' && prev) {
+    const dRows = (cur.rows ?? 0) - (prev.rows ?? 0);
+    const secs = Math.max(1, (cur.elapsedMs - prev.elapsedMs) / 1000);
+    const moved = dRows > 0
+      ? `+${groupNum(dRows)} row(s) (${groupNum(Math.round(dRows / secs))} rows/s)`
+      : 'COPY advancing';
+    return `restore: ${scope}, ${moved} — ${wait} (${at}).`;
+  }
+  return `restore: ${scope}, nothing new this poll — ${wait} (${at}).`;
+}
+
+/**
+ * Run one probe. Never throws and never hangs: a probe that fails must not kill the restore it is
+ * watching, and a probe that parks forever is the failure it exists to report.
+ */
+export async function pollOnce(
+  runner: QueryRunner,
+  incoming: string,
+  elapsedMs: number,
+  timeoutMs: number = PROBE_TIMEOUT_MS,
+): Promise<HeartbeatSample> {
+  let timer: any;
+  const timeout = new Promise<undefined>((resolve) => {
+    timer = setTimeout(() => resolve(undefined), timeoutMs);
+    timer.unref?.();
+  });
+  try {
+    const rows = await Promise.race([
+      runner(heartbeatQuery(incoming)).catch(() => undefined),
+      timeout,
+    ]);
+    return toSample(Array.isArray(rows) ? rows[0] : undefined, elapsedMs);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export interface HeartbeatOptions {
+  /** The schema the restore is landing into. */
+  incoming: string;
+  /** Per-poll line sink. */
+  log: (msg: string) => void;
+  /** Escalation sink for a sustained stall or a lock wait. Defaults to `log`. */
+  warn?: (msg: string) => void;
+  intervalMs?: number;
+  /** Injectable clock, for tests. */
+  now?: () => number;
+  /** Per-poll cap; exposed for tests. */
+  timeoutMs?: number;
+  /**
+   * Every sample, raw, before any classification. This is how a caller acts on what the probe sees
+   * rather than only reading about it — db:swap uses it to kill a psql that the server has no
+   * backend for. Must not throw; a sink that does is ignored so it cannot kill the heartbeat.
+   */
+  onSample?: (sample: HeartbeatSample) => void;
+}
+
+/**
+ * Poll the idle operator connection until stopped. Returns the stopper, which clears the timer and
+ * awaits any in-flight poll so nothing logs after the caller has moved on.
+ *
+ * Ticks are SKIPPED while one is in flight, never queued. Chaining them (`inFlight.then(tick)`)
+ * serializes but does not drop: one slow poll lets `setInterval` stack dozens behind it, and they
+ * all drain at once the moment it resolves — a wall of identical lines that buries the real signal.
+ * Observed for real: a 7-minute poll produced ~40 stacked lines in a 5-second burst.
+ */
+export function startHeartbeat(runner: QueryRunner, opts: HeartbeatOptions): () => Promise<void> {
+  const now = opts.now ?? (() => Date.now());
+  const warn = opts.warn ?? opts.log;
+  const startedAt = now();
+  let prev: HeartbeatSample | null = null;
+  let consecutiveStalls = 0;
+  let lastProgressMs = 0;
+  let inFlight: Promise<void> = Promise.resolve();
+  let running = false;
+  let stopped = false;
+
+  const tick = async () => {
+    const cur = await pollOnce(runner, opts.incoming, now() - startedAt, opts.timeoutMs);
+    if (stopped) return;
+    // Raw sample first: a caller acting on the probe (the dead-backend watchdog) must see every
+    // sample, and must never be able to break the heartbeat by throwing.
+    try { opts.onSample?.(cur); } catch { /* a sink's failure is not the probe's problem */ }
+    const liveness = classify(prev, cur);
+    const line = formatSample(prev, cur, liveness);
+
+    if (liveness === 'blocked') {
+      warn(line);
+    } else if (liveness === 'client-stall') {
+      consecutiveStalls += 1;
+      // Two conditions, not one. A stall must SUSTAIN (one flat poll is normal between COPY
+      // chunks) AND enough quiet must have passed since progress was last seen — otherwise the
+      // warning fires on the tail of a healthy run that is about to finish.
+      const quietMs = cur.elapsedMs - lastProgressMs;
+      if (consecutiveStalls >= STALL_POLLS && quietMs >= STALL_GRACE_MS) {
+        warn(
+          `${line} STALLED for ${consecutiveStalls} consecutive polls and ${humanElapsed(quietMs)} with no progress — this is the deadlock signature (the server is waiting for data psql is not sending).`,
+        );
+      } else {
+        opts.log(line);
+      }
+    } else {
+      consecutiveStalls = 0;
+      if (liveness === 'progressing') lastProgressMs = cur.elapsedMs;
+      opts.log(line);
+    }
+    prev = cur;
+  };
+
+  const timer = setInterval(() => {
+    if (running || stopped) return; // SKIP, never queue — see the doc comment.
+    running = true;
+    inFlight = tick()
+      .catch(() => {})
+      .finally(() => { running = false; });
+  }, opts.intervalMs ?? DEFAULT_INTERVAL_MS);
+  // Never hold the process open on the heartbeat alone.
+  (timer as any).unref?.();
+
+  return async () => {
+    stopped = true;
+    clearInterval(timer);
+    await inFlight.catch(() => {});
+  };
+}