@everystack/cli 0.4.33 → 0.4.35

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.33",
+  "version": "0.4.35",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/commands/db-backup.ts

@@ -25,7 +25,7 @@ import {
   parseBackupRef, keyForId, crossStageGuard, restoreTargetGuard,
   backupKey, backupId, metaKey, utcStamp,
 } from '../backup.js';
-import { pgEnvFromUrl } from './db.js';
+import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { pollTaskUntilStopped } from '../task-poll.js';
 import { step, success, fail, info, warn } from '../output.js';
 
@@ -57,7 +57,9 @@ export function resolveBackupVenue(flags: Record<string, string>): BackupVenue {
  * restoring it would rewind that history AND makes pg_restore --clean fail on DROP SCHEMA everystack).
  */
 export function pgDumpFullArgs(): string[] {
-  return ['-Fc', '--no-owner', '--no-privileges', '--no-comments', '--exclude-schema=everystack'];
+  // -d carries keepalives only; the credential still rides the PG* env (see db.ts). A full dump of
+  // a large database goes quiet for long stretches, which is exactly when an unkept connection dies.
+  return ['-d', pgKeepaliveConninfo(), '-Fc', '--no-owner', '--no-privileges', '--no-comments', '--exclude-schema=everystack'];
 }
 
 /** The .meta.json sidecar — byte-shape identical to the server's runBackup meta (backup-run.ts:156),

package/src/cli/commands/db-export.ts

@@ -26,7 +26,7 @@ import { fingerprintModels } from '../schema-fingerprint.js';
 import { resolveModelsPath } from '../models-path.js';
 import { loadModels } from './db-generate.js';
 import { loadDeclaredDerived } from '../declared-derived.js';
-import { pgEnvFromUrl } from './db.js';
+import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { utcStamp } from '../backup.js';
 import { pollTaskUntilStopped } from '../task-poll.js';
 import { step, success, fail, info } from '../output.js';
@@ -99,7 +99,8 @@ export function localArtifactMeta(opts: {
 
 /** pg_dump args for the local venue — same dump shape as the ops venue, written straight to a file. */
 export function pgDumpLocalArgs(schema: string, dumpPath: string): string[] {
-  return ['-Fc', '--no-owner', '--no-privileges', '--no-comments', `--schema=${schema}`, '-f', dumpPath];
+  // -d carries keepalives only; the credential still rides the PG* env (see db.ts).
+  return ['-d', pgKeepaliveConninfo(), '-Fc', '--no-owner', '--no-privileges', '--no-comments', `--schema=${schema}`, '-f', dumpPath];
 }
 
 /**

package/src/cli/commands/db-reconcile.ts

@@ -206,7 +206,7 @@ export async function executeReconcile(
       // 'sql'-kind objects are invisible to the catalog: provenance records the source
       // hash on both sides plus HOW TO REMOVE it (the escape hatch's whole contract).
       if (src.kind === 'sql') {
-        bookkeeping.push(renderProvenanceUpsert(identity, src.hash, src.hash, 'sql', src.drop));
+        bookkeeping.push(renderProvenanceUpsert(identity, src.hash, src.hash, 'sql', src.drop, src.bodyHash));
         continue;
       }
       const liveObj = liveById.get(identity);
@@ -214,7 +214,7 @@ export async function executeReconcile(
         throw new Error(`reconcile applied but ${identity} is not introspectable afterwards — provenance not recorded, investigate`);
       }
       const dropSql = src.kind === 'trigger' && src.table ? triggerDropSql(src.name, src.table) : undefined;
-      bookkeeping.push(renderProvenanceUpsert(identity, src.hash, liveObj.defHash, src.kind, dropSql));
+      bookkeeping.push(renderProvenanceUpsert(identity, src.hash, liveObj.defHash, src.kind, dropSql, src.bodyHash));
     }
     for (const m of rendered.migrate) bookkeeping.push(renderProvenanceMigrate(m.from, m.to));
     for (const identity of rendered.remove) bookkeeping.push(renderProvenanceDelete(identity));
@@ -292,7 +292,7 @@ export function formatBytes(bytes: number): string {
 }
 
 const VERB_GLYPH: Record<string, string> = {
-  create: '+', replace: '~', drop: '-', refresh: '↻', baseline: '◦', rebaseline: '≈', prune: '·',
+  create: '+', replace: '~', drop: '-', refresh: '↻', baseline: '◦', rebaseline: '≈', prune: '·', backfill: '◌',
 };
 
 export function buildReconcileReport(plan: ReconcilePlan): string[] {
@@ -330,9 +330,12 @@ export function buildReconcileReport(plan: ReconcilePlan): string[] {
 }
 
 /** Anything that makes `--check` fail: pending work or unresolved findings.
- *  Baseline and rebaseline are explicit adoption the operator asked for, not findings. */
+ *  Baseline and rebaseline are explicit adoption the operator asked for, not findings; a
+ *  backfill is record-only bookkeeping (body_hash arming) — neither is a CI failure. An
+ *  UNAPPLIED authz-only grant change still fails here via `regrants` below (the real signal);
+ *  an empty regrant delta means live already matches declared, so green is correct. */
 export function checkFails(plan: ReconcilePlan): boolean {
-  return plan.actions.some((a) => a.action !== 'baseline' && a.action !== 'rebaseline')
+  return plan.actions.some((a) => a.action !== 'baseline' && a.action !== 'rebaseline' && a.action !== 'backfill')
     || plan.drift.length > 0
     || plan.needsBaseline.length > 0
     || plan.blocked.length > 0

package/src/cli/commands/db-swap.ts

@@ -27,14 +27,17 @@ import { resolveModelsPath } from '../models-path.js';
 import { loadModels } from './db-generate.js';
 import { loadDeclaredDerived } from '../declared-derived.js';
 import { createUrlRunner } from '../db-source.js';
+import type { QueryRunner } from '../authz-contract.js';
 import { executeSwap, type SwapVerdict } from '../swap-execute.js';
+import { startHeartbeat, humanElapsed } from '../swap-heartbeat.js';
+import { formatBytes } from '../bundle-weight.js';
 import { rewriteStatementLine, opensCopyData, closesCopyData } from '../schema-rewrite.js';
 import { resolveOperatorUrlViaStage } from '../direct-venue.js';
 import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
 import { resolveConfig, opsFunction } from '../config.js';
 import { invokeAction, presignGet } from '../aws.js';
 import { keyForArtifactId, metaKey } from '../backup.js';
-import { pgEnvFromUrl } from './db.js';
+import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { step, success, fail, warn, info } from '../output.js';
 
 /** A COPY-aware line transform that rewrites the schema token on statement lines only. */
@@ -70,37 +73,254 @@ function schemaRewriteStream(from: string, to: string): Transform {
 }
 
 /**
- * Restore a `-Fc` artifact into `<schema>_incoming`: pg_restore -f - (archive → SQL) → the schema
- * rewrite → psql. The archive names `<schema>`; the rewrite lands it as `<incoming>`, COPY-data-safe.
+ * Consecutive polls reporting NO server-side backend before the watchdog kills psql. Three at the
+ * 10s cadence is 30s of agreement, which is well past any single-poll blip and far short of the
+ * eight minutes the unguarded version cost.
  */
-async function restoreIntoIncoming(url: string, artifactPath: string, schema: string, incoming: string): Promise<void> {
-  // Connect via PG* env, not `-d <url>`. libpq VALIDATES URI query params against its keyword
-  // list and REJECTS non-keywords like `search_path` ("invalid URI query parameter") — and the
-  // operator URL db:operator-url mints bakes search_path in (fine for postgres.js, fatal for a
-  // libpq client). pgEnvFromUrl extracts only libpq keywords (dropping search_path, routing
-  // sslmode→PGSSLMODE) and puts the password in PGPASSWORD, off the process argv. pg_restore -f -
-  // just reads the archive file to SQL on stdout (no connection).
-  const restore = spawn('pg_restore', ['-f', '-', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
-  const psql = spawn('psql', ['-v', 'ON_ERROR_STOP=1'], {
-    stdio: ['pipe', 'ignore', 'pipe'],
-    env: { ...process.env, ...pgEnvFromUrl(url) },
-  });
-  let rErr = '', pErr = '';
-  restore.stderr.on('data', (d) => { rErr += d.toString(); });
-  psql.stderr.on('data', (d) => { pErr += d.toString(); });
-  const psqlExit = new Promise<void>((res, rej) => {
-    psql.on('error', rej);
-    psql.on('close', (c) => c === 0 ? res() : rej(new Error(`psql exited ${c}: ${pErr.trim()}`)));
-  });
-  const restoreExit = new Promise<void>((res, rej) => {
-    restore.on('error', rej);
-    restore.on('close', (c) => c === 0 ? res() : rej(new Error(`pg_restore exited ${c}: ${rErr.trim()}`)));
+const DEAD_BACKEND_POLLS = 3;
+
+/**
+ * TOC entry types that a DERIVED object can appear as. Order matters: the longer types must be
+ * tested first, or `MATERIALIZED VIEW DATA` parses as `MATERIALIZED VIEW` with a mangled schema.
+ */
+const DERIVED_TOC_TYPES = [
+  'MATERIALIZED VIEW DATA',
+  'MATERIALIZED VIEW',
+  'PROCEDURE',
+  'FUNCTION',
+  'TRIGGER',
+  'VIEW',
+] as const;
+
+/**
+ * Strip the DECLARED DERIVED objects out of a pg_restore TOC listing.
+ *
+ * An artifact should carry BASE STATE — tables, data, indexes, constraints, sequences. It should
+ * not carry the derived layer, because db:reconcile owns that and rebuilds it from the descriptors.
+ * `pg_dump --schema=<s>` cannot make that distinction: it dumps everything in the schema, derived
+ * objects included.
+ *
+ * Shipping them is not merely redundant, it DEADLOCKS the swap. A derived object inside the base
+ * schema may reference the derived schema built on top of it (`stats.draft_value` returns
+ * `SETOF stats_view.draft_value_row`). The restore then cannot run until the derived layer exists,
+ * while the derived layer cannot be built until the new base tables land. Neither can go first.
+ * Measured on a real artifact: the only entries referencing the derived schema were the 16
+ * reconcile-managed functions; every other entry was state.
+ *
+ * Filtering by TOC entry rather than by SQL text is deliberate: `pg_restore -L` is the supported
+ * mechanism for restoring a subset, and the alternative is pattern-matching a six-million-line SQL
+ * file. Entries are COMMENTED OUT rather than deleted so the listing stays diffable.
+ */
+export function filterDerivedFromToc(
+  toc: string,
+  declaredIdentities: Iterable<string>,
+): { listing: string; skipped: string[] } {
+  const declared = new Set(declaredIdentities);
+  const skipped: string[] = [];
+  const listing = toc.split('\n').map((line) => {
+    // `<dumpId>; <catalogOid> <oid> <TYPE> <schema> <name(args)> <owner>`
+    const m = /^(\d+;\s+\d+\s+\d+)\s+(.+)$/.exec(line);
+    if (!m) return line; // header/comment/blank — pass through untouched
+    const rest = m[2];
+    const type = DERIVED_TOC_TYPES.find((t) => rest.startsWith(`${t} `));
+    if (!type) return line; // not a derived-capable entry (TABLE, INDEX, POLICY, ...)
+    const after = rest.slice(type.length + 1);
+    const parts = after.split(/\s+/);
+    if (parts.length < 2) return line;
+    const schema = parts[0];
+    // The owner is the last token; everything between it and the schema is the name (+args).
+    const nameWithArgs = parts.slice(1, -1).join(' ');
+    const name = nameWithArgs.replace(/\(.*$/, ''); // drop the argument list
+    if (!declared.has(`${schema}.${name}`)) return line;
+    skipped.push(`${schema}.${name} (${type.toLowerCase()})`);
+    return `;${line}`;
+  }).join('\n');
+  return { listing, skipped };
+}
+
+/** A pass-through that counts the bytes crossing it — Phase A's progress signal. */
+function countingTap(onBytes: (total: number) => void): Transform {
+  let total = 0;
+  return new Transform({
+    transform(chunk, _enc, cb) {
+      total += chunk.length;
+      onBytes(total);
+      cb(null, chunk);
+    },
   });
-  await Promise.all([
-    pipeline(restore.stdout!, schemaRewriteStream(schema, incoming), psql.stdin!),
-    restoreExit,
-    psqlExit,
-  ]);
+}
+
+/** The observation sinks the restore reports through. Injected so the phases stay testable. */
+interface RestoreIO {
+  /** The operator connection — IDLE for the whole restore, so the Phase B heartbeat reuses it. */
+  runner: QueryRunner;
+  log: (msg: string) => void;
+  warn: (msg: string) => void;
+  /**
+   * `schema.name` of every DECLARED derived object. These are stripped from the restore: the
+   * artifact carries base state, db:reconcile owns the derived layer. See filterDerivedFromToc.
+   */
+  declaredIdentities: string[];
+}
+
+/**
+ * Restore a `-Fc` artifact into `<schema>_incoming`, in TWO phases split by a local temp file:
+ *
+ *   Phase A (local, no network): pg_restore -f - (archive → SQL) → the COPY-safe schema rewrite →
+ *     a local `.sql` file. The archive names `<schema>`; the rewrite lands it as `<incoming>`.
+ *   Phase B (network): psql -f <file> streams that file to the target at psql's own pace.
+ *
+ * Why the temp file and not a live `pg_restore | rewrite | psql` pipe: decoupling the producer
+ * (local, fast) from the consumer removes all cross-process backpressure, and it makes Phase A
+ * measurable on its own. Costs one temp file (~the uncompressed dump), cleaned up in `finally`.
+ *
+ * A CORRECTION, because the original note here sent two debugging sessions down the wrong path.
+ * It claimed the three-way pipe DEADLOCKED at the DDL→data boundary and that the temp file was the
+ * cure. The temp file went in, and the restore STILL died at the same place. The pipe was never the
+ * root cause; the `write EPIPE` it produced was a downstream symptom.
+ *
+ * The real cause (proven 2026-07-27, see db.ts's keepalive note): the connection dies during the
+ * long quiet stretch of the index/constraint phase, the server terminates the backend, and the
+ * client — which never sees a FIN or RST — blocks forever on a socket its kernel still calls
+ * ESTABLISHED. The fix is TCP keepalives on psql's connection, plus the watchdog below, NOT the
+ * process topology. The temp file is kept because it is genuinely better instrumented, not because
+ * it fixes a deadlock.
+ *
+ * Both phases are INSTRUMENTED. This ran silent once — a multi-GB push with no output at all — and
+ * a run that died unattended was indistinguishable from one still working. Now: a byte counter on
+ * Phase A, a server-side heartbeat on Phase B (see swap-heartbeat.ts), and both child processes'
+ * stderr streamed as it arrives instead of being withheld until exit.
+ */
+async function restoreIntoIncoming(
+  url: string,
+  artifactPath: string,
+  schema: string,
+  incoming: string,
+  io: RestoreIO,
+): Promise<void> {
+  const tmpDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'everystack-restore-'));
+  const sqlPath = path.join(tmpDir, `${incoming}.sql`);
+  const t0 = Date.now();
+  try {
+    // Phase A0: strip the declared derived layer from the restore. The artifact is base STATE;
+    // db:reconcile owns derived. Landing them here is redundant at best and deadlocks the restore
+    // when a derived object in the base schema references the derived schema above it.
+    const tocPath = path.join(tmpDir, 'toc.list');
+    const toc = await new Promise<string>((res, rej) => {
+      const p = spawn('pg_restore', ['-l', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
+      let out = '', err = '';
+      p.stdout.on('data', (d) => { out += d.toString(); });
+      p.stderr.on('data', (d) => { err += d.toString(); });
+      p.on('error', rej);
+      p.on('close', (c) => c === 0 ? res(out) : rej(new Error(`pg_restore -l exited ${c}: ${err.trim()}`)));
+    });
+    const { listing, skipped } = filterDerivedFromToc(toc, io.declaredIdentities);
+    await fs.promises.writeFile(tocPath, listing, 'utf8');
+    io.log(
+      skipped.length
+        ? `restore: skipping ${skipped.length} declared derived object(s) — db:reconcile owns them (${skipped.slice(0, 6).join(', ')}${skipped.length > 6 ? `, +${skipped.length - 6} more` : ''}).`
+        : 'restore: the artifact carries no declared derived objects.',
+    );
+
+    // Phase A: pg_restore → rewrite → local file. Only local processes; nothing can stall here.
+    io.log(`restore phase A: pg_restore → schema rewrite (${schema} → ${incoming}) → ${sqlPath}`);
+    const restore = spawn('pg_restore', ['-L', tocPath, '-f', '-', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
+    let rErr = '';
+    restore.stderr.on('data', (d) => {
+      const s = d.toString();
+      rErr += s;
+      // Surface as it happens — a warning withheld until exit is a warning nobody can act on.
+      for (const line of s.split('\n').map((l: string) => l.trim()).filter(Boolean)) io.log(`pg_restore: ${line}`);
+    });
+    const restoreExit = new Promise<void>((res, rej) => {
+      restore.on('error', rej);
+      restore.on('close', (c) => c === 0 ? res() : rej(new Error(`pg_restore exited ${c}: ${rErr.trim()}`)));
+    });
+    let written = 0;
+    const phaseATimer = setInterval(() => {
+      io.log(`restore phase A: ${formatBytes(written)} of SQL written (t+${humanElapsed(Date.now() - t0)})...`);
+    }, 5_000);
+    (phaseATimer as any).unref?.();
+    try {
+      await Promise.all([
+        pipeline(
+          restore.stdout!,
+          countingTap((n) => { written = n; }),
+          schemaRewriteStream(schema, incoming),
+          fs.createWriteStream(sqlPath),
+        ),
+        restoreExit,
+      ]);
+    } finally {
+      clearInterval(phaseATimer);
+    }
+    const aMs = Date.now() - t0;
+    io.log(`restore phase A done: ${formatBytes(written)} of SQL in ${humanElapsed(aMs)}.`);
+
+    // Phase B: psql reads the local file and streams to the target. The credential rides PG* env,
+    // never argv (libpq also REJECTS non-keyword URI params like the `search_path` the operator URL
+    // bakes in — fine for postgres.js, fatal for a libpq URI). `-d` carries ONLY keepalives, which
+    // have no PG* env equivalent and are what keep this connection from dying in the index phase.
+    io.log(`restore phase B: psql streaming ${formatBytes(written)} to the target — heartbeat every 10s.`);
+    const psql = spawn('psql', ['-d', pgKeepaliveConninfo(), '-v', 'ON_ERROR_STOP=1', '-f', sqlPath], {
+      stdio: ['ignore', 'ignore', 'pipe'],
+      env: { ...process.env, ...pgEnvFromUrl(url) },
+    });
+    let pErr = '';
+    psql.stderr.on('data', (d) => {
+      const s = d.toString();
+      pErr += s;
+      for (const line of s.split('\n').map((l: string) => l.trim()).filter(Boolean)) io.warn(`psql: ${line}`);
+    });
+
+    // The watchdog. Keepalives should prevent the dead-backend hang, but if it happens anyway the
+    // heartbeat SEES it — pg_stat_activity, read over our own live connection, reports no psql
+    // backend while the psql process sits there forever. Detection without action is what cost an
+    // eight-minute wait: turn it into a kill and a named failure.
+    let deadBackendPolls = 0;
+    const stopHeartbeat = startHeartbeat(io.runner, {
+      incoming,
+      log: io.log,
+      warn: io.warn,
+      onSample: (sample) => {
+        if (sample.state === null) deadBackendPolls += 1;
+        else deadBackendPolls = 0;
+        if (deadBackendPolls === DEAD_BACKEND_POLLS && psql.exitCode === null) {
+          io.warn(
+            `restore: psql is still running but the server reports NO backend for it after ${deadBackendPolls} consecutive polls. `
+            + 'The connection died and psql will never notice (no FIN/RST reaches it). Killing it rather than hanging.',
+          );
+          psql.kill('SIGTERM');
+          // SIGTERM will not land if psql is parked in a blocking read on a dead socket.
+          const hardKill = setTimeout(() => { if (psql.exitCode === null) psql.kill('SIGKILL'); }, 5_000);
+          (hardKill as any).unref?.();
+        }
+      },
+    });
+    const bStart = Date.now();
+    try {
+      await new Promise<void>((res, rej) => {
+        psql.on('error', rej);
+        psql.on('close', (c) => {
+          if (c === 0) return res();
+          if (deadBackendPolls >= DEAD_BACKEND_POLLS) {
+            return rej(new Error(
+              'the restore connection died mid-load and psql hung on it (the server had no backend for it). '
+              + 'psql was killed by the watchdog; NOTHING was swapped and live is untouched. '
+              + 'This is the network path dropping a connection that goes quiet during the index/constraint phase — '
+              + 'keepalives are now set, so if you are seeing this the path is dropping the flow faster than a 30s probe interval.',
+            ));
+          }
+          rej(new Error(`psql exited ${c}: ${pErr.trim()}`));
+        });
+      });
+    } finally {
+      await stopHeartbeat();
+    }
+    io.log(`restore phase B done in ${humanElapsed(Date.now() - bStart)} (restore total ${humanElapsed(Date.now() - t0)}).`);
+  } finally {
+    await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
+  }
 }
 
 /**
@@ -229,14 +449,24 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
     process.exit(1);
   }
 
+  // --rebuild-derived carries a real outage window: the derived layer does not exist between the
+  // swap committing and db:reconcile --apply finishing. State it BEFORE the work starts — saying it
+  // only afterward tells the operator about an outage they are already in.
+  if (flags['rebuild-derived'] === 'true') {
+    warn('--rebuild-derived drops the dependent derived objects as part of the swap. They do NOT exist until db:reconcile --apply finishes — an outage window proportional to the size of the derived layer.');
+  }
+
   const modelsPath = resolveModelsPath(flags.models);
   let models: ModelDescriptor[];
   let declaredFingerprint: string;
+  let declaredDerivedObjects: Array<{ identity: string }> = [];
   try {
     step(`Loading models from ${modelsPath}...`);
     models = await loadModels(modelsPath);
     const declaredDb = await loadDeclaredDerived(flags.models);
     declaredFingerprint = fingerprintModels(models, { schemas: [schema], sequences: declaredDb?.sequences }).hash;
+    // The identities db:reconcile can regenerate — what makes a dependent safe to drop.
+    declaredDerivedObjects = declaredDb?.objects ?? [];
   } catch (err: any) { fail(err.message); process.exit(1); }
 
   // Resolve --from to a local plain -Fc dump (a local file, or an S3 export id fetched down).
@@ -263,7 +493,20 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
         models, schema,
         artifactFingerprint,
         declaredFingerprint,
-        applyIncoming: async () => { await restoreIntoIncoming(url!, artifact.dumpPath, schema, `${schema}_incoming`); },
+        // What db:reconcile can regenerate — the set a dependent must be in to be safe to drop.
+        declaredIdentities: declaredDerivedObjects.map((o) => o.identity),
+        rebuildDerived: flags['rebuild-derived'] === 'true',
+        log: (m) => info(m),
+        // The runner is handed in and USED: it is idle for the whole restore, so the Phase B
+        // heartbeat reads the loading backend's state over it (swap-heartbeat.ts).
+        applyIncoming: async (r) => {
+          await restoreIntoIncoming(url!, artifact.dumpPath, schema, `${schema}_incoming`, {
+            runner: r,
+            log: (m) => info(m),
+            warn: (m) => warn(m),
+            declaredIdentities: declaredDerivedObjects.map((o) => o.identity),
+          });
+        },
         snapshot: snapshotViaStage
           ? async () => {
               step('Snapshotting the stage before the swap (db:backup)...');
@@ -278,8 +521,19 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
     if (res.status === 'swapped') {
       success(`Swapped ${schema} — the artifact is live (no refresh ran).`);
       for (const w of res.warnings ?? []) warn(`verify warning: ${w.name}${w.detail ? ` — ${w.detail}` : ''}`);
+      // The derived layer was dropped with the swap. Say so LOUDLY: until reconcile runs, every
+      // reader of those objects is looking at a schema that no longer has them.
+      if (flags['rebuild-derived'] === 'true') {
+        warn(`the derived objects depending on ${schema} were dropped — they do NOT exist until you regenerate them.`);
+        warn(`  run now: everystack db:reconcile --apply --stage ${stage ?? '<stage>'} --direct`);
+      }
     } else {
       fail(`db:swap ${res.status}: ${res.reason}`);
+      // Name every object CASCADE would have destroyed, one per line — a comma-joined list of
+      // dozens is unreadable, and this is the list the operator has to act on.
+      if (res.status === 'refused-dependents') {
+        for (const d of res.dependents ?? []) info(`  would be destroyed: ${d.schema}.${d.name} (${d.kind})`);
+      }
       process.exit(1);
     }
   } catch (err: any) {

package/src/cli/commands/db.ts

@@ -38,6 +38,51 @@ export function pgEnvFromUrl(url: string): Record<string, string> {
   return env;
 }
 
+// --- TCP keepalives: why every long-running pg binary needs them -------------------------------
+//
+// Verified failure, 2026-07-27, on a real db:swap against a deployed stage. A restore's COPY phase
+// saturates the connection; the index/constraint phase that FOLLOWS it sends one statement and then
+// goes quiet for minutes. During that quiet period a stateful middlebox on the path (NAT gateway,
+// firewall) drops the flow's state. The server's own keepalive then finds a dead peer and
+// terminates the backend. The CLIENT, on the far side of the break, never receives a FIN or RST:
+// its socket stays ESTABLISHED and it blocks on a read that will never return.
+//
+// Measured: psql alive at 8m54s with 0.05s CPU, socket ESTABLISHED and Send-Q 0, while a SECOND
+// connection to the same database confirmed via pg_stat_activity that no psql backend existed.
+//
+// Client keepalives fix both halves. Probing every 30s of quiet keeps the middlebox's state alive
+// so the flow is never dropped, and if the peer does die the client declares it dead in ~80s
+// (30 + 5 x 10) instead of hanging forever.
+//
+// These MUST travel as libpq connection parameters — there is no PG* environment variable for
+// keepalives. Passing them via `-d` keyword/value form still leaves host/user/password/sslmode to
+// the PG* env, because libpq resolves each parameter from the conninfo first and the environment
+// second. So the credential stays off argv. Mirrors @everystack/server's backup.ts.
+
+export const PG_KEEPALIVE_IDLE_S = 30;
+export const PG_KEEPALIVE_INTERVAL_S = 10;
+export const PG_KEEPALIVE_COUNT = 5;
+
+/** Escape a value for libpq keyword/value conninfo (single-quoted, backslash-escaped). */
+function conninfoValue(v: string): string {
+  return `'${v.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+}
+
+/**
+ * A libpq keyword/value conninfo carrying ONLY keepalives (plus `dbname` when given). Everything
+ * else still resolves from the PG* env — see the note above. Safe to log: it holds no credential.
+ */
+export function pgKeepaliveConninfo(dbName?: string): string {
+  const parts = [
+    'keepalives=1',
+    `keepalives_idle=${PG_KEEPALIVE_IDLE_S}`,
+    `keepalives_interval=${PG_KEEPALIVE_INTERVAL_S}`,
+    `keepalives_count=${PG_KEEPALIVE_COUNT}`,
+  ];
+  if (dbName) parts.unshift(`dbname=${conninfoValue(dbName)}`);
+  return parts.join(' ');
+}
+
 export async function dbMigrateCommand(flags: Record<string, string>): Promise<void> {
   step('Resolving deployed config...');
   let config;
@@ -160,10 +205,14 @@ export interface ProvisionSecretPlan {
 }
 
 /**
- * Decide what db:provision writes and what it announces. Both spellings of each secret
- * are written — PascalCase is the only name an `sst.Secret` component can carry (so the
- * wiring text and the store finally agree), the raw name keeps env-style consumers and
- * `secrets export` working. Pure, so the announcement contract is pinned by tests.
+ * Decide what db:provision writes and what it announces. Writes the canonical
+ * SCREAMING_SNAKE names only (DATABASE_URL / ADMIN_DATABASE_URL) — the names the
+ * reference app declares (`new sst.Secret('DATABASE_URL')`), getDatabaseUrl() /
+ * getAdminDatabaseUrl() read, and `secrets export` emits to .env. (The old both-
+ * spellings write hedged against an sst.Secret needing a PascalCase name — it does
+ * not; underscore names are valid, so the PascalCase copy had no consumer.) The
+ * READS of `existing` below still tolerate the legacy PascalCase spelling so a stage
+ * provisioned by an older CLI is still detected. Pure — the contract is pinned by tests.
  */
 export function buildProvisionSecretPlan(args: {
   result: { loginRole: string; adminRole?: string; adminVerified?: boolean | null };
@@ -174,13 +223,12 @@ export function buildProvisionSecretPlan(args: {
   const { result, authUrl, adminUrl, existing } = args;
   const updates: Record<string, string> = {
     DATABASE_URL: authUrl,
-    DatabaseUrl: authUrl,
   };
   const notes: string[] = [];
   const warnings: string[] = [];
 
   const prevAuthRole = roleOfUrl(existing.DATABASE_URL ?? existing.DatabaseUrl);
-  notes.push(`DATABASE_URL / DatabaseUrl: ${prevAuthRole ?? 'unset'} → ${result.loginRole}`);
+  notes.push(`DATABASE_URL: ${prevAuthRole ?? 'unset'} → ${result.loginRole}`);
   if (prevAuthRole && prevAuthRole !== result.loginRole) {
     warnings.push(
       `DATABASE_URL was already set (role '${prevAuthRole}') — any function linked to it connects as '${result.loginRole}' after its next cold start. Ensure grants are in place: everystack db:reconcile && everystack db:doctor.`,
@@ -189,10 +237,9 @@ export function buildProvisionSecretPlan(args: {
 
   if (result.adminRole && adminUrl) {
     updates.ADMIN_DATABASE_URL = adminUrl;
-    updates.AdminDatabaseUrl = adminUrl;
     const prevAdminRole = roleOfUrl(existing.AdminDatabaseUrl ?? existing.ADMIN_DATABASE_URL);
     const verified = result.adminVerified === true ? 'login verified' : 'login NOT verified';
-    notes.push(`AdminDatabaseUrl / ADMIN_DATABASE_URL: ${prevAdminRole ?? 'unset'} → ${result.adminRole} (${verified})`);
+    notes.push(`ADMIN_DATABASE_URL: ${prevAdminRole ?? 'unset'} → ${result.adminRole} (${verified})`);
     if (result.adminVerified !== true) {
       warnings.push(
         `The '${result.adminRole}' login could not be verified from the ops function — confirm operator connectivity before relying on it: everystack db:doctor.`,

package/src/cli/derived-apply.ts

@@ -41,6 +41,10 @@ export const ENSURE_RECONCILER_SQL: string[] = [
   // databases upgrade in place; their old rows (relations/functions) never need these.
   `ALTER TABLE everystack.derived_provenance ADD COLUMN IF NOT EXISTS kind text`,
   `ALTER TABLE everystack.derived_provenance ADD COLUMN IF NOT EXISTS drop_sql text`,
+  // body_hash = the source hash MINUS plain grants (authz-only-change discriminator). Idempotent;
+  // legacy rows have NULL until the reconciler re-records them, and a NULL body_hash falls back to
+  // the rebuild-on-src-change behavior (no regression) — the fix only ARMS once body_hash is recorded.
+  `ALTER TABLE everystack.derived_provenance ADD COLUMN IF NOT EXISTS body_hash text`,
   `CREATE TABLE IF NOT EXISTS everystack.schema_log (
   id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
   applied_at timestamptz NOT NULL DEFAULT now(),
@@ -221,6 +225,12 @@ export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[]):
         record.push(action.identity);
         break;
       }
+      // Arm the authz-only fast path on an up-to-date object: no DDL — the upsert just adds the
+      // body_hash column value the record loop always writes now. Idempotent; converges in one run.
+      case 'backfill': {
+        record.push(action.identity);
+        break;
+      }
       case 'prune': {
         remove.push(action.identity);
         break;
@@ -235,10 +245,10 @@ export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[]):
   return { statements, record, remove, migrate: [...plan.migrations] };
 }
 
-export function renderProvenanceUpsert(identity: string, srcHash: string, defHash: string, kind?: string, dropSql?: string): string {
-  return `INSERT INTO everystack.derived_provenance (identity, src_hash, def_hash, kind, drop_sql, applied_at)
-VALUES (${escapeLiteral(identity)}, ${escapeLiteral(srcHash)}, ${escapeLiteral(defHash)}, ${nullable(kind)}, ${nullable(dropSql)}, now())
-ON CONFLICT (identity) DO UPDATE SET src_hash = EXCLUDED.src_hash, def_hash = EXCLUDED.def_hash, kind = EXCLUDED.kind, drop_sql = EXCLUDED.drop_sql, applied_at = now()`;
+export function renderProvenanceUpsert(identity: string, srcHash: string, defHash: string, kind?: string, dropSql?: string, bodyHash?: string): string {
+  return `INSERT INTO everystack.derived_provenance (identity, src_hash, def_hash, kind, drop_sql, body_hash, applied_at)
+VALUES (${escapeLiteral(identity)}, ${escapeLiteral(srcHash)}, ${escapeLiteral(defHash)}, ${nullable(kind)}, ${nullable(dropSql)}, ${nullable(bodyHash)}, now())
+ON CONFLICT (identity) DO UPDATE SET src_hash = EXCLUDED.src_hash, def_hash = EXCLUDED.def_hash, kind = EXCLUDED.kind, drop_sql = EXCLUDED.drop_sql, body_hash = EXCLUDED.body_hash, applied_at = now()`;
 }
 
 /** A table rename carried its triggers along — same object, new identity; the provenance

package/src/cli/derived-compile.ts

@@ -22,14 +22,22 @@ import type {
   FunctionDescriptor, SqlDescriptor, TriggerSpec, Ability, SetofReturn, DependsOnRef,
 } from '@everystack/model';
 import { hashSourceContent, parseQualified, DECLARED_SOURCE_FILE, type Attachment, type SourceObject } from './derived-source.js';
+import { isPlainGrantAttachment } from './derived-grants.js';
 import { findInvokerReachabilityGaps } from './derived-lint.js';
 
 /** The provenance marker for descriptor-compiled objects (SourceObject.file). */
 const DECLARED = DECLARED_SOURCE_FILE;
 
-/** `public.x` renders bare (authored style — hash parity with hand-written sources); other schemas qualify. */
+/**
+ * Always schema-qualify — the CREATE/GRANT/COMMENT target must be `schema.name` so the object lands
+ * in its DECLARED schema regardless of the apply-time search_path order. A bare `public` name landed
+ * in search_path[0] instead (derivedSearchPath puts public LAST), so a new public derived object
+ * created while a non-public derived schema existed mislanded into the wrong schema and the record
+ * loop couldn't find it. (The old bare-public rendering chased hash parity with hand-written db/sql
+ * sources; db/sql is retired — B7 — so that rationale is dead. Qualifying is the correctness rule.)
+ */
 function renderName(schema: string, name: string): string {
-  return schema === 'public' ? name : `${schema}.${name}`;
+  return `${schema}.${name}`;
 }
 
 /** The declared name of a dependable ref, for rendering (`SETOF posts`) and identity. */
@@ -243,6 +251,9 @@ function make(kind: SourceObject['kind'], rawName: string, sql: string, attachme
     identity: extra.identity ?? `${schema}.${name}`,
     sql, attachments,
     hash: hashSourceContent(sql, attachments),
+    // bodyHash excludes PLAIN grants (column-scoped grants stay in — attacl isn't drift-checked, so
+    // a column-grant change must still rebuild). Equal across an authz-only plain-grant change.
+    bodyHash: hashSourceContent(sql, attachments.filter((a) => !isPlainGrantAttachment(a))),
     file: DECLARED, seq,
     ...extra,
   });