@everystack/cli 0.4.35 → 0.4.38

package/src/cli/commands/db-swap.ts

@@ -26,6 +26,9 @@ import { fingerprintModels } from '../schema-fingerprint.js';
 import { resolveModelsPath } from '../models-path.js';
 import { loadModels } from './db-generate.js';
 import { loadDeclaredDerived } from '../declared-derived.js';
+import type { SourceObject } from '../derived-source.js';
+import { pairedDerivedSchemas, renderPairedDerivedBuild, renderSwapSchemaUsage, swapSchemaRoles, expectedIncomingObjects, renderPairedProvenance } from '../swap-pair.js';
+import { introspectDerived } from '../derived-introspect.js';
 import { createUrlRunner } from '../db-source.js';
 import type { QueryRunner } from '../authz-contract.js';
 import { executeSwap, type SwapVerdict } from '../swap-execute.js';
@@ -34,9 +37,12 @@ import { formatBytes } from '../bundle-weight.js';
 import { rewriteStatementLine, opensCopyData, closesCopyData } from '../schema-rewrite.js';
 import { resolveOperatorUrlViaStage } from '../direct-venue.js';
 import { withMutationLease, MutationLeaseError } from '../mutation-lease.js';
-import { resolveConfig, opsFunction } from '../config.js';
-import { invokeAction, presignGet } from '../aws.js';
-import { keyForArtifactId, metaKey } from '../backup.js';
+import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
+import { invokeAction, presignGet, createRdsSnapshot, describeRdsSnapshots } from '../aws.js';
+import { keyForArtifactId, metaKey, utcStamp } from '../backup.js';
+import { rdsSnapshotIdentifier } from '../rds-snapshot.js';
+import { pollTaskUntilStopped } from '../task-poll.js';
+import { decideSnapshotMode, confirmPhysicalSnapshot, interpretBackupPoll, type SnapshotModeRequest } from '../swap-snapshot.js';
 import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { step, success, fail, warn, info } from '../output.js';
 
@@ -261,11 +267,29 @@ async function restoreIntoIncoming(
     // never argv (libpq also REJECTS non-keyword URI params like the `search_path` the operator URL
     // bakes in — fine for postgres.js, fatal for a libpq URI). `-d` carries ONLY keepalives, which
     // have no PG* env equivalent and are what keep this connection from dying in the index phase.
+    //
+    // psql reads the file from STDIN (`-f -`) rather than opening it itself, purely so the restore
+    // knows its own write position. That position is the fact the heartbeat was missing: a server
+    // parked in `Client/ClientRead` is a STALL when bytes remain unsent and the successful TAIL when
+    // they do not, and those two used to print the same line. `-f -` keeps psql's `psql:<stdin>:N:`
+    // error prefixes, so the line number of a failing statement survives the change (verified
+    // against psql 16).
     io.log(`restore phase B: psql streaming ${formatBytes(written)} to the target — heartbeat every 10s.`);
-    const psql = spawn('psql', ['-d', pgKeepaliveConninfo(), '-v', 'ON_ERROR_STOP=1', '-f', sqlPath], {
-      stdio: ['ignore', 'ignore', 'pipe'],
+    const psql = spawn('psql', ['-d', pgKeepaliveConninfo(), '-v', 'ON_ERROR_STOP=1', '-f', '-'], {
+      stdio: ['pipe', 'ignore', 'pipe'],
       env: { ...process.env, ...pgEnvFromUrl(url) },
     });
+    let fedBytes = 0;
+    let feedDone = false;
+    // A psql that exits early (ON_ERROR_STOP) makes this pipeline fail with EPIPE. That is a
+    // DOWNSTREAM symptom — psql's own exit code and stderr are the authority on what went wrong, and
+    // a previous version of this code mistook the EPIPE for the cause and chased the wrong bug for
+    // two sessions. So the feed's error is swallowed here and psql's exit decides.
+    const feeding = pipeline(
+      fs.createReadStream(sqlPath),
+      countingTap((n) => { fedBytes = n; }),
+      psql.stdin!,
+    ).then(() => { feedDone = true; }).catch(() => { /* psql's exit is the authority */ });
     let pErr = '';
     psql.stderr.on('data', (d) => {
       const s = d.toString();
@@ -282,6 +306,9 @@ async function restoreIntoIncoming(
       incoming,
       log: io.log,
       warn: io.warn,
+      // The client half of the picture. Without it the heartbeat cried "deadlock signature" over the
+      // last poll of a run that had landed every row and was about to succeed.
+      clientFeed: () => ({ fedBytes, totalBytes: written, done: feedDone }),
       onSample: (sample) => {
         if (sample.state === null) deadBackendPolls += 1;
         else deadBackendPolls = 0;
@@ -316,8 +343,11 @@ async function restoreIntoIncoming(
       });
     } finally {
       await stopHeartbeat();
+      // The feed is already finished on the success path; on a failure path it is rejecting with
+      // EPIPE. Either way, await it so no stream work outlives the phase.
+      await feeding;
     }
-    io.log(`restore phase B done in ${humanElapsed(Date.now() - bStart)} (restore total ${humanElapsed(Date.now() - t0)}).`);
+    io.log(`restore phase B done in ${humanElapsed(Date.now() - bStart)} (restore total ${humanElapsed(Date.now() - t0)}); fed ${formatBytes(fedBytes)} of ${formatBytes(written)}.`);
   } finally {
     await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
   }
@@ -404,6 +434,60 @@ async function resolveSwapArtifact(
   return fetchArtifactFromS3(from, stage, fingerprintFlag);
 }
 
+/**
+ * Take (or account for) the pre-swap rollback point, and do not return until it EXISTS.
+ *
+ * Every branch here either produces a confirmed rollback point or throws — and a throw at this point
+ * means executeSwap never reaches the restore, so live is untouched. That property is the entire
+ * reason this is not a fire-and-forget dispatch any more.
+ */
+async function takePreSwapSnapshot(
+  plan: Exclude<ReturnType<typeof decideSnapshotMode>, { mode: 'refuse' }>,
+  ctx: { stage?: string; region?: string; opsFn?: string },
+): Promise<void> {
+  if (plan.mode === 'attested') {
+    info(`pre-swap rollback point: ${plan.ref} (attested via --snapshot-ref — no new snapshot taken).`);
+    return;
+  }
+
+  if (plan.mode === 'none') {
+    warn('NO pre-swap snapshot (--snapshot none). If this swap lands bad data there is no rollback point — the retiring schema is dropped once verify passes.');
+    return;
+  }
+
+  if (plan.mode === 'physical') {
+    step(`Snapshotting the instance before the swap (RDS physical snapshot of ${plan.instanceId})...`);
+    const snapshotId = rdsSnapshotIdentifier(`${ctx.stage ?? 'swap'}-swap`, utcStamp(new Date()));
+    const region = ctx.region!;
+    const { id } = await confirmPhysicalSnapshot({
+      create: (sid) => createRdsSnapshot(region, plan.instanceId, sid),
+      describe: () => describeRdsSnapshots(region, plan.instanceId),
+      log: (m) => info(m),
+      sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+      now: () => Date.now(),
+    }, { instanceId: plan.instanceId, snapshotId });
+    info(`rollback point CONFIRMED: RDS snapshot ${id} is available — restore the instance from it if this swap goes wrong.`);
+    return;
+  }
+
+  // Logical: dispatch the Task and WAIT. The dispatch returning is not the backup existing — that
+  // conflation is what let a pg_dump run concurrently with the restore it was supposed to precede.
+  step('Snapshotting the stage before the swap (db:backup — waiting for the dump to finish)...');
+  const dispatched: any = await invokeAction(ctx.region!, ctx.opsFn!, 'db:backup', {
+    stage: ctx.stage,
+    actor: process.env.USER ?? null,
+  });
+  if (dispatched?.error) throw new Error(`the pre-swap backup would not dispatch, so the swap was NOT applied: ${dispatched.error}`);
+  const { runId, taskArn, id } = dispatched as { runId: string; taskArn: string; id: string };
+  info(`backup ${id} dispatched (run ${runId}) — waiting for the task to stop before the restore starts.`);
+  const verdict = interpretBackupPoll(
+    await pollTaskUntilStopped(ctx.region!, ctx.opsFn!, { runId, taskArn }),
+    { runId, id },
+  );
+  if (!verdict.ok) throw new Error(verdict.reason);
+  info(`rollback point CONFIRMED: backup ${id} complete — restore with db:restore --from ${id} --confirm.`);
+}
+
 export async function dbSwapCommand(flags: Record<string, string>): Promise<void> {
   const schema = flags.schema;
   const from = flags.from;
@@ -413,17 +497,40 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
   if (!from) { fail('db:swap needs --from <artifact.dump | artifact-id> (the schema-scoped -Fc archive to land).'); process.exit(1); }
 
   // Resolve the venue.
-  //  - --database-url (or DATABASE_URL): a local/direct operator connection.
+  //  - --database-url: a local/direct operator connection. EXPLICIT FLAG ONLY.
   //  - --stage --direct: resolve the stage's OPERATOR connection from its ops Lambda and execute
   //    CLI-side with an unbounded clock (a multi-GB restore blows the 900s Lambda ceiling). The
   //    operator never holds a URL; the swap snapshots the stage via db:backup before it lands.
   //  - --stage alone: refuse, naming --direct — the ops-Lambda venue can't hold the restore clock.
-  let url = flags['database-url'] || process.env.DATABASE_URL;
-  let snapshotViaStage = false;
+  //
+  // `process.env.DATABASE_URL` is NOT a venue here, and used to be.
+  //
+  // It was read first, and the stage branch was guarded by `if (!url && stage)` — so an exported
+  // DATABASE_URL SILENTLY OVERRODE `--stage`. An operator asking for dev got whatever the
+  // environment named, the stage's snapshot was skipped (the branch that sets snapshotViaStage
+  // never ran, hence the "direct v1" warning), and the swap reported success against a database
+  // nobody had asked for. A consumer hit exactly this: two runs differing only by an unrelated
+  // diagnostic flag went to different databases, because one shell had the variable exported and
+  // the other did not. Their dev derived layer was untouched because dev was never the target.
+  //
+  // On a DESTRUCTIVE verb an ambient variable must never choose the target, and db:export already
+  // states the rule: --database-url is explicit-flag-only, never the env — the venue must be
+  // deliberate. This is that rule, applied where it mattered most and was missing.
+  const urlFlag = flags['database-url'];
+  if (urlFlag && stage) {
+    fail(`db:swap got BOTH --database-url and --stage ${stage} — that is two different targets and the wrong one is destructive. Pass exactly one.`);
+    process.exit(1);
+  }
+  if (!urlFlag && !stage && process.env.DATABASE_URL) {
+    fail('db:swap will not take its target from the DATABASE_URL environment variable — a destructive swap must name its target explicitly. Pass --database-url <url> (local/direct) or --stage <name> --direct.');
+    process.exit(1);
+  }
+  let url = urlFlag;
   let region: string | undefined;
   let opsFn: string | undefined;
+  let stageConfig: CliConfig | undefined;
 
-  if (!url && stage) {
+  if (stage) {
     if (!direct) {
       fail('db:swap --stage needs --direct: a schema restore can exceed the ops-Lambda 900-second clock, so the swap runs CLI-side with an unbounded clock (credential-free — the operator never holds a URL). Re-run with --stage ' + stage + ' --direct.');
       process.exit(1);
@@ -433,13 +540,12 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
       process.exit(1);
     }
     try {
-      const config = await resolveConfig(stage);
-      region = config.region;
-      opsFn = opsFunction(config);
+      stageConfig = await resolveConfig(stage);
+      region = stageConfig.region;
+      opsFn = opsFunction(stageConfig);
       step('Resolving the operator connection from the stage (--direct)...');
       const op = await resolveOperatorUrlViaStage(stage);
       url = op.url;
-      snapshotViaStage = true;
       info(`operator credential resolved (${op.source}) — swapping CLI-side, unbounded clock.`);
     } catch (err: any) { fail(err.message); process.exit(1); }
   }
@@ -449,17 +555,43 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
     process.exit(1);
   }
 
+  // THE ROLLBACK POINT. Decided here — before the models load, before the artifact is fetched, and
+  // long before anything is renamed — so a refusal costs nothing but a second.
+  //
+  // This step used to be a `db:backup` the swap did not wait for. The ops action dispatches a Task
+  // and returns, so the swap printed "snapshot on record" and began restoring while the pg_dump was
+  // still running: a consumer measured the restore blocked ~5 minutes on
+  // `Lock/relation HELD BY pid [pg_dump]`, the swap contending with its own backup. And a task that
+  // failed to start left a destructive swap running against a rollback point that did not exist.
+  //
+  // A physical RDS snapshot is now the default where the target is RDS (a control-plane call: no
+  // locks, no buffer-cache read, no client connection), the logical backup is the non-RDS fallback
+  // and is now WAITED ON, and the bare `--database-url` venue refuses rather than warning.
+  const snapshotPlan = decideSnapshotMode({
+    venue: stage ? 'stage' : 'url',
+    instanceId: flags.instance ?? stageConfig?.databaseInstanceId,
+    snapshotRef: flags['snapshot-ref'],
+    requested: flags.snapshot as SnapshotModeRequest | undefined,
+  });
+  if (snapshotPlan.mode === 'refuse') {
+    fail(snapshotPlan.reason);
+    process.exit(1);
+  }
+
   // --rebuild-derived carries a real outage window: the derived layer does not exist between the
   // swap committing and db:reconcile --apply finishing. State it BEFORE the work starts — saying it
-  // only afterward tells the operator about an outage they are already in.
+  // only afterward tells the operator about an outage they are already in. It is now the OPT-OUT:
+  // the paired swap below is the default and has no window at all.
   if (flags['rebuild-derived'] === 'true') {
     warn('--rebuild-derived drops the dependent derived objects as part of the swap. They do NOT exist until db:reconcile --apply finishes — an outage window proportional to the size of the derived layer.');
+    warn('  the paired swap (the default, without this flag) rebuilds the layer over the incoming data and renames it in the same transaction — no window. Drop the flag unless you specifically want the old behaviour.');
   }
 
   const modelsPath = resolveModelsPath(flags.models);
   let models: ModelDescriptor[];
   let declaredFingerprint: string;
-  let declaredDerivedObjects: Array<{ identity: string }> = [];
+  let declaredDerivedObjects: SourceObject[] = [];
+  let paired: string[] = [];
   try {
     step(`Loading models from ${modelsPath}...`);
     models = await loadModels(modelsPath);
@@ -467,8 +599,19 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
     declaredFingerprint = fingerprintModels(models, { schemas: [schema], sequences: declaredDb?.sequences }).hash;
     // The identities db:reconcile can regenerate — what makes a dependent safe to drop.
     declaredDerivedObjects = declaredDb?.objects ?? [];
+    // The PAIRED swap is the default whenever a declared derived schema hangs off this one: the
+    // layer is rebuilt over the incoming tables and renamed in the same transaction, so it is
+    // never absent. --rebuild-derived is the explicit opt-out (drop, swap, reconcile after), kept
+    // for the case where rebuilding twice is not worth the zero-downtime guarantee.
+    if (flags['rebuild-derived'] !== 'true' && declaredDb) {
+      paired = pairedDerivedSchemas(models, declaredDb.derived, schema);
+    }
   } catch (err: any) { fail(err.message); process.exit(1); }
 
+  if (paired.length > 0) {
+    info(`paired swap: ${paired.join(', ')} will be rebuilt over the incoming data and renamed in the SAME transaction — the derived layer is never absent.`);
+  }
+
   // Resolve --from to a local plain -Fc dump (a local file, or an S3 export id fetched down).
   let artifact: ResolvedArtifact;
   try {
@@ -496,6 +639,68 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
         // What db:reconcile can regenerate — the set a dependent must be in to be safe to drop.
         declaredIdentities: declaredDerivedObjects.map((o) => o.identity),
         rebuildDerived: flags['rebuild-derived'] === 'true',
+        paired,
+        // Schema-level USAGE, re-applied in the swap transaction and asserted after it commits.
+        // The incoming schemas arrive with no schema ACL, so without this the swap lands correct
+        // data behind schemas the app cannot enter — every endpoint 500s with "does not exist".
+        schemaUsage: renderSwapSchemaUsage(models, declaredDerivedObjects, schema, paired),
+        schemaUsageRoles: [...swapSchemaRoles(models, declaredDerivedObjects, [schema, ...paired])]
+          .flatMap(([s, roles]) => roles.map((role) => ({ schema: s, role }))),
+        // Every declared object the paired build must produce. Checked before the rename, so a
+        // partial layer refuses with live untouched rather than committing a silent shortfall.
+        expectedDerived: paired.length
+          ? expectedIncomingObjects(declaredDerivedObjects, schema, paired)
+          : undefined,
+        // Tell the reconciler what the build made. Without this the next db:reconcile — days
+        // later, for an unrelated edit — sees the whole layer as drift and rebuilds it under
+        // ACCESS EXCLUSIVE. The objects are correct; only the bookkeeping was missing.
+        recordProvenance: paired.length
+          ? async (r) => {
+              const live = await introspectDerived(r);
+              const prov = renderPairedProvenance(declaredDerivedObjects, live.objects, schema, paired);
+              for (const st of prov.statements) await r(st);
+              if (prov.recorded.length === 0) {
+                warn(`provenance recorded NOTHING — no declared object in ${[schema, ...paired].join(', ')} matched a live catalog entry. `
+                  + `The swap itself succeeded, but the next db:reconcile will treat this layer as drift. Run db:reconcile --check to see what it thinks.`);
+              } else {
+                info(`provenance recorded for ${prov.recorded.length} object(s) — a post-swap db:reconcile is NOT required.`);
+                if (prov.unmatched.length) {
+                  warn(`  ${prov.unmatched.length} declared object(s) had no live catalog entry and were NOT recorded: ${prov.unmatched.slice(0, 10).join(', ')}${prov.unmatched.length > 10 ? ', …' : ''}`);
+                }
+              }
+            }
+          : undefined,
+        // Build the incoming derived layer over <schema>_incoming, before the rename. Skipped
+        // entirely when nothing pairs — an all-public app's swap is byte-identical to before.
+        buildPairedDerived: paired.length
+          ? async (r) => {
+              step(`Building the incoming derived layer (${paired.join(', ')})...`);
+              const built = renderPairedDerivedBuild(declaredDerivedObjects, schema, paired);
+              for (const s of built.statements) await r(s);
+              // --dump-build: diagnostic only, no behaviour change. Writes what the build EMITTED
+              // and what the catalog HOLDS immediately afterwards, before the rename. Those two
+              // together separate "the build rendered the wrong SQL" from "the build was fine and
+              // the rename lost it" — a distinction that is otherwise only reachable by racing a
+              // second connection against the build window.
+              if (flags['dump-build']) {
+                const twins = paired.map((p) => `${p}_incoming`);
+                const rows = await r(
+                  `SELECT n.nspname || '.' || c.relname AS identity, pg_get_viewdef(c.oid) AS definition
+                     FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+                    WHERE n.nspname = ANY (ARRAY[${twins.map((t) => `'${t.replace(/'/g, "''")}'`).join(',')}])
+                      AND c.relkind IN ('v','m') ORDER BY 1`,
+                );
+                fs.writeFileSync(flags['dump-build'], JSON.stringify({
+                  schema, paired,
+                  searchPath: built.statements[0],
+                  statementCount: built.statements.length,
+                  statements: built.statements,
+                  liveInIncomingAfterBuild: rows,
+                }, null, 2));
+                info(`--dump-build: wrote ${built.statements.length} rendered statement(s) and ${rows.length} live definition(s) to ${flags['dump-build']}`);
+              }
+            }
+          : undefined,
         log: (m) => info(m),
         // The runner is handed in and USED: it is idle for the whole restore, so the Phase B
         // heartbeat reads the loading backend's state over it (swap-heartbeat.ts).
@@ -507,14 +712,7 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
             declaredIdentities: declaredDerivedObjects.map((o) => o.identity),
           });
         },
-        snapshot: snapshotViaStage
-          ? async () => {
-              step('Snapshotting the stage before the swap (db:backup)...');
-              const r: any = await invokeAction(region!, opsFn!, 'db:backup', { stage });
-              if (r?.error) throw new Error(`pre-swap snapshot failed, so the swap was NOT applied: ${r.error}`);
-              info(`snapshot on record: ${r?.id ?? 'backup complete'} — restore with db:restore --from ${r?.id ?? '<id>'} --confirm.`);
-            }
-          : async () => { warn('no snapshot taken (direct v1) — take one first: everystack db:backup --database-url … before a production swap.'); },
+        snapshot: () => takePreSwapSnapshot(snapshotPlan, { stage, region, opsFn }),
       }),
     );
 
@@ -526,6 +724,13 @@ export async function dbSwapCommand(flags: Record<string, string>): Promise<void
       if (flags['rebuild-derived'] === 'true') {
         warn(`the derived objects depending on ${schema} were dropped — they do NOT exist until you regenerate them.`);
         warn(`  run now: everystack db:reconcile --apply --stage ${stage ?? '<stage>'} --direct`);
+      } else if (paired.length > 0) {
+        // Say it in the OUTPUT, not just the docs. The two-step "swap then reconcile" was correct
+        // advice for the unpaired swap and is baked into deploy scripts written against it. Run
+        // now, that reconcile drops and recreates the layer the swap just built correctly, holding
+        // ACCESS EXCLUSIVE — turning a zero-downtime swap into an outage. Seen in the field.
+        info(`the derived layer (${paired.join(', ')}) is live and complete — a post-swap db:reconcile is NOT required.`);
+        info(`  if your deploy script still runs one, remove it: it would drop and rebuild these objects under ACCESS EXCLUSIVE and cause the outage this swap exists to avoid.`);
       }
     } else {
       fail(`db:swap ${res.status}: ${res.reason}`);

package/src/cli/db-source.ts

@@ -70,13 +70,10 @@ export interface UrlRunner {
 }
 
 /**
- * A QueryRunner over a direct postgres.js connection. One connection is enough —
- * introspection is a handful of sequential catalog queries.
+ * Load the postgres.js driver, or explain how to get it. Shared by every direct-connection
+ * runner below so the missing-driver instructions can never drift between them.
  */
-export async function createUrlRunner(
-  url: string,
-  load: () => Promise<any> = () => import('postgres'),
-): Promise<UrlRunner> {
+async function loadPostgresDriver(load: () => Promise<any>): Promise<any> {
   let mod: any;
   try {
     mod = await load();
@@ -85,7 +82,18 @@ export async function createUrlRunner(
       'The direct-connection path needs the "postgres" driver. It ships with @everystack/server; in a project without it: pnpm add -D postgres',
     );
   }
-  const postgres = mod.default ?? mod;
+  return mod.default ?? mod;
+}
+
+/**
+ * A QueryRunner over a direct postgres.js connection. One connection is enough —
+ * introspection is a handful of sequential catalog queries.
+ */
+export async function createUrlRunner(
+  url: string,
+  load: () => Promise<any> = () => import('postgres'),
+): Promise<UrlRunner> {
+  const postgres = await loadPostgresDriver(load);
   // max_lifetime: null — the driver's default recycles a connection after a random 30–60
   // minutes, resolving the in-flight query and THEN killing the session. Under reconcile's
   // BEGIN-across-calls transaction that is a silent session swap mid-batch (the reconcile
@@ -114,15 +122,7 @@ export async function createUrlPipelineRunner(
   url: string,
   load: () => Promise<any> = () => import('postgres'),
 ): Promise<UrlPipelineRunner> {
-  let mod: any;
-  try {
-    mod = await load();
-  } catch {
-    throw new Error(
-      'The direct-connection path needs the "postgres" driver. It ships with @everystack/server; in a project without it: pnpm add -D postgres',
-    );
-  }
-  const postgres = mod.default ?? mod;
+  const postgres = await loadPostgresDriver(load);
   const sql = postgres(url, { max: 1, onnotice: () => {}, ...sslDefaults(url) });
   return {
     query: async (query: string) => Array.from(await sql.unsafe(query)),
@@ -132,6 +132,57 @@ export async function createUrlPipelineRunner(
   };
 }
 
+export interface UrlProbeRunner {
+  /** Read-only introspection, for the contract pull/diff. */
+  runner: QueryRunner;
+  /** The self-reverting red-team probe. See `probe` below. */
+  probe: (setup: string, read: string) => Promise<any[]>;
+  /** Close the client so the process can exit cleanly. */
+  end: () => Promise<void>;
+}
+
+/** Thrown to abort the probe transaction; never escapes `probe`. */
+const PROBE_ROLLBACK = Symbol('authz_probe_rollback');
+
+/**
+ * The direct-connection twin of the server's `db:authz:probe` action
+ * (`@everystack/server` plugin.ts) — the local venue for `db:authz:test` / `db:authz:owner`.
+ *
+ * The probe SQL must WRITE to test INSERT/UPDATE/DELETE privileges, so the only thing
+ * standing between a red-team run and a mutated developer database is the rollback. It is
+ * therefore unconditional: the transaction body always throws `PROBE_ROLLBACK` after
+ * reading the outcome rows, so postgres.js aborts it on every path — success included.
+ * There is no code path that commits. `SET ROLE` is transactional (its effect disappears
+ * when the transaction aborts), so the session's role is restored by the same rollback.
+ *
+ * `max: 1` is load-bearing, not tuning: the probe's SET ROLE / savepoint state only makes
+ * sense on the single connection that ran the setup.
+ */
+export async function createUrlProbeRunner(
+  url: string,
+  load: () => Promise<any> = () => import('postgres'),
+): Promise<UrlProbeRunner> {
+  const postgres = await loadPostgresDriver(load);
+  const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => {}, ...sslDefaults(url) });
+  return {
+    runner: async (query: string) => Array.from(await sql.unsafe(query)),
+    probe: async (setup: string, read: string) => {
+      let rows: any[] = [];
+      try {
+        await sql.begin(async (tx: any) => {
+          await tx.unsafe(setup);
+          rows = Array.from(await tx.unsafe(read));
+          throw PROBE_ROLLBACK; // discard every probe write — unconditional
+        });
+      } catch (err: unknown) {
+        if (err !== PROBE_ROLLBACK) throw err;
+      }
+      return rows;
+    },
+    end: () => sql.end({ timeout: 5 }),
+  };
+}
+
 /**
  * Managed Postgres (RDS / Supabase / Neon) requires TLS — a direct connection without it
  * is rejected with pg_hba "no encryption". Default `ssl: 'require'` for remote hosts so the

package/src/cli/derived-introspect.ts

@@ -218,6 +218,36 @@ WHERE d.classid = 'pg_rewrite'::regclass
   AND rn.nspname NOT LIKE 'pg_%'
   AND NOT EXISTS (
     SELECT 1 FROM pg_depend dep WHERE dep.objid = rp.oid AND dep.deptype = 'e'
+  )
+UNION
+-- A FUNCTION depending on a view/matview's composite ROW TYPE — RETURNS SETOF <view>,
+-- or a view rowtype as an argument.
+--
+-- Both branches above are rooted at pg_rewrite, so their dependent side is always a view or
+-- matview; this edge has a function on the dependent side and is recorded against pg_type, not
+-- pg_class. Nothing found it, so the cascade could not see that dropping the view required
+-- dropping the function first — and PostgreSQL refuses the drop:
+--
+--   cannot drop materialized view post_engagement because other objects depend on it
+--   DETAIL: function analytics.engagement_for_author depends on type analytics_view.post_engagement
+--
+-- Latent until something UPSTREAM of such a view actually changes, which is why an app can carry
+-- this shape for a long time and only meet it the first time the view has to rebuild.
+SELECT DISTINCT
+  dn.nspname, dp.proname, rn.nspname, rc.relname
+FROM pg_depend d
+JOIN pg_proc dp ON dp.oid = d.objid
+JOIN pg_namespace dn ON dn.oid = dp.pronamespace
+JOIN pg_type rt ON rt.oid = d.refobjid
+JOIN pg_class rc ON rc.oid = rt.typrelid
+JOIN pg_namespace rn ON rn.oid = rc.relnamespace
+WHERE d.classid = 'pg_proc'::regclass
+  AND d.refclassid = 'pg_type'::regclass
+  AND rc.relkind IN ('v', 'm')
+  AND rn.nspname NOT IN ('pg_catalog', 'information_schema')
+  AND rn.nspname NOT LIKE 'pg_%'
+  AND NOT EXISTS (
+    SELECT 1 FROM pg_depend dep WHERE dep.objid = dp.oid AND dep.deptype = 'e'
   );
 `.trim();
 

package/src/cli/derived-plan.ts

@@ -419,9 +419,19 @@ export function planReconcile(
   };
 
   // Relation roots whose live dependents must be handled: rebuilds and drops.
+  //
+  // FUNCTIONS ARE IN THE CLOSURE, not filtered out. A function whose return type is a view's
+  // composite rowtype (`RETURNS SETOF <view>`) makes PostgreSQL refuse to drop that view while
+  // the function exists. Excluding functions here left the cascade unable to express that, so a
+  // view backing a function's return type could never be rebuilt: the plan plotted the view's
+  // drop, counted the function as up to date, and the apply died on the drop.
+  //
+  // A function pulled in this way must be DROP + CREATE, never CREATE OR REPLACE. Replace does
+  // not drop, so the view's drop still runs with the function present and fails exactly as
+  // before — the `rebuild` map is what puts an identity in BOTH the drop set and the create set.
   const relationRoots = [...rebuild.keys(), ...drop.keys()];
   for (const root of relationRoots) {
-    const closure = closureOf(root).filter((id) => liveById.get(id) && isRelation(liveById.get(id)!.kind));
+    const closure = closureOf(root).filter((id) => liveById.has(id));
     const blockers = closure.filter((id) => !srcById.has(id) && !drop.has(id));
     if (blockers.length > 0) {
       blocked.push({
@@ -434,7 +444,12 @@ export function planReconcile(
     }
     for (const dep of closure) {
       if (!rebuild.has(dep) && !drop.has(dep) && srcById.has(dep)) {
+        const kind = liveById.get(dep)!.kind;
         rebuild.set(dep, `dependency rebuild (depends on ${root})`);
+        // Promoting a function out of the replace lane: a plain replace here would be a no-op
+        // against the problem, and leaving it in both lanes would emit a replace AND a
+        // drop+create for the same object.
+        if (!isRelation(kind)) fnReplace.delete(dep);
       }
     }
   }

package/src/cli/index.ts

@@ -373,7 +373,7 @@ Usage:
   everystack db:restore        --from <id> [--stage <name>] --confirm  Restore a backup INTO the stage's DB (DESTRUCTIVE)
   everystack db:backup:download <id> [--stage <name>]  Presigned URL to download a backup's dump (valid 1h)
   everystack db:export         --schema <name> [--stage <name> | --database-url <url> [--out <file.dump>]] [--models <barrel>]  Schema-scoped pg_dump artifact, stamped with the DECLARED schema fingerprint (the canonical-sync export; db:swap gates on that stamp). --stage dumps the stage's private DB via the ops Lambda → S3; --database-url (explicit flag, never the env) dumps a reachable DB to a local .dump + .meta.json — the build-locally → publish → swap on-ramp
-  everystack db:swap           --schema <name> --database-url <url> --from <artifact.dump> [--fingerprint <hash>]  Land a schema artifact atomically: fingerprint gate → restore into <schema>_incoming (COPY-safe rewrite) → one txn (drop+rename+recreate app→schema FKs, re-apply authz) → verify → drop retiring. Refresh-free; app.* untouched. DESTRUCTIVE (--stage/--direct ops venue rides stage-write-lanes)
+  everystack db:swap           --schema <name> --from <artifact.dump | artifact-id> [--stage <name> --direct | --database-url <url>] --confirm [--fingerprint <hash>] [--snapshot physical|logical|none] [--snapshot-ref <id>] [--rebuild-derived] [--dump-build <file.json>]  Land a schema artifact atomically: fingerprint gate → pre-flight refusals → CONFIRMED snapshot → restore into <schema>_incoming (COPY-safe rewrite) → build the paired derived layer → one txn (drop+rename+recreate app→schema FKs, re-apply authz + schema USAGE) → assertions → drop retiring. Refresh-free; app.* untouched; the derived layer is never absent. DESTRUCTIVE. The venue is EXPLICIT — DATABASE_URL in the env is refused, and --stage requires --direct (a multi-GB restore exceeds the ops-Lambda 900s clock). The rollback point defaults to a PHYSICAL RDS snapshot on a stage that exposes databaseInstanceId (no locks, no pg_dump contending with the restore) and a WAITED logical db:backup otherwise; a bare --database-url refuses without --snapshot-ref <id> or --snapshot none. docs/schema-swap.md
   everystack db:generate       [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--schema-out db/schema.generated.ts] [--allow-drops] [--apply] [--dry-run]  Diff models vs the live DB → next migration file, or with --apply execute it directly (one transaction, schema_log recorded, verified by re-diff — no drizzle folder needed; direct connection only; DROPs held back unless --allow-drops). --dry-run prints the edge and writes NOTHING (no migration, no journal entry, no schema refresh) — the preview verb; db:diff computes a models-vs-models edge with no database at all. The resolved --schema-out is recorded in the migration journal: later flag-less runs reuse it (flag > recorded > default), a differing flag updates the record and says so
   everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--derived-out <file.ts>] [--abilities public-read]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. --derived-out <file.ts> extracts the derived layer (descriptors + sequences) as its own self-contained module — alone it leaves the models untouched (the hand-maintained-barrel splice); with --out the models render omits the now-external derived layer. Every model scaffolds its authz decision as comments (db:check fails until authored); --abilities public-read stamps the common stanza (public read, admin write) uncommented — explicit generated code, never a runtime default. --matviews-as-tables renders every matview as defineMaterializedTable with INTROSPECTED fields (the canonical-sync flip: a pipeline-owned table everystack migrates but never refreshes) — names land in an exported materializedTables array to spread into your models; fields come back nullable/unkeyed (matviews carry no PK/NOT NULL) — tighten on review; add --suggest-keys to probe the LIVE rows for functionally-unique columns (one scan per matview) and surface each as a commented .primaryKey() suggestion. docs/derived-objects.md#flipping-a-matview-to-a-materialized-table---matviews-as-tables
                                Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
@@ -394,10 +394,11 @@ Usage:
   everystack db:template:refresh [--database-url <url>] [--models <barrel>] [--seed "<cmd>"] [--no-seed]  (Re)build the dev template <base>_tpl FROM THE DECLARED STATE (both layers, fingerprint MATCH bar) + seed-as-code (the app's db:seed script, run with DATABASE_URL pointed at the template; --seed overrides). All or nothing: a failed build/seed leaves NO template. Never a data copy
   everystack db:branch         [--list | --drop --confirm | --prune --confirm] [--database-url <url>]  Mint (or find) the current git branch's database from the template (CREATE DATABASE … TEMPLATE — schema, authz, derived layer, and seed rows inherited), print its DATABASE_URL, then db:sync evolves it with the checkout. --list maps every branch DB to its branch; --prune drops the ones whose branch is gone (never guesses: unknown mappings are kept)
   everystack db:fork           --from-stage <src> --stage <target> --confirm [--backup <id>]  Fork one DEPLOYED stage's database into another: back up the source (or reuse --backup <id>), presign the dump (the operator's credentials ARE the cross-stage authorization; expires in 1h), restore into the target via its ops Lambda. Production is never a target (that's db:restore); forking FROM production warns about PII; the branch's schema edge then lands via db:plan → db:apply (descent composes). Teardown: sst remove --stage <target>
-  everystack db:authz:pull     [--stage <name>] [--dir authz]  Introspect live authz (rls/grants/policies/secdef) → reviewable contract files
-  everystack db:authz:diff     [--stage <name>] [--dir authz]  Validate the live DB against the committed contract (non-zero exit on drift)
-  everystack db:authz:test     [--stage <name>] [--dir authz]  Red-team enforcement: SET ROLE + attempt per role/table/command (non-zero exit on a hole)
-  everystack db:authz:owner    [--stage <name>] [--models db/models/index.ts]  Red-team owner isolation: two JWT identities per owner-scoped table — catches IDOR (non-zero exit on a leak)
+  everystack db:authz:pull     [--stage <name> | --database-url <url>] [--dir authz]  Introspect live authz (rls/grants/policies/secdef) → reviewable contract files
+  everystack db:authz:diff     [--stage <name> | --database-url <url>] [--dir authz]  Validate the live DB against the committed contract (non-zero exit on drift)
+  everystack db:authz:test     [--stage <name> | --database-url <url>] [--dir authz]  Red-team enforcement: SET ROLE + attempt per role/table/command (non-zero exit on a hole)
+  everystack db:authz:owner    [--stage <name> | --database-url <url>] [--models db/models/index.ts]  Red-team owner isolation: two JWT identities per owner-scoped table — catches IDOR (non-zero exit on a leak)
+                               All four accept a DIRECT venue (--database-url, or an inherited ADMIN_DATABASE_URL/DATABASE_URL) so a brownfield authz migration can be rehearsed against a local database instead of a deployed stage. Same SQL, same evaluation, same verdict — every run names the venue it judged. The red-team probes WRITE to test INSERT/UPDATE/DELETE privileges and are always rolled back.
   everystack db:authz:report   [--dir authz]  Render the committed contract as a human-readable authorization review (no DB)
   everystack console           --stage <name> [--sandbox]  Interactive REPL on deployed Lambda
   everystack status            [--stage <name>] [--hours <n>]  Platform health: CDN, Lambda, rollup summary

package/src/cli/migration-compile.ts

@@ -26,6 +26,7 @@ import { compileTableContract } from './authz-compile.js';
 import { emitReconcileSql } from './authz-reconcile.js';
 import { emitSchemaSql, nextvalSequence, type SchemaChange } from './schema-diff.js';
 import { compileDerived } from './derived-compile.js';
+import { renderEnsureObjectSchemas } from './derived-apply.js';
 
 /** The empty (not-yet-created) authz state for a table — the greenfield baseline. */
 function emptyTable(table: string): TableContract {
@@ -155,7 +156,17 @@ export function compileModuleMigration(modules: Module[], opts: CompileTableOpti
   //    objects — compiled in topological order, after every table. Greenfield = one
   //    complete script: state + compute; from then on the layer deploys via db:reconcile.
   const models = modules.flatMap((m) => m.models);
-  for (const obj of compileDerived(models, modules.flatMap((m) => m.derived))) {
+  const derivedObjects = compileDerived(models, modules.flatMap((m) => m.derived));
+
+  // 4a. A schema that ONLY derived objects live in has no model to create it, so phase 2
+  //     never does — and the first CREATE VIEW in it fails with `schema … does not exist`.
+  //     db:reconcile has always handled this; the greenfield module migration did not, so a
+  //     derived-only schema was buildable by reconcile and not by a from-scratch deploy.
+  //     Same renderer as the reconcile path, deliberately: one implementation, one behaviour.
+  //     Idempotent — a schema phase 2 already created is a no-op here.
+  sql.push(...renderEnsureObjectSchemas(derivedObjects).map((s) => `${s};`));
+
+  for (const obj of derivedObjects) {
     sql.push(`${obj.sql};`);
     for (const a of obj.attachments) sql.push(`${a.sql};`);
   }