@everystack/cli 0.4.35 → 0.4.36

package/src/cli/schema-swap.ts

@@ -99,6 +99,14 @@ export interface SwapPlan {
   /** The schema the live data is renamed to before the retiring drop (outside the txn). */
   retiring: string;
   incoming: string;
+  /**
+   * EVERY retiring schema the swap produced — the base schema first, then each paired derived
+   * schema. The caller drops all of them after verify; dropping only `retiring` would strand the
+   * derived twins as permanent clutter that the next swap then collides with.
+   */
+  retiringSchemas: string[];
+  /** The paired derived schemas, in the order they were renamed. Empty for an unpaired swap. */
+  paired: string[];
 }
 
 export interface SwapOptions {
@@ -108,6 +116,27 @@ export interface SwapOptions {
   incoming?: string;
   /** Where the live schema is renamed. Default `<schema>_retiring`; pass a stamped name for uniqueness. */
   retiring?: string;
+  /**
+   * Derived schemas swapping ALONGSIDE the base schema (the paired swap). Each is renamed in the
+   * SAME transaction as the base, so a reader never sees a base schema paired with a derived layer
+   * built over the retiring one. Their incoming twins must already be built — see swap-pair.
+   */
+  paired?: string[];
+  /** Suffix for the paired schemas' incoming twins. Must match the build. Default `_incoming`. */
+  incomingSuffix?: string;
+  /** Suffix for the paired schemas' retiring names. Default `_retiring`. */
+  retiringSuffix?: string;
+  /**
+   * `GRANT USAGE ON SCHEMA` for every schema in the swap set (see renderSwapSchemaUsage), applied
+   * INSIDE the swap transaction after the renames.
+   *
+   * The incoming schemas carry no schema-level ACL — the base one is restored `--no-privileges`,
+   * the derived twins are freshly created — so a swap that re-applies only table authz commits a
+   * set of schemas no application role can enter. Found on real infrastructure: every API endpoint
+   * 500'd, and because PostgreSQL reports missing USAGE as ABSENCE the error read
+   * `relation "..." does not exist`, which points nowhere near the cause.
+   */
+  schemaUsage?: string[];
 }
 
 /**
@@ -135,15 +164,40 @@ export function renderSchemaSwap(models: ModelDescriptor[], opts: SwapOptions):
     .map((m) => compileTableContract(m));
   const authz = emitSwapAuthzSql({ tables: statsContracts, functions: [] });
 
+  // The paired derived schemas rename in the SAME transaction as the base. Order between pairs
+  // does not matter — nothing is resolved by name inside the transaction; the objects already
+  // bind their sources by OID, and a rename does not disturb an OID. What matters is that all of
+  // them commit together, so there is no instant where the new base serves under a derived layer
+  // still welded to the retiring one.
+  const paired = opts.paired ?? [];
+  const incomingSuffix = opts.incomingSuffix ?? '_incoming';
+  const retiringSuffix = opts.retiringSuffix ?? '_retiring';
+  const pairRenames: string[] = [];
+  const retiringSchemas = [retiring];
+  for (const p of paired) {
+    const pRetiring = `${p}${retiringSuffix}`;
+    const pIncoming = `${p}${incomingSuffix}`;
+    assertSafeSchema(p);
+    assertSafeSchema(pRetiring);
+    assertSafeSchema(pIncoming);
+    pairRenames.push(`ALTER SCHEMA "${p}" RENAME TO "${pRetiring}";`);
+    pairRenames.push(`ALTER SCHEMA "${pIncoming}" RENAME TO "${p}";`);
+    retiringSchemas.push(pRetiring);
+  }
+
   const statements = [
     ...fks.map((f) => f.dropSql),
     `ALTER SCHEMA "${schema}" RENAME TO "${retiring}";`,
     `ALTER SCHEMA "${incoming}" RENAME TO "${schema}";`,
+    ...pairRenames,
     ...fks.map((f) => f.addSql),
     ...authz,
+    // Schema-level USAGE last: the table grants above are dead without it, and it must ride the
+    // same transaction so there is no committed instant where the new schemas serve unreachable.
+    ...(opts.schemaUsage ?? []),
   ];
 
-  return { statements, crossSchemaFks: fks, retiring, incoming };
+  return { statements, crossSchemaFks: fks, retiring, incoming, retiringSchemas, paired: [...paired] };
 }
 
 /** The drop of the retiring schema, run AFTER the swap transaction commits and verify passes. */

package/src/cli/swap-execute.ts

@@ -71,6 +71,41 @@ export interface ExecuteSwapOptions {
    * are still refused: consent cannot cover an object nothing knows how to rebuild.
    */
   rebuildDerived?: boolean;
+  /**
+   * The PAIRED swap: derived schemas that swap alongside the base schema, with their incoming
+   * twins already built over `<schema>_incoming` (see swap-pair). This is the zero-downtime path
+   * — the derived layer is recreated in parallel and renamed in the same transaction, so it is
+   * never absent and never welded to the retiring schema.
+   *
+   * Supplying this makes the CASCADE gate treat the whole set as one unit: an object depending on
+   * a paired schema is INSIDE the swap and rides along, exactly as an object inside the base
+   * schema always has.
+   */
+  paired?: string[];
+  /**
+   * Build the incoming derived layer. Runs after the artifact lands in `<schema>_incoming` and
+   * before the swap transaction, while every live schema keeps serving. Paired swaps only.
+   */
+  buildPairedDerived?: (runner: QueryRunner) => Promise<void>;
+  /**
+   * Schema-level USAGE applied inside the swap transaction (renderSwapSchemaUsage), and the
+   * (schema, role) pairs asserted after it commits. Without these a swap can land correct data
+   * behind schemas no application role can enter — see diffSchemaUsage.
+   */
+  schemaUsage?: string[];
+  schemaUsageRoles?: Array<{ schema: string; role: string }>;
+  /**
+   * The `_incoming`-qualified identities the paired build must produce (expectedIncomingObjects).
+   * Asserted before the rename: a partial derived layer refuses while live is untouched, instead
+   * of committing a swap that reports success with objects missing.
+   */
+  expectedDerived?: string[];
+  /**
+   * Record provenance for the objects the paired build created, once they are live at their final
+   * identities. Without it the next db:reconcile sees the whole layer as drift and rebuilds it —
+   * an expensive, ACCESS EXCLUSIVE no-op that surfaces days later on an unrelated run.
+   */
+  recordProvenance?: (runner: QueryRunner) => Promise<void>;
 }
 
 /** One table's landed-vs-live count, the intrinsic post-swap assertion's unit. */
@@ -158,19 +193,36 @@ export function dropDependentSql(d: CrossSchemaDependent): string {
  * Verified against a live PostgreSQL 16 fixture: a three-deep view chain, a function returning
  * SETOF a table, a view inside the schema, and an unrelated view in another schema.
  */
-export function crossSchemaDependentsQuery(schema: string, incoming: string, retiring: string): string {
+export function crossSchemaDependentsQuery(
+  schema: string,
+  incoming: string,
+  retiring: string,
+  opts: { paired?: string[]; incomingSuffix?: string; retiringSuffix?: string } = {},
+): string {
   const q = (s: string) => s.replace(/'/g, "''");
-  const skip = `ARRAY['${q(incoming)}','${q(retiring)}']`;
-  const notSelf = `n.nspname <> '${q(schema)}' AND n.nspname <> ALL (${skip})`;
+  const arr = (xs: string[]) => `ARRAY[${xs.map((s) => `'${q(s)}'`).join(',')}]`;
+  const paired = opts.paired ?? [];
+  const inSuffix = opts.incomingSuffix ?? '_incoming';
+  const reSuffix = opts.retiringSuffix ?? '_retiring';
+
+  // A PAIRED derived schema is INSIDE the swap: its objects are rebuilt into the incoming twin
+  // and renamed in the same transaction, so they are not at risk and must not trigger a refusal.
+  // They also seed the walk — something outside the set depending on a PAIRED schema is destroyed
+  // by that schema's rename + drop just as surely as a dependent of the base schema is.
+  const swapped = [schema, ...paired];
+  const twins = paired.flatMap((p) => [`${p}${inSuffix}`, `${p}${reSuffix}`]);
+  const skip = arr([incoming, retiring, ...twins]);
+  const notSelf = `n.nspname <> ALL (${arr(swapped)}) AND n.nspname <> ALL (${skip})`;
+  const seedIn = `= ANY (${arr(swapped)})`;
   return `WITH RECURSIVE seed AS (
   SELECT c.oid AS oid, 'pg_class'::regclass AS cls
-    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = '${q(schema)}'
+    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname ${seedIn}
   UNION
   SELECT t.oid, 'pg_type'::regclass
-    FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE n.nspname = '${q(schema)}'
+    FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE n.nspname ${seedIn}
   UNION
   SELECT p.oid, 'pg_proc'::regclass
-    FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname = '${q(schema)}'
+    FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname ${seedIn}
 ), closure AS (
   SELECT oid, cls FROM seed
   UNION
@@ -199,6 +251,33 @@ SELECT DISTINCT n.nspname, p.proname,
  ORDER BY 1, 2`;
 }
 
+/**
+ * Every view / matview / function physically living in `schemas` — what a paired swap is about to
+ * replace wholesale.
+ *
+ * The paired swap renames a derived schema out and its rebuilt twin in, then drops the retiring
+ * one. The twin is built from the DECLARED descriptors, so anything live-but-undeclared in that
+ * schema has no counterpart in the twin and the drop destroys it. Pairing widens the set of
+ * schemas the gate considers "inside" the swap; this is what keeps that from becoming a licence to
+ * delete. Same reasoning as the undeclared-dependent refusal, applied one schema over.
+ */
+export function schemaObjectsQuery(schemas: string[]): string {
+  const q = (s: string) => s.replace(/'/g, "''");
+  const list = `ARRAY[${schemas.map((s) => `'${q(s)}'`).join(',')}]`;
+  return `SELECT n.nspname AS dep_schema, c.relname AS dep_name,
+       CASE c.relkind WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' ELSE c.relkind::text END AS dep_kind,
+       ''::text AS dep_args
+  FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+ WHERE n.nspname = ANY (${list}) AND c.relkind IN ('v','m')
+UNION
+SELECT n.nspname, p.proname,
+       CASE p.prokind WHEN 'p' THEN 'procedure' ELSE 'function' END,
+       pg_get_function_identity_arguments(p.oid)
+  FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
+ WHERE n.nspname = ANY (${list})
+ ORDER BY 1, 2`;
+}
+
 /** The refusal message — names what is at risk, and which of the two refusals this is. */
 export function crossSchemaDependentsRefusal(
   schema: string,
@@ -253,6 +332,72 @@ export function isFatalVerdict(verdict: SwapVerdict): boolean {
   return checks.some((c) => !c.ok && (c.severity ?? 'fatal') === 'fatal');
 }
 
+/**
+ * The reachability assertion: every role that holds a declared grant in a swapped schema must
+ * still have USAGE on it after the swap.
+ *
+ * The row-count assertion proves the DATA landed. This proves the data can be REACHED, which is a
+ * different failure and a quieter one: a swap that lands perfect data behind a schema no
+ * application role can enter reports success while every endpoint 500s. PostgreSQL surfaces a
+ * missing schema USAGE as ABSENCE rather than denial, so the error the operator sees is
+ * `relation "..." does not exist` — pointing at the table, not the grant.
+ *
+ * Pure: the impure half reads has_schema_privilege and hands the rows here.
+ */
+export function diffSchemaUsage(rows: Array<{ schema: string; role: string; ok: boolean }>): SwapCheck[] {
+  return rows
+    .filter((r) => !r.ok)
+    .map((r) => ({
+      name: `usage:${r.schema}:${r.role}`,
+      ok: false,
+      severity: 'fatal' as const,
+      detail: `role ${r.role} has no USAGE on schema ${r.schema} after the swap — it holds declared grants there, so every one of them is unreachable (PostgreSQL will report the tables as "does not exist").`,
+    }));
+}
+
+/** `has_schema_privilege` for each (role, schema) pair — catalog-only, locks nothing. */
+export function schemaUsageQuery(pairs: Array<{ schema: string; role: string }>): string {
+  const q = (s: string) => s.replace(/'/g, "''");
+  const values = pairs.map((p) => `('${q(p.schema)}','${q(p.role)}')`).join(',');
+  return `SELECT s AS schema, r AS role,
+       (EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r)
+        AND has_schema_privilege(r, s, 'USAGE')) AS ok
+  FROM (VALUES ${values}) AS t(s, r)`;
+}
+
+/**
+ * The derived-layer completeness assertion: every declared object in the swap set must actually
+ * EXIST in the incoming schemas after the build.
+ *
+ * The row-count assertion proves the base tables landed. This proves the derived layer did. They
+ * are different failures, and this one was silent: a build that throws is caught, but a build that
+ * quietly produces 38 of 79 objects sailed straight through to a successful swap. Seen in the
+ * field — a consumer's run printed the success line with half the layer missing.
+ *
+ * The same silent-success class as the original `--direct` swap reporting success without landing
+ * anything. That got a per-table count; the derived layer never got the equivalent until now.
+ *
+ * Pure: the caller reads the incoming catalog and hands both sides here.
+ */
+export function diffDerivedObjects(expected: string[], present: string[]): SwapCheck[] {
+  const have = new Set(present);
+  const missing = expected.filter((id) => !have.has(id)).sort();
+  if (missing.length === 0) return [];
+  const shown = missing.slice(0, 20).join(', ');
+  return [{
+    name: 'derived:incomplete',
+    ok: false,
+    severity: 'fatal',
+    detail: `the incoming derived layer is INCOMPLETE — ${missing.length} of ${expected.length} declared object(s) were not built: `
+      + `${shown}${missing.length > 20 ? `, and ${missing.length - 20} more` : ''}.`,
+  }];
+}
+
+/** Every view / matview / function physically present in the given schemas, as `schema.name`. */
+export function objectsPresentQuery(schemas: string[]): string {
+  return schemaObjectsQuery(schemas);
+}
+
 /** Double-quote a Postgres identifier read from the catalog (embedded quotes doubled). */
 function quoteIdent(name: string): string {
   return `"${name.replace(/"/g, '""')}"`;
@@ -282,12 +427,43 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
     };
   }
 
-  const plan = renderSchemaSwap(opts.models, { schema: opts.schema, incoming: opts.incoming, retiring: opts.retiring });
+  const paired = opts.paired ?? [];
+  const plan = renderSchemaSwap(opts.models, {
+    schema: opts.schema,
+    incoming: opts.incoming,
+    retiring: opts.retiring,
+    paired,
+    schemaUsage: opts.schemaUsage,
+  });
 
   // 1b. THE CASCADE GATE. Refuse before the snapshot — before anything moves at all — if any object
   //     outside this schema depends on it. The swap's final DROP SCHEMA ... CASCADE would destroy
   //     them silently, and a silent destroyer is the worst thing this command could be.
-  const depRows = await runner(crossSchemaDependentsQuery(opts.schema, plan.incoming, plan.retiring));
+  // 1a. PAIRED PRE-FLIGHT. A paired schema is replaced wholesale by a twin built from the declared
+  //     descriptors, so anything live in it that is NOT declared has no counterpart in the twin
+  //     and the retiring drop would destroy it. Pairing must never become a quieter way to lose an
+  //     object than the unpaired gate already refuses to be.
+  if (paired.length > 0) {
+    const declared = new Set(opts.declaredIdentities ?? []);
+    const liveRows = await runner(schemaObjectsQuery(paired));
+    const orphans: CrossSchemaDependent[] = liveRows
+      .map((r: any) => ({ schema: r.dep_schema, name: r.dep_name, kind: r.dep_kind, args: r.dep_args || undefined }))
+      .filter((d: CrossSchemaDependent) => !declared.has(dependentIdentity(d)));
+    if (orphans.length > 0) {
+      const name = (d: CrossSchemaDependent) => `${d.schema}.${d.name} (${d.kind})`;
+      return {
+        status: 'refused-dependents',
+        reason: `${orphans.length} object(s) live in the paired schema(s) ${paired.join(', ')} but are NOT declared, so the rebuilt schema would not contain them and the swap would DESTROY them: `
+          + `${orphans.slice(0, 20).map(name).join(', ')}${orphans.length > 20 ? `, and ${orphans.length - 20} more` : ''}. `
+          + `A paired swap replaces the whole derived schema with one built from db/models — anything not declared there has nothing to rebuild it. `
+          + `Nothing was changed. Declare them in db/models, or drop them yourself if they are genuinely disposable.`,
+        dependents: orphans,
+        undeclaredDependents: orphans,
+      };
+    }
+  }
+
+  const depRows = await runner(crossSchemaDependentsQuery(opts.schema, plan.incoming, plan.retiring, { paired }));
   const dependents: CrossSchemaDependent[] = depRows.map((r: any) => ({
     schema: r.dep_schema, name: r.dep_name, kind: r.dep_kind, args: r.dep_args || undefined,
   }));
@@ -341,6 +517,51 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
     };
   }
 
+  // 3b. PAIRED: build the derived layer over the incoming base tables, while every live schema
+  //     keeps serving. This is the whole point — the layer exists in full before the rename, so
+  //     there is no window in which it is absent and no refresh that could read stale rows.
+  //     A failure here leaves live untouched: nothing has been renamed yet.
+  if (opts.buildPairedDerived) {
+    const twins = paired.map((p) => `${p}_incoming`);
+    for (const t of twins) await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(t)} CASCADE`);
+    try {
+      await opts.buildPairedDerived(runner);
+    } catch (err: any) {
+      for (const t of twins) {
+        try { await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(t)} CASCADE`); } catch { /* best effort */ }
+      }
+      try { await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(plan.incoming)} CASCADE`); } catch { /* best effort */ }
+      return {
+        status: 'refused-integrity',
+        reason: `building the incoming derived layer failed, so the swap was NOT applied and live is untouched: ${String(err?.message ?? err)}. `
+          + `The declared descriptors must compose against the INCOMING base schema — if the artifact's shape no longer matches what the derived layer selects, that mismatch surfaces here rather than after the rename.`,
+      };
+    }
+    // COMPLETENESS. A build that throws is caught above; a build that quietly produces only some
+    // of its objects is not, and used to reach a successful swap. Assert BEFORE the rename, while
+    // live is still untouched — a refusal here costs nothing, where the same finding after the
+    // rename would cost a snapshot restore.
+    if (opts.expectedDerived?.length) {
+      const rows = await runner(objectsPresentQuery(twins.length ? [plan.incoming, ...twins] : [plan.incoming]));
+      const present = rows.map((r: any) => `${r.dep_schema}.${r.dep_name}`);
+      const checks = diffDerivedObjects(opts.expectedDerived, present);
+      if (checks.length > 0) {
+        for (const t of twins) {
+          try { await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(t)} CASCADE`); } catch { /* best effort */ }
+        }
+        try { await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(plan.incoming)} CASCADE`); } catch { /* best effort */ }
+        return {
+          status: 'refused-integrity',
+          reason: `${checks[0].detail} The swap was NOT applied and live is untouched. `
+            + `A partial derived layer is what a swap that reports success while half the layer is missing looks like from the inside — this refuses instead.`,
+          verdict: { ok: false, checks },
+        };
+      }
+      log(`derived layer complete: ${opts.expectedDerived.length} declared object(s) present in the incoming schemas`);
+    }
+    log(`built the incoming derived layer into ${twins.join(', ')} — live still serving the old one`);
+  }
+
   // 4. The atomic swap. A FK re-validation failure (a bad artifact) rolls the whole thing back.
   await runner('BEGIN');
   try {
@@ -372,6 +593,26 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
     };
   }
 
+  // 5a-bis. The REACHABILITY assertion. Counts prove the data landed; this proves an application
+  //     role can still get to it. A swap that commits perfect data behind a schema nobody can
+  //     enter looks like a success and reads, at the app, as every table having vanished.
+  if (opts.schemaUsageRoles?.length) {
+    const usageRows = await runner(schemaUsageQuery(opts.schemaUsageRoles));
+    const usageChecks = diffSchemaUsage(
+      usageRows.map((r: any) => ({ schema: r.schema, role: r.role, ok: r.ok === true })),
+    );
+    if (usageChecks.length > 0) {
+      if (opts.rollbackToSnapshot) await opts.rollbackToSnapshot();
+      return {
+        status: 'rolled-back-verify',
+        reason: `the swap landed but the schemas are unreachable: ${usageChecks.map((c) => c.detail).join(' ')} `
+          + `${opts.rollbackToSnapshot ? 'Rolled back to the pre-swap snapshot.' : 'NO snapshot was configured to roll back to.'}`,
+        verdict: { ok: false, checks: usageChecks },
+      };
+    }
+    log(`reachability: ${opts.schemaUsageRoles.length} (role, schema) pair(s) verified — the app can still read through the swapped schemas`);
+  }
+
   // 5b. Verify (post-commit). Fatal → roll back to the snapshot; warn → surface only.
   let verdict: SwapVerdict | undefined;
   if (opts.verify) {
@@ -386,8 +627,25 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
     }
   }
 
-  // 6. Drop the retiring schema — the swap is committed and verified.
-  await runner(dropRetiringSql(plan.retiring));
+  // 6. Drop EVERY retiring schema — the swap is committed and verified. On a paired swap that is
+  //    the base plus each derived twin; dropping only the base would strand the old derived
+  //    schemas, which the next swap then collides with on its own rename.
+  for (const r of plan.retiringSchemas) await runner(dropRetiringSql(r));
+
+  // 7. Bookkeeping. The build created the derived layer with raw DDL; the reconciler knows nothing
+  //    about it until this runs. Recorded AFTER the retiring drop so the catalog read behind it
+  //    sees only the live objects. A failure here does not undo a good swap — the data is correct
+  //    and serving; the cost is a redundant rebuild on the next reconcile, which is what this
+  //    prevents rather than something it can break.
+  if (opts.recordProvenance) {
+    try {
+      await opts.recordProvenance(runner);
+    } catch (err: any) {
+      log(`WARNING: the swap succeeded but recording provenance failed: ${String(err?.message ?? err)}. `
+        + `The derived layer is live and correct; the next db:reconcile will rebuild it needlessly. `
+        + `Run db:reconcile --rebaseline to record it without DDL.`);
+    }
+  }
 
   const warnings = (verdict?.checks ?? []).filter((c) => !c.ok && c.severity === 'warn');
   return { status: 'swapped', verdict, ...(warnings.length ? { warnings } : {}) };