@everystack/cli 0.4.84 → 0.4.86

package/src/cli/state-apply.ts

@@ -26,7 +26,12 @@ import { FUNCTIONS_SQL, contractFunctionRow } from './security-catalog';
 import { generateMigrationSql, HELD_DROP_PREFIX } from './migration-generate';
 import type { SequenceDescriptor } from '@everystack/model';
 import { fingerprintLive, FINGERPRINT_VERSION } from './schema-fingerprint';
-import { ENSURE_RECONCILER_SQL, renderSchemaLogInsert, renderSchemaLogFingerprintUpdate } from './derived-apply';
+import {
+  ENSURE_RECONCILER_SQL,
+  renderSchemaLogInsert,
+  renderSchemaLogFingerprintUpdate,
+  renderSchemaLogCompletionUpdate,
+} from './derived-apply';
 import { isPureComment, statementBody, statementHead } from './sql-statement';
 
 const NOTICE_PREFIX = '-- NOTICE';
@@ -52,6 +57,32 @@ export function classifyGeneratedStatements(statements: string[]): ClassifiedSta
   return { executable, heldDrops, notices };
 }
 
+/**
+ * Split an executable stream into the EDGE and the CHECKS: `ALTER TABLE … VALIDATE CONSTRAINT`
+ * is held out of the batch and run afterwards, one statement at a time.
+ *
+ * This is what makes the two-step foreign-key add worth splitting at all. `ADD CONSTRAINT …
+ * NOT VALID` holds an ACCESS EXCLUSIVE lock until its transaction COMMITS, so a VALIDATE in the
+ * same transaction runs under that lock and blocks every writer for the whole scan — precisely
+ * the outage the split exists to avoid. Run after the commit, the VALIDATE takes its own, much
+ * weaker SHARE UPDATE EXCLUSIVE lock, which readers and writers pass through.
+ *
+ * The second consequence is durability, and it is the point of doing it here rather than in the
+ * emitter: a VALIDATE that fails no longer rolls back the edge. The key stays installed and
+ * NOT VALID — enforced on every new write, unproven against the old rows — which is a state the
+ * next `db:plan` sees and finishes with a VALIDATE alone. An edge that rolled back instead would
+ * have thrown away a correct, committed schema change because of data it does not own.
+ */
+export function splitValidations(executable: readonly string[]): { batch: string[]; validations: string[] } {
+  const batch: string[] = [];
+  const validations: string[] = [];
+  for (const statement of executable) {
+    if (/^ALTER\s+TABLE\s+[\s\S]+?\sVALIDATE\s+CONSTRAINT\b/i.test(statementBody(statement))) validations.push(statement);
+    else batch.push(statement);
+  }
+  return { batch, validations };
+}
+
 /**
  * The destructive subset of an executable statement stream — destructive
  * means DATA LOSS (brick 9's classification, design decision 11):
@@ -105,12 +136,14 @@ const ADDITIVE_MATCHERS: RegExp[] = [
   /^GRANT\b/,
   /^COMMENT\s+ON\b/,
   // ADD COLUMN alone. Its siblings `ADD CONSTRAINT` (FOREIGN KEY / CHECK / UNIQUE) and
-  // `ADD PRIMARY KEY` are deliberately NOT here: each one VALIDATES every row already in the
-  // table and then RESTRICTS every write after it. An FK add fails on orphan rows, a CHECK add
-  // fails on any violating row, a UNIQUE add fails on existing duplicates, and ADD PRIMARY KEY
-  // fails on duplicates or NULLs *and* silently makes every included column NOT NULL. Calling
-  // those additive tells the reviewer the plan is safe when it can abort the apply outright —
-  // the same false zero described above. They read as `unclassified`, like `SET NOT NULL`.
+  // `ADD PRIMARY KEY` are deliberately NOT here: each one RESTRICTS every write after it, and
+  // all but the NOT VALID form also VALIDATE every row already in the table. A CHECK add fails
+  // on any violating row, a UNIQUE add fails on existing duplicates, and ADD PRIMARY KEY fails
+  // on duplicates or NULLs *and* silently makes every included column NOT NULL. A foreign key
+  // is emitted `NOT VALID` and so cannot fail on the old rows — but its `VALIDATE CONSTRAINT`
+  // can, and it still refuses every future write that breaks it. Calling any of these additive
+  // tells the reviewer the plan is safe when it can abort the apply outright — the same false
+  // zero described above. They read as `unclassified`, like `SET NOT NULL`.
   /^ALTER\s+TABLE\s+[\s\S]+?\sADD\s+COLUMN\b/,
   // Enabling/forcing RLS only ever RESTRICTS; it cannot widen access or lose a row.
   /^ALTER\s+TABLE\s+[\s\S]+?\s(ENABLE|FORCE)\s+ROW\s+LEVEL\s+SECURITY\b/,
@@ -344,6 +377,47 @@ export interface StateApplyOptions {
    * row is still a witness.
    */
   descentNote?: () => string | undefined;
+  /**
+   * Read the live base fingerprint, for the row written the instant the batch commits.
+   *
+   * The `state apply` row is only a descent WITNESS if it carries a `to_fingerprint`
+   * (SCHEMA_LOG_WITNESS_SQL requires one), and it must be the hash of the database AS IT IS at
+   * that moment — with the new key installed and NOT VALID. Stamping the plan's target state
+   * there would be a prediction, not a record: if the process then dies mid-VALIDATE, the next
+   * apply recomputes the live hash, does not match the stamped one, and the row it was given to
+   * protect itself with is inert. The plan's own endpoint is stamped by the caller AFTER the
+   * last check passes.
+   *
+   * Optional: without it the row still goes in before the checks, and the caller fills the
+   * fingerprint in when it re-introspects (db:sync's lane). Called ONLY when there are checks to
+   * run — with none there is no window between the commit and the record to protect.
+   */
+  readFingerprint?: () => Promise<string>;
+  /**
+   * `VALIDATE CONSTRAINT`s the caller REMOVED from `statements` before calling, because the row
+   * check it ran first already counted the offending rows (see fk-preflight's `withheldValidations`).
+   *
+   * They are pending validations that were never attempted, and they are recorded and reported
+   * exactly like ones that ran and failed — the key is installed, enforced forward and unproven
+   * either way, so giving the operator two vocabularies for one situation would only make them
+   * work out that it is the same situation. Running them instead would take a lock and a full
+   * scan to be told what is already known.
+   */
+  skippedValidations?: readonly PendingValidation[];
+}
+
+/** A `VALIDATE CONSTRAINT` that did not pass — the key is installed but still unproven. */
+export interface PendingValidation {
+  /** The exact statement to re-run once the data satisfies the key. */
+  sql: string;
+  /** PostgreSQL's message, trimmed for the record — or why the check was never attempted. */
+  error: string;
+  /**
+   * The check was WITHHELD, not run: the row preflight had already counted the offending rows.
+   * Set only on that path, so `validationPendingReason` can name the data fix without having to
+   * pattern-match an error string it wrote itself.
+   */
+  skipped?: boolean;
 }
 
 export interface StateApplyResult {
@@ -354,32 +428,117 @@ export interface StateApplyResult {
   batchSql: string;
   /** schema_log row id for this apply, when one was written. */
   logId?: number;
+  /**
+   * Constraints left NOT VALID because their post-commit `VALIDATE` failed. The edge itself
+   * COMMITTED — this is not a rollback and must never be reported as one. Empty on a clean apply.
+   */
+  pendingValidations: PendingValidation[];
 }
 
 /**
  * Execute the executable subset of a generated diff as one batch and record
  * it in schema_log. A failed batch rolls back (one implicit transaction),
  * logs the failure best-effort, and rethrows.
+ *
+ * ORDER OF WRITES, and it is the point: batch → RECORD → checks → amend the record. The
+ * `VALIDATE CONSTRAINT` pass is the longest thing an apply does (a full scan per key) and the
+ * one a wall clock or a dropped connection actually kills. Recording after it meant that death
+ * left the DDL committed with nothing on the record — and a live state no commit on any ref
+ * declares reads to the NEXT apply as proven drift, which is the loudest possible wrong answer.
+ * So the row is written as soon as the batch has committed, describing exactly what is true then
+ * (the key installed, none of the checks run yet), and amended when the checks are over.
+ */
+/** The subset of StateApplyOptions the schema_log memoir writes need — nothing execution-only. */
+type SchemaLogFields = Pick<StateApplyOptions, 'actor' | 'gitRef' | 'planRef' | 'fromFingerprint' | 'fpVersion' | 'dirty'>;
+
+/**
+ * One row per unproven key, carrying the exact statement to re-run. Best-effort, like every
+ * other memoir write: failing to record why a key is pending must not mask that it is. Shared by
+ * both the normal path (checks run here) and the all-withheld early return (no checks run here,
+ * but the withheld ones are just as real a pending state and owe the same rows).
  */
+async function recordPendingValidations(
+  runner: QueryRunner,
+  pendingValidations: readonly PendingValidation[],
+  options: SchemaLogFields,
+): Promise<void> {
+  for (const pending of pendingValidations) {
+    try {
+      await runner(renderSchemaLogInsert({
+        kind: 'state validate', sql: pending.sql,
+        outcome: `pending: ${pending.error}`,
+        actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
+        fromFingerprint: options.fromFingerprint,
+        fpVersion: options.fpVersion ?? FINGERPRINT_VERSION, dirty: options.dirty ?? false,
+      }));
+    } catch { /* the memoir is best-effort; the row already says what committed */ }
+  }
+}
+
 export async function applyGeneratedStatements(
   runner: QueryRunner,
   statements: string[],
   options: StateApplyOptions = {},
 ): Promise<StateApplyResult> {
   const classified = classifyGeneratedStatements(statements);
+  // Checks the caller withheld before calling. They are outstanding from the first line of this
+  // function: the outcome written before the batch already counts them, and they come back in the
+  // result whether or not any check actually ran here.
+  const skipped: PendingValidation[] = [...(options.skippedValidations ?? [])];
   if (classified.executable.length === 0) {
-    return { applied: false, classified, batchSql: '' };
+    // No DDL to run — but a re-plan against keys a PREVIOUS apply already withheld still owes
+    // the audit trail a row: the promise (docs/cli.md) is every apply lands in schema_log, and
+    // an apply whose only content is "these keys are still unproven" is a real apply, not a
+    // no-op. Without this, re-running db:apply/db:plan against nothing but a withheld VALIDATE
+    // produces a `validation-pending` result the operator sees and a database that remembers
+    // nothing about it.
+    if (skipped.length > 0) {
+      const now = options.now ?? Date.now;
+      const started = now();
+      let logId: number | undefined;
+      // Same annotation the normal path folds in (a honoured lineage waiver, a proven drift
+      // origin) — an all-withheld apply is still the row a LATER apply's fast-forward proof
+      // reads, so it must carry the same note or that proof silently loses the annotation.
+      const note = options.descentNote?.();
+      const outcome = ['applied', note, `no schema change; ${skipped.length} constraint(s) still PENDING validation`]
+        .filter(Boolean).join(' — ');
+      try {
+        await runner(ENSURE_RECONCILER_SQL.join(';\n'));
+        const logRows = await runner(renderSchemaLogInsert({
+          kind: 'state apply', sql: '',
+          outcome,
+          actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
+          fromFingerprint: options.fromFingerprint,
+          fpVersion: options.fpVersion ?? FINGERPRINT_VERSION, dirty: options.dirty ?? false,
+          durationMs: now() - started,
+        }));
+        logId = logRows?.[0]?.id !== undefined ? Number(logRows[0].id) : undefined;
+      } catch { /* the memoir is best-effort — the caller's own result still names the debt */ }
+      await recordPendingValidations(runner, skipped, options);
+      return { applied: false, classified, batchSql: '', pendingValidations: skipped, ...(logId !== undefined ? { logId } : {}) };
+    }
+    return { applied: false, classified, batchSql: '', pendingValidations: skipped };
   }
 
   const now = options.now ?? Date.now;
   await runner(ENSURE_RECONCILER_SQL.join(';\n'));
 
+  // The row checks run AFTER the edge commits — see splitValidations for why that is the whole
+  // point of the two-step add, and why a failed check is a pending state rather than a rollback.
+  const { batch, validations } = splitValidations(classified.executable);
+
   // Semicolon on its own line: a statement whose last line is a comment
   // cannot swallow the separator.
-  const batchSql = classified.executable.join('\n;\n');
+  //
+  // `batchSql` is the batch that RAN IN THIS TRANSACTION — the checks are not in it. The record
+  // written below is a record of what committed, and the checks had not been attempted at that
+  // point; they join the row on the amendment, where their outcome is known.
+  const batchSql = batch.join('\n;\n');
+  const validationSql = validations.join('\n;\n');
+  const fullSql = [batchSql, validationSql].filter((s) => s.length > 0).join('\n;\n');
   const started = now();
   try {
-    await runner(batchSql);
+    if (batch.length > 0) await runner(batchSql);
   } catch (err: any) {
     try {
       await runner(renderSchemaLogInsert({
@@ -395,16 +554,67 @@ export async function applyGeneratedStatements(
   }
 
   const note = options.descentNote?.();
+  // The edge APPLIED — that word is load-bearing (the descent ledger matches `applied%`, and the
+  // schema changes really are committed). What is still unproven is named in the same line, so
+  // nobody reads a half-proven key as a finished one.
+  const outcomeWith = (pending: number): string =>
+    ['applied', note, pending > 0 ? `${pending} constraint(s) PENDING validation` : undefined]
+      .filter(Boolean)
+      .join(' — ');
+
+  // The hash of what just committed, read now rather than predicted. Only when checks follow:
+  // with none, the record goes in the same breath as the commit (as it always has) and the
+  // caller's own post-apply introspection stamps it — reading it twice would buy nothing and
+  // cost a full introspection on every apply. Best-effort: a fingerprint that cannot be read
+  // must not cost us the record.
+  let landedAfterBatch: string | undefined;
+  if (validations.length > 0 && options.readFingerprint) {
+    try {
+      landedAfterBatch = await options.readFingerprint();
+    } catch { /* the record matters more than the stamp */ }
+  }
+
+  // THE WITNESS, written before the long step. Every check is still outstanding here, and the
+  // outcome says exactly that — this row is true at the instant it is written, not a prediction
+  // of how the checks will go.
   const logRows = await runner(renderSchemaLogInsert({
-    kind: 'state apply', sql: batchSql, outcome: note ? `applied — ${note}` : 'applied',
+    kind: 'state apply', sql: batchSql, outcome: outcomeWith(validations.length + skipped.length),
     actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
     fromFingerprint: options.fromFingerprint,
+    ...(landedAfterBatch ? { toFingerprint: landedAfterBatch } : {}),
     fpVersion: options.fpVersion ?? FINGERPRINT_VERSION, dirty: options.dirty ?? false,
     durationMs: now() - started,
   }));
   const logId = logRows?.[0]?.id !== undefined ? Number(logRows[0].id) : undefined;
 
-  return { applied: true, classified, batchSql, ...(logId !== undefined ? { logId } : {}) };
+  // Every check is attempted, even after one fails: the operator is owed the COMPLETE list of
+  // keys still unproven, not the first one. Each runs on its own, so one failure cannot take a
+  // sibling's successful validation down with it.
+  const pendingValidations: PendingValidation[] = [...skipped];
+  for (const statement of validations) {
+    try {
+      await runner(statement);
+    } catch (err: any) {
+      pendingValidations.push({ sql: statement, error: String(err?.message ?? err).slice(0, 500) });
+    }
+  }
+
+  // The amendment: the SAME row, finished. A second `state apply` row would put two applies in
+  // the ledger for one edge — and the descent witness reads that ledger. So the checks and their
+  // result land on the row that already describes this apply, never beside it.
+  if (validations.length > 0 && logId !== undefined) {
+    try {
+      await runner(renderSchemaLogCompletionUpdate(logId, {
+        outcome: outcomeWith(pendingValidations.length),
+        sql: fullSql,
+        durationMs: now() - started,
+      }));
+    } catch { /* the memoir is best-effort; the row already says what committed */ }
+  }
+
+  await recordPendingValidations(runner, pendingValidations, options);
+
+  return { applied: true, classified, batchSql, pendingValidations, ...(logId !== undefined ? { logId } : {}) };
 }
 
 // ---------------------------------------------------------------------------
@@ -433,6 +643,9 @@ export interface StateSyncOutcome {
   toFingerprint: string;
   /** Executable statements still differing after apply — empty is the bar. */
   remaining: string[];
+  /** Constraints whose post-commit `VALIDATE` failed: installed, enforced, still unproven. They
+   *  also show up in `remaining` as a VALIDATE the next run re-emits. */
+  pendingValidations: PendingValidation[];
   /** schema_log row id, when DDL ran. */
   logId?: number;
   /** Fresh post-apply introspection (the pre-apply one when nothing ran). */
@@ -463,7 +676,7 @@ export async function applyStateAndVerify(
   if (classified.executable.length === 0) {
     return {
       statements, classified, applied: false,
-      fromFingerprint, toFingerprint: fromFingerprint, remaining: [],
+      fromFingerprint, toFingerprint: fromFingerprint, remaining: [], pendingValidations: [],
       after: { snapshot: current, contract: liveAuthz },
     };
   }
@@ -484,6 +697,7 @@ export async function applyStateAndVerify(
   return {
     statements, classified, applied: true,
     fromFingerprint, toFingerprint, remaining,
+    pendingValidations: result.pendingValidations,
     ...(result.logId !== undefined ? { logId: result.logId } : {}),
     after: { snapshot, contract },
   };