@everystack/cli 0.3.13 → 0.3.15

package/src/cli/derived-introspect.ts

@@ -0,0 +1,314 @@
+/**
+ * derived-introspect — the live-catalog side of the derived-object reconciler.
+ *
+ * Reads what the database actually has for the compute layer: functions,
+ * views, and materialized views, each with its canonical definition
+ * (`pg_get_functiondef` / `pg_get_viewdef`), its indexes (matviews), its
+ * comment, and its size — plus the dependency edges between derived objects
+ * (from pg_rewrite/pg_depend) and the reconciler's provenance claims.
+ *
+ * The live definition hash (`defHash`) is computed over the CATALOG's
+ * canonical deparse, which never matches the authored source text — Postgres
+ * reformats. It is only ever compared against the def hash the reconciler
+ * recorded at apply time; source hashes and def hashes live in different
+ * universes and never meet. See derived-plan.ts for the decision table.
+ *
+ * Same contract as schema-introspect: SQL runs through an injected
+ * QueryRunner; every mapper is pure and unit-tested without a database.
+ */
+
+import { createHash } from 'node:crypto';
+import { IGNORED_SCHEMAS, type QueryRunner } from './authz-contract.js';
+import { normalizeSql, type DerivedKind } from './derived-source.js';
+
+export interface LiveObject {
+  kind: DerivedKind;
+  schema: string;
+  name: string;
+  /** `schema.name` — join key against source objects and provenance. */
+  identity: string;
+  /** Canonical definition as the catalog deparses it. */
+  definition: string;
+  /** `pg_get_indexdef` for each index on a matview, sorted for stability. */
+  indexes: string[];
+  /** Object comment, when present (part of the def hash — a dropped comment is drift). */
+  comment?: string;
+  /** sha256 over normalized definition + indexes + comment. */
+  defHash: string;
+  /** Matview only: whether it holds data. */
+  populated?: boolean;
+  /** Matview only: pg_total_relation_size — the rebuild-cost signal. */
+  bytes?: number;
+  /** Matview only: reltuples estimate. */
+  rows?: number;
+}
+
+/** dependent identity → referenced identity (both derived objects). */
+export interface DependencyEdge {
+  dependent: string;
+  referenced: string;
+}
+
+/** What the reconciler recorded the last time it touched this object. */
+export interface ProvenanceRow {
+  identity: string;
+  srcHash: string;
+  defHash: string;
+}
+
+export interface DerivedCatalog {
+  objects: LiveObject[];
+  edges: DependencyEdge[];
+  provenance: ProvenanceRow[];
+}
+
+// ---------------------------------------------------------------------------
+// Introspection SQL.
+// ---------------------------------------------------------------------------
+
+/** Views and matviews with canonical definitions, size, and comment. */
+export const DERIVED_RELATIONS_SQL = `
+SELECT
+  n.nspname AS schema,
+  c.relname AS name,
+  c.relkind AS kind,
+  pg_get_viewdef(c.oid) AS definition,
+  obj_description(c.oid, 'pg_class') AS comment,
+  c.relispopulated AS populated,
+  pg_total_relation_size(c.oid) AS bytes,
+  c.reltuples::bigint AS rows
+FROM pg_class c
+JOIN pg_namespace n ON n.oid = c.relnamespace
+WHERE c.relkind IN ('v', 'm')
+  AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+  AND n.nspname NOT LIKE 'pg_%'
+  AND NOT EXISTS (
+    SELECT 1 FROM pg_depend dep WHERE dep.objid = c.oid AND dep.deptype = 'e'
+  )
+ORDER BY n.nspname, c.relname;
+`.trim();
+
+/** Plain functions and procedures with canonical definitions and comment. */
+export const DERIVED_FUNCTIONS_SQL = `
+SELECT
+  n.nspname AS schema,
+  p.proname AS name,
+  pg_get_functiondef(p.oid) AS definition,
+  obj_description(p.oid, 'pg_proc') AS comment
+FROM pg_proc p
+JOIN pg_namespace n ON n.oid = p.pronamespace
+WHERE p.prokind IN ('f', 'p')
+  AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+  AND n.nspname NOT LIKE 'pg_%'
+  AND NOT EXISTS (
+    SELECT 1 FROM pg_depend dep WHERE dep.objid = p.oid AND dep.deptype = 'e'
+  )
+ORDER BY n.nspname, p.proname;
+`.trim();
+
+/** Every index on a materialized view, as its canonical CREATE INDEX text. */
+export const MATVIEW_INDEXES_SQL = `
+SELECT
+  n.nspname AS schema,
+  t.relname AS name,
+  pg_get_indexdef(idx.indexrelid) AS indexdef
+FROM pg_index idx
+JOIN pg_class t ON t.oid = idx.indrelid
+JOIN pg_namespace n ON n.oid = t.relnamespace
+WHERE t.relkind = 'm'
+  AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+  AND n.nspname NOT LIKE 'pg_%'
+ORDER BY n.nspname, t.relname, idx.indexrelid;
+`.trim();
+
+/**
+ * Dependency edges between derived objects, through each view/matview's
+ * rewrite rule: view → view/matview it selects from, and view → function it
+ * calls. (Function → function edges do not exist in pg_depend — function
+ * bodies are opaque to it; source order is the fallback there.)
+ */
+export const DERIVED_DEPENDS_SQL = `
+SELECT DISTINCT
+  dn.nspname AS dependent_schema,
+  dc.relname AS dependent_name,
+  rn.nspname AS referenced_schema,
+  rc.relname AS referenced_name
+FROM pg_depend d
+JOIN pg_rewrite rw ON rw.oid = d.objid
+JOIN pg_class dc ON dc.oid = rw.ev_class
+JOIN pg_namespace dn ON dn.oid = dc.relnamespace
+JOIN pg_class rc ON rc.oid = d.refobjid
+JOIN pg_namespace rn ON rn.oid = rc.relnamespace
+WHERE d.classid = 'pg_rewrite'::regclass
+  AND d.refclassid = 'pg_class'::regclass
+  AND dc.oid <> rc.oid
+  AND dc.relkind IN ('v', 'm')
+  AND rc.relkind IN ('v', 'm')
+UNION
+SELECT DISTINCT
+  dn.nspname, dc.relname, rn.nspname, rp.proname
+FROM pg_depend d
+JOIN pg_rewrite rw ON rw.oid = d.objid
+JOIN pg_class dc ON dc.oid = rw.ev_class
+JOIN pg_namespace dn ON dn.oid = dc.relnamespace
+JOIN pg_proc rp ON rp.oid = d.refobjid
+JOIN pg_namespace rn ON rn.oid = rp.pronamespace
+WHERE d.classid = 'pg_rewrite'::regclass
+  AND d.refclassid = 'pg_proc'::regclass
+  AND dc.relkind IN ('v', 'm');
+`.trim();
+
+/** The reconciler's provenance claims. The table may not exist yet — callers tolerate that. */
+export const PROVENANCE_SQL = `
+SELECT identity, src_hash, def_hash FROM everystack.derived_provenance;
+`.trim();
+
+// ---------------------------------------------------------------------------
+// Pure mappers.
+// ---------------------------------------------------------------------------
+
+export interface RelationRow {
+  schema: string;
+  name: string;
+  kind: unknown;
+  definition: unknown;
+  comment: unknown;
+  populated: unknown;
+  bytes: unknown;
+  rows: unknown;
+}
+
+export interface FunctionRow {
+  schema: string;
+  name: string;
+  definition: unknown;
+  comment: unknown;
+}
+
+export interface IndexDefRow {
+  schema: string;
+  name: string;
+  indexdef: unknown;
+}
+
+export interface DependsRow {
+  dependent_schema: string;
+  dependent_name: string;
+  referenced_schema: string;
+  referenced_name: string;
+}
+
+export interface ProvenanceRawRow {
+  identity: string;
+  src_hash: unknown;
+  def_hash: unknown;
+}
+
+/** sha256 over an object's normalized definition + sorted indexes + comment. */
+export function computeDefHash(definition: string, indexes: string[], comment?: string): string {
+  const content = [normalizeSql(definition), ...[...indexes].sort().map(normalizeSql), comment ?? '']
+    .join('\n');
+  return createHash('sha256').update(content).digest('hex');
+}
+
+export interface DerivedRows {
+  relations: RelationRow[];
+  functions: FunctionRow[];
+  indexes: IndexDefRow[];
+  depends: DependsRow[];
+  provenance: ProvenanceRawRow[];
+}
+
+/**
+ * Fold introspected rows into the catalog. Deterministic: objects sorted by
+ * identity, indexes sorted, edges sorted — a re-introspection of an unchanged
+ * database is byte-stable.
+ */
+export function assembleDerivedCatalog(rows: DerivedRows): DerivedCatalog {
+  const indexesByIdentity = new Map<string, string[]>();
+  for (const row of rows.indexes) {
+    if (IGNORED_SCHEMAS.has(row.schema)) continue;
+    const identity = `${row.schema}.${row.name}`;
+    const list = indexesByIdentity.get(identity) ?? [];
+    list.push(String(row.indexdef));
+    indexesByIdentity.set(identity, list);
+  }
+
+  const objects: LiveObject[] = [];
+
+  for (const row of rows.relations) {
+    if (IGNORED_SCHEMAS.has(row.schema)) continue;
+    const kind: DerivedKind = String(row.kind) === 'm' ? 'materialized view' : 'view';
+    const identity = `${row.schema}.${row.name}`;
+    const definition = String(row.definition ?? '');
+    const indexes = [...(indexesByIdentity.get(identity) ?? [])].sort();
+    const comment = row.comment == null ? undefined : String(row.comment);
+    const obj: LiveObject = {
+      kind, schema: row.schema, name: row.name, identity,
+      definition, indexes,
+      ...(comment !== undefined ? { comment } : {}),
+      defHash: computeDefHash(definition, indexes, comment),
+    };
+    if (kind === 'materialized view') {
+      obj.populated = row.populated == null ? undefined : String(row.populated) !== 'false';
+      obj.bytes = row.bytes == null ? undefined : Number(row.bytes);
+      obj.rows = row.rows == null ? undefined : Math.max(0, Number(row.rows));
+    }
+    objects.push(obj);
+  }
+
+  for (const row of rows.functions) {
+    if (IGNORED_SCHEMAS.has(row.schema)) continue;
+    const identity = `${row.schema}.${row.name}`;
+    const definition = String(row.definition ?? '');
+    const comment = row.comment == null ? undefined : String(row.comment);
+    objects.push({
+      kind: 'function', schema: row.schema, name: row.name, identity,
+      definition, indexes: [],
+      ...(comment !== undefined ? { comment } : {}),
+      defHash: computeDefHash(definition, [], comment),
+    });
+  }
+
+  const edges: DependencyEdge[] = rows.depends
+    .filter((r) => !IGNORED_SCHEMAS.has(r.dependent_schema) && !IGNORED_SCHEMAS.has(r.referenced_schema))
+    .map((r) => ({
+      dependent: `${r.dependent_schema}.${r.dependent_name}`,
+      referenced: `${r.referenced_schema}.${r.referenced_name}`,
+    }))
+    .sort((a, b) => (a.dependent + a.referenced).localeCompare(b.dependent + b.referenced));
+
+  const provenance: ProvenanceRow[] = rows.provenance
+    .map((r) => ({ identity: r.identity, srcHash: String(r.src_hash ?? ''), defHash: String(r.def_hash ?? '') }))
+    .sort((a, b) => a.identity.localeCompare(b.identity));
+
+  return {
+    objects: objects.sort((a, b) => a.identity.localeCompare(b.identity)),
+    edges,
+    provenance,
+  };
+}
+
+/**
+ * Introspect the live derived layer. Runs the read-only SQL through the
+ * injected runner; a missing provenance table (first run against a database
+ * the reconciler has never touched) folds to an empty claims list.
+ */
+export async function introspectDerived(run: QueryRunner): Promise<DerivedCatalog> {
+  const [relations, functions, indexes, depends] = await Promise.all([
+    run(DERIVED_RELATIONS_SQL), run(DERIVED_FUNCTIONS_SQL), run(MATVIEW_INDEXES_SQL), run(DERIVED_DEPENDS_SQL),
+  ]);
+  let provenance: ProvenanceRawRow[] = [];
+  try {
+    provenance = (await run(PROVENANCE_SQL)) as ProvenanceRawRow[];
+  } catch {
+    // everystack.derived_provenance does not exist yet — nothing has been reconciled.
+  }
+  return assembleDerivedCatalog({
+    relations: relations as RelationRow[],
+    functions: functions as FunctionRow[],
+    indexes: indexes as IndexDefRow[],
+    depends: depends as DependsRow[],
+    provenance,
+  });
+}

package/src/cli/derived-plan.ts

@@ -0,0 +1,279 @@
+/**
+ * derived-plan — the reconciler's pure decision core.
+ *
+ * Joins the three inputs on identity — source objects (db/sql), the live
+ * catalog, and the provenance claims — and decides, per object: skip, create,
+ * replace, refresh, drop, baseline, or prune. Hand-edited live objects are
+ * DRIFT: reported, never silently overwritten (`overwriteDrift` makes the
+ * overwrite explicit, and the drift stays in the report so the operator sees
+ * what they overwrote). Never-reconciled matches are `needsBaseline`: the
+ * reconciler refuses to guess whether live matches source — `baseline: true`
+ * records the trust explicitly, once.
+ *
+ * Graph work: replacing (or dropping) a relation rebuilds its live transitive
+ * dependents — dropped in reverse-topological order, recreated in source
+ * order (the declared dependency order). Replacing a function refreshes the
+ * matviews that call it (functions replace in place via CREATE OR REPLACE; a
+ * matview's stored rows are stale the moment its function changes). A live
+ * dependent with no source to rebuild from BLOCKS its chain: nothing in that
+ * chain applies, and the report names the blocker.
+ *
+ * Decisions here never read timestamps and never trust provenance where the
+ * catalog can contradict it — a poisoned claim converges to a rebuild or a
+ * drift report, not a silent skip.
+ */
+
+import type { ParsedSources, SourceObject, DerivedKind } from './derived-source.js';
+import type { DerivedCatalog, LiveObject } from './derived-introspect.js';
+
+export type ReconcileVerb = 'create' | 'replace' | 'refresh' | 'drop' | 'baseline' | 'prune';
+
+export interface ReconcileAction {
+  action: ReconcileVerb;
+  identity: string;
+  kind: DerivedKind;
+  reason: string;
+  /** Rebuild-cost signal from the live catalog (matviews). */
+  bytes?: number;
+  rows?: number;
+}
+
+export interface DriftFinding {
+  identity: string;
+  detail: string;
+}
+
+export interface BlockedFinding {
+  identity: string;
+  reason: string;
+}
+
+export interface ReconcilePlan {
+  /** In execution order: drops (reverse-topo), function replaces, creates (source order), refreshes, baselines, prunes. */
+  actions: ReconcileAction[];
+  /** Managed objects that are up to date. */
+  skipped: string[];
+  /** Hand-edited managed objects — reported always, rebuilt only under overwriteDrift. */
+  drift: DriftFinding[];
+  /** Source↔live matches the reconciler has never touched — baseline explicitly or rebuild. */
+  needsBaseline: string[];
+  /** Live derived objects with no source and no provenance — not ours, never touched. */
+  unmanaged: string[];
+  /** Chains held because a live dependent has no source to rebuild from. */
+  blocked: BlockedFinding[];
+  /** Source-parse warnings, passed through so one report carries everything. */
+  warnings: string[];
+  /** Sum of live bytes over rebuilt + refreshed matviews — the headline cost estimate. */
+  totalRebuildBytes: number;
+}
+
+export interface ReconcileOptions {
+  /** Record provenance for never-reconciled source↔live matches instead of flagging them. */
+  baseline?: boolean;
+  /** Rebuild drifted objects from source (the drift finding stays in the report). */
+  overwriteDrift?: boolean;
+}
+
+const isRelation = (kind: DerivedKind): boolean => kind !== 'function';
+
+export function planReconcile(
+  source: ParsedSources,
+  live: DerivedCatalog,
+  options: ReconcileOptions = {},
+): ReconcilePlan {
+  const srcById = new Map(source.objects.map((o) => [o.identity, o]));
+  const liveById = new Map(live.objects.map((o) => [o.identity, o]));
+  const provById = new Map(live.provenance.map((p) => [p.identity, p]));
+
+  const skipped: string[] = [];
+  const drift: DriftFinding[] = [];
+  const needsBaseline: string[] = [];
+  const unmanaged: string[] = [];
+  const blocked: BlockedFinding[] = [];
+
+  /** identity → rebuild reason, for relations that must drop + recreate. */
+  const rebuild = new Map<string, string>();
+  /** Function replacements (in place). */
+  const fnReplace = new Map<string, string>();
+  /** Creations with no live counterpart. */
+  const create = new Map<string, string>();
+  /** Managed objects the source removed. */
+  const drop = new Map<string, string>();
+  const baseline: string[] = [];
+  const prune: string[] = [];
+
+  // -------------------------------------------------------------------------
+  // The decision table.
+  // -------------------------------------------------------------------------
+
+  for (const src of source.objects) {
+    const liveObj = liveById.get(src.identity);
+    const prov = provById.get(src.identity);
+
+    if (!liveObj) {
+      create.set(src.identity, prov
+        ? 'missing from the database (recorded as applied — dropped by hand?)'
+        : 'new in source');
+      continue;
+    }
+    if (!prov) {
+      if (options.baseline) baseline.push(src.identity);
+      else needsBaseline.push(src.identity);
+      continue;
+    }
+
+    const srcChanged = src.hash !== prov.srcHash;
+    const liveChanged = liveObj.defHash !== prov.defHash;
+
+    if (liveChanged) {
+      drift.push({
+        identity: src.identity,
+        detail: srcChanged
+          ? 'live definition differs from what the reconciler applied AND the source changed — resolve by hand or rebuild with overwriteDrift'
+          : 'live definition differs from what the reconciler applied (hand-edited?)',
+      });
+      if (options.overwriteDrift) {
+        if (isRelation(src.kind)) rebuild.set(src.identity, 'rebuilt from source (drift overwritten)');
+        else fnReplace.set(src.identity, 'replaced from source (drift overwritten)');
+      }
+      continue;
+    }
+    if (srcChanged) {
+      if (isRelation(src.kind)) rebuild.set(src.identity, 'source changed');
+      else fnReplace.set(src.identity, 'source changed');
+      continue;
+    }
+    skipped.push(src.identity);
+  }
+
+  for (const liveObj of live.objects) {
+    if (srcById.has(liveObj.identity)) continue;
+    if (provById.has(liveObj.identity)) drop.set(liveObj.identity, 'removed from source');
+    else unmanaged.push(liveObj.identity);
+  }
+
+  for (const prov of live.provenance) {
+    if (!srcById.has(prov.identity) && !liveById.has(prov.identity)) prune.push(prov.identity);
+  }
+
+  // -------------------------------------------------------------------------
+  // The graph: cascade rebuilds, refreshes, and blocked chains.
+  // -------------------------------------------------------------------------
+
+  const dependents = new Map<string, string[]>();
+  for (const e of live.edges) {
+    const list = dependents.get(e.referenced) ?? [];
+    list.push(e.dependent);
+    dependents.set(e.referenced, list);
+  }
+
+  const closureOf = (root: string): string[] => {
+    const seen = new Set<string>();
+    const queue = [...(dependents.get(root) ?? [])];
+    while (queue.length) {
+      const next = queue.shift()!;
+      if (seen.has(next)) continue;
+      seen.add(next);
+      queue.push(...(dependents.get(next) ?? []));
+    }
+    return [...seen];
+  };
+
+  // Relation roots whose live dependents must be handled: rebuilds and drops.
+  const relationRoots = [...rebuild.keys(), ...drop.keys()];
+  for (const root of relationRoots) {
+    const closure = closureOf(root).filter((id) => liveById.get(id) && isRelation(liveById.get(id)!.kind));
+    const blockers = closure.filter((id) => !srcById.has(id) && !drop.has(id));
+    if (blockers.length > 0) {
+      blocked.push({
+        identity: root,
+        reason: `live dependents with no source to rebuild from: ${blockers.join(', ')} — held`,
+      });
+      rebuild.delete(root);
+      drop.delete(root);
+      continue;
+    }
+    for (const dep of closure) {
+      if (!rebuild.has(dep) && !drop.has(dep) && srcById.has(dep)) {
+        rebuild.set(dep, `dependency rebuild (depends on ${root})`);
+      }
+    }
+  }
+
+  // Matviews that call a replaced function hold stale rows — refresh them,
+  // unless the rebuild already recreates them fresh.
+  const refresh = new Map<string, string>();
+  for (const fnId of fnReplace.keys()) {
+    for (const dep of closureOf(fnId)) {
+      const liveDep = liveById.get(dep);
+      if (!liveDep || liveDep.kind !== 'materialized view') continue;
+      if (rebuild.has(dep) || drop.has(dep)) continue;
+      refresh.set(dep, `depends on replaced function ${fnId}`);
+    }
+  }
+
+  // -------------------------------------------------------------------------
+  // Ordering.
+  // -------------------------------------------------------------------------
+
+  /** Reverse-topological among the drop set: dependents drop before their dependencies. */
+  const orderDrops = (ids: string[]): string[] => {
+    const inSet = new Set(ids);
+    const depth = new Map<string, number>();
+    const depthOf = (id: string, seen: Set<string>): number => {
+      if (depth.has(id)) return depth.get(id)!;
+      if (seen.has(id)) return 0; // cycle guard — Postgres would not have allowed it anyway
+      seen.add(id);
+      const deps = (dependents.get(id) ?? []).filter((d) => inSet.has(d));
+      const d = deps.length === 0 ? 0 : 1 + Math.max(...deps.map((x) => depthOf(x, seen)));
+      depth.set(id, d);
+      return d;
+    };
+    return [...ids].sort((a, b) => depthOf(a, new Set()) - depthOf(b, new Set()) || a.localeCompare(b));
+  };
+
+  const seqOf = (id: string): number => srcById.get(id)?.seq ?? Number.MAX_SAFE_INTEGER;
+
+  const dropSet = [...new Set([...rebuild.keys(), ...drop.keys()])].filter((id) => liveById.has(id));
+  const createSet = [...new Set([...rebuild.keys(), ...create.keys()])].filter((id) => srcById.has(id));
+
+  const actionFor = (id: string, action: ReconcileVerb, reason: string): ReconcileAction => {
+    const src = srcById.get(id);
+    const liveObj = liveById.get(id);
+    const kind = (src?.kind ?? liveObj?.kind) as DerivedKind;
+    const cost = liveObj?.kind === 'materialized view'
+      ? { ...(liveObj.bytes !== undefined ? { bytes: liveObj.bytes } : {}), ...(liveObj.rows !== undefined ? { rows: liveObj.rows } : {}) }
+      : {};
+    return { action, identity: id, kind, reason, ...cost };
+  };
+
+  const actions: ReconcileAction[] = [
+    ...orderDrops(dropSet).map((id) => actionFor(id, 'drop', rebuild.get(id) ?? drop.get(id) ?? '')),
+    ...[...fnReplace.entries()].sort((a, b) => seqOf(a[0]) - seqOf(b[0]))
+      .map(([id, reason]) => actionFor(id, 'replace', reason)),
+    ...createSet.sort((a, b) => seqOf(a) - seqOf(b))
+      .map((id) => actionFor(id, 'create', rebuild.get(id) ?? create.get(id) ?? '')),
+    ...[...refresh.entries()].sort((a, b) => seqOf(a[0]) - seqOf(b[0]))
+      .map(([id, reason]) => actionFor(id, 'refresh', reason)),
+    ...baseline.sort().map((id) => actionFor(id, 'baseline', 'recording provenance for an existing match (trusted once, explicitly)')),
+    ...prune.sort().map((id) => ({
+      action: 'prune' as const, identity: id, kind: 'view' as DerivedKind,
+      reason: 'stale provenance — object gone from both source and database',
+    })),
+  ];
+
+  const totalRebuildBytes = actions
+    .filter((a) => (a.action === 'create' || a.action === 'refresh') && a.bytes !== undefined)
+    .reduce((sum, a) => sum + (a.bytes ?? 0), 0);
+
+  return {
+    actions,
+    skipped: skipped.sort(),
+    drift,
+    needsBaseline: needsBaseline.sort(),
+    unmanaged: unmanaged.sort(),
+    blocked,
+    warnings: [...source.warnings],
+    totalRebuildBytes,
+  };
+}