@everystack/cli 0.3.13 → 0.3.15

package/README.md

@@ -216,6 +216,7 @@ everystack db:psql --stage dev [-c SQL] # interactive psql / one-off query via L
 # Model-driven schema (@everystack/model)
 everystack db:generate                  # diff Models vs the live DB → next migration
 everystack db:pull                      # introspect a live DB → field() Models (brownfield on-ramp)
+everystack db:reconcile                 # deploy the derived layer (functions/views/matviews) from db/sql — no migrations
 
 # Security
 everystack db:doctor                    # is the DB least-privilege + RLS-subject?

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.3.13",
+  "version": "0.3.15",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/commands/db-authz.ts

@@ -44,6 +44,7 @@ import { writeContract, loadContract } from '../authz-contract-io.js';
 import { renderContractMarkdown } from '../authz-render.js';
 import { FUNCTIONS_SQL, catalogFunctionToDescriptor } from '../security-catalog.js';
 import { resolveConfig, opsFunction } from '../config.js';
+import { resolveModelsPath } from '../models-path.js';
 import { invokeAction } from '../aws.js';
 import { step, success, fail, info, warn } from '../output.js';
 
@@ -239,7 +240,7 @@ async function loadModels(modelsPath: string): Promise<ModelDescriptor[]> {
  * exit on any leak or vacuous policy; an `unprobed` table (one identity) is a warning, not a fail.
  */
 export async function dbAuthzOwnerCommand(flags: Record<string, string>): Promise<void> {
-  const modelsPath = flags.models || 'models/index.ts';
+  const modelsPath = resolveModelsPath(flags.models);
   const schema = flags.schema || 'public';
 
   let probes: OwnerProbe[];

package/src/cli/commands/db-generate.ts

@@ -28,11 +28,11 @@ import { generateMigrationSql, unmodeledTables, formatMigrationFile, planMigrati
 import { introspectContract, type QueryRunner } from '../authz-contract.js';
 import { FUNCTIONS_SQL, catalogFunctionToDescriptor } from '../security-catalog.js';
 import { resolveDbSource, createUrlRunner, type DbSource } from '../db-source.js';
+import { resolveModelsPath } from '../models-path.js';
 import { resolveConfig, opsFunction } from '../config.js';
 import { invokeAction } from '../aws.js';
 import { step, success, fail, info, warn } from '../output.js';
 
-const DEFAULT_MODELS = 'models/index.ts';
 const DEFAULT_MIGRATIONS = 'drizzle';
 const DEFAULT_SCHEMA_OUT = 'db/schema.generated.ts';
 
@@ -123,7 +123,7 @@ async function dbGenerateInit(modelsPath: string, migrationsDir: string, schemaO
 }
 
 export async function dbGenerateCommand(flags: Record<string, string>): Promise<void> {
-  const modelsPath = flags.models || DEFAULT_MODELS;
+  const modelsPath = resolveModelsPath(flags.models);
   const migrationsDir = path.resolve(flags.dir || DEFAULT_MIGRATIONS);
   const schemaOut = path.resolve(flags['schema-out'] || DEFAULT_SCHEMA_OUT);
   const name = flags.name || 'generated';

package/src/cli/commands/db-reconcile.ts

@@ -0,0 +1,300 @@
+/**
+ * `everystack db:reconcile` — deploy the derived layer (functions, views,
+ * matviews) by reconciling the live database against the db/sql sources.
+ *
+ *   db:reconcile [--sql-dir db/sql] [--stage <name> | --database-url <url>]
+ *                [--apply] [--check] [--baseline] [--overwrite-drift] [--json]
+ *
+ * Derived objects do not migrate; they deploy. The plan is computed fresh
+ * every run: source objects (hashed, comment/whitespace-insensitive) joined
+ * against the live catalog and the reconciler's provenance claims. Default
+ * mode prints the plan with rebuild-cost estimates. `--check` is the CI gate
+ * (exit 1 when anything differs). `--apply` executes: the DDL goes as ONE
+ * multi-statement batch (one implicit transaction — a failed create rolls
+ * back its drops), then provenance is re-recorded from fresh introspection,
+ * and the full applied SQL is appended to `everystack.schema_log`.
+ *
+ * Hand-edited live objects are DRIFT — reported, applied over only with
+ * `--overwrite-drift`. Never-reconciled matches need `--baseline` once (the
+ * explicit trust that live == source, recorded). `--apply` needs a direct
+ * connection (`--database-url` / DATABASE_URL): the ops Lambda `db:query`
+ * path is read-only by design, so plan/check work anywhere but writes go
+ * through a credentialed connection you chose to hold.
+ */
+
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { execSync } from 'node:child_process';
+import type { QueryRunner } from '../authz-contract.js';
+import { parseSqlSources, type SourceFile } from '../derived-source.js';
+import { introspectDerived } from '../derived-introspect.js';
+import { planReconcile, type ReconcilePlan, type ReconcileOptions } from '../derived-plan.js';
+import {
+  renderReconcileSql,
+  renderProvenanceUpsert,
+  renderProvenanceDelete,
+  renderSchemaLogInsert,
+  ENSURE_RECONCILER_SQL,
+} from '../derived-apply.js';
+import { resolveDbSource, createUrlRunner, type DbSource } from '../db-source.js';
+import { resolveConfig, opsFunction } from '../config.js';
+import { invokeAction } from '../aws.js';
+import { step, success, fail, info, warn } from '../output.js';
+
+const DEFAULT_SQL_DIR = 'db/sql';
+
+// ---------------------------------------------------------------------------
+// The testable core.
+// ---------------------------------------------------------------------------
+
+export interface ExecuteOptions extends ReconcileOptions {
+  apply?: boolean;
+  actor?: string | null;
+  gitRef?: string | null;
+  planRef?: string | null;
+  /** Injectable clock for tests. */
+  now?: () => number;
+}
+
+export interface ReconcileRun {
+  plan: ReconcilePlan;
+  /** True when DDL/bookkeeping was actually executed. */
+  applied: boolean;
+  /** The DDL batch that was (or would be) sent. */
+  statements: string[];
+  /** Why apply was refused, when it was. */
+  refusal?: string;
+}
+
+/**
+ * Plan — and under `apply`, execute — one reconcile. Pure orchestration over
+ * an injected QueryRunner; the CLI shell below owns flags, files, and exits.
+ */
+export async function executeReconcile(
+  runner: QueryRunner,
+  sources: SourceFile[],
+  options: ExecuteOptions = {},
+): Promise<ReconcileRun> {
+  const live = await introspectDerived(runner);
+  const parsed = parseSqlSources(sources);
+  const plan = planReconcile(parsed, live, options);
+  const rendered = renderReconcileSql(plan, parsed.objects);
+
+  if (!options.apply) return { plan, applied: false, statements: rendered.statements };
+
+  if (plan.blocked.length > 0) {
+    return {
+      plan, applied: false, statements: rendered.statements,
+      refusal: `blocked chains: ${plan.blocked.map((b) => b.identity).join(', ')} — resolve the unmanaged dependents first`,
+    };
+  }
+  if (plan.drift.length > 0 && !options.overwriteDrift) {
+    return {
+      plan, applied: false, statements: rendered.statements,
+      refusal: `drift on ${plan.drift.map((d) => d.identity).join(', ')} — inspect, then re-run with --overwrite-drift to rebuild from source`,
+    };
+  }
+  if (rendered.statements.length === 0 && rendered.record.length === 0 && rendered.remove.length === 0) {
+    return { plan, applied: false, statements: [] };
+  }
+
+  const now = options.now ?? Date.now;
+  await runner(ENSURE_RECONCILER_SQL.join(';\n'));
+
+  const started = now();
+  let outcome = 'applied';
+  try {
+    if (rendered.statements.length > 0) {
+      await runner(rendered.statements.join(';\n'));
+    }
+  } catch (err: any) {
+    outcome = 'failed';
+    // The batch rolled back; record the attempt, then surface the failure.
+    try {
+      await runner(renderSchemaLogInsert({
+        kind: 'compute reconcile', sql: rendered.statements.join(';\n'),
+        outcome: `failed: ${String(err?.message ?? err).slice(0, 500)}`,
+        actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
+        durationMs: now() - started,
+      }));
+    } catch { /* the memoir is best-effort on failure */ }
+    throw err;
+  }
+
+  // The batch committed — re-read the catalog so provenance records the def
+  // hashes of what NOW exists, not what we hoped would exist.
+  const after = await introspectDerived(runner);
+  const liveById = new Map(after.objects.map((o) => [o.identity, o]));
+  const srcById = new Map(parsed.objects.map((o) => [o.identity, o]));
+
+  const bookkeeping: string[] = [];
+  for (const identity of rendered.record) {
+    const src = srcById.get(identity);
+    const liveObj = liveById.get(identity);
+    if (!src || !liveObj) {
+      throw new Error(`reconcile applied but ${identity} is not introspectable afterwards — provenance not recorded, investigate`);
+    }
+    bookkeeping.push(renderProvenanceUpsert(identity, src.hash, liveObj.defHash));
+  }
+  for (const identity of rendered.remove) bookkeeping.push(renderProvenanceDelete(identity));
+  bookkeeping.push(renderSchemaLogInsert({
+    kind: 'compute reconcile',
+    sql: rendered.statements.join(';\n') || '-- provenance-only (baseline/prune)',
+    outcome,
+    actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
+    durationMs: now() - started,
+  }));
+  await runner(bookkeeping.join(';\n'));
+
+  return { plan, applied: true, statements: rendered.statements };
+}
+
+// ---------------------------------------------------------------------------
+// Report rendering (pure).
+// ---------------------------------------------------------------------------
+
+export function formatBytes(bytes: number): string {
+  if (bytes >= 1024 ** 3) return `${(bytes / 1024 ** 3).toFixed(1)} GB`;
+  if (bytes >= 1024 ** 2) return `${(bytes / 1024 ** 2).toFixed(1)} MB`;
+  if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${bytes} B`;
+}
+
+const VERB_GLYPH: Record<string, string> = {
+  create: '+', replace: '~', drop: '-', refresh: '↻', baseline: '◦', prune: '·',
+};
+
+export function buildReconcileReport(plan: ReconcilePlan): string[] {
+  const lines: string[] = [];
+  if (plan.skipped.length > 0) lines.push(`= ${plan.skipped.length} up to date`);
+  for (const a of plan.actions) {
+    const cost = a.bytes !== undefined ? `; ${formatBytes(a.bytes)}${a.rows ? `, ~${a.rows} rows` : ''}` : '';
+    lines.push(`${VERB_GLYPH[a.action] ?? '?'} ${a.action} ${a.identity} (${a.reason}${cost})`);
+  }
+  const rebuilt = plan.actions.filter((a) => (a.action === 'create' || a.action === 'refresh') && a.bytes !== undefined);
+  if (plan.totalRebuildBytes > 0) {
+    lines.push(`estimated rebuild: ${formatBytes(plan.totalRebuildBytes)} across ${rebuilt.length} matview(s)`);
+  }
+  for (const d of plan.drift) lines.push(`! drift ${d.identity} — ${d.detail}`);
+  if (plan.needsBaseline.length > 0) {
+    lines.push(`! needs baseline (${plan.needsBaseline.length}): ${plan.needsBaseline.join(', ')} — run with --baseline to record trust, once`);
+  }
+  if (plan.unmanaged.length > 0) {
+    lines.push(`! unmanaged (${plan.unmanaged.length}): ${plan.unmanaged.join(', ')} — not in source, never touched`);
+  }
+  for (const b of plan.blocked) lines.push(`! blocked ${b.identity} — ${b.reason}`);
+  for (const w of plan.warnings) lines.push(`! ${w}`);
+  if (lines.length === 0) lines.push('nothing to do — derived layer matches source');
+  return lines;
+}
+
+/** Anything that makes `--check` fail: pending work or unresolved findings. */
+export function checkFails(plan: ReconcilePlan): boolean {
+  return plan.actions.some((a) => a.action !== 'baseline')
+    || plan.drift.length > 0
+    || plan.needsBaseline.length > 0
+    || plan.blocked.length > 0;
+}
+
+// ---------------------------------------------------------------------------
+// The CLI shell.
+// ---------------------------------------------------------------------------
+
+/** A QueryRunner backed by the ops Lambda `db:query` action (read-only). */
+function lambdaRunner(region: string, fn: string): QueryRunner {
+  return async (sql: string) => {
+    const result: any = await invokeAction(region, fn, 'db:query', { sql });
+    if (result?.error) throw new Error(`Query failed: ${result.error}`);
+    return result?.rows ?? [];
+  };
+}
+
+async function readSqlDir(dir: string): Promise<SourceFile[]> {
+  let entries: string[];
+  try {
+    entries = await fs.readdir(dir);
+  } catch {
+    throw new Error(`No SQL source directory at ${dir} (set --sql-dir).`);
+  }
+  const files = entries.filter((f) => f.endsWith('.sql')).sort();
+  return Promise.all(files.map(async (file) => ({
+    file: path.join(dir, file),
+    sql: await fs.readFile(path.join(dir, file), 'utf-8'),
+  })));
+}
+
+function currentGitRef(): string | null {
+  try {
+    return execSync('git rev-parse HEAD', { stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim();
+  } catch {
+    return null;
+  }
+}
+
+export async function dbReconcileCommand(flags: Record<string, string>): Promise<void> {
+  const sqlDir = flags['sql-dir'] || DEFAULT_SQL_DIR;
+  const apply = flags.apply === 'true';
+  const check = flags.check === 'true';
+
+  let dbSource: DbSource;
+  try {
+    dbSource = resolveDbSource(flags);
+  } catch (err: any) {
+    fail(err.message);
+    process.exit(1);
+  }
+
+  if (apply && dbSource.kind !== 'url') {
+    fail('--apply needs a direct connection (--database-url or DATABASE_URL) — the ops Lambda query path is read-only by design.');
+    process.exit(1);
+  }
+
+  let sources: SourceFile[];
+  try {
+    sources = await readSqlDir(sqlDir);
+  } catch (err: any) {
+    fail(err.message);
+    process.exit(1);
+  }
+
+  let runner: QueryRunner;
+  let end: (() => Promise<void>) | undefined;
+  if (dbSource.kind === 'url') {
+    step(dbSource.from === 'flag' ? 'Connecting via --database-url...' : 'Connecting via DATABASE_URL...');
+    ({ runner, end } = await createUrlRunner(dbSource.url));
+  } else {
+    step('Resolving deployed config...');
+    const config = await resolveConfig(flags.stage);
+    runner = lambdaRunner(config.region, opsFunction(config));
+  }
+
+  try {
+    const run = await executeReconcile(runner, sources, {
+      apply,
+      baseline: flags.baseline === 'true',
+      overwriteDrift: flags['overwrite-drift'] === 'true',
+      actor: process.env.USER ?? null,
+      gitRef: currentGitRef(),
+    });
+
+    if (flags.json === 'true') {
+      console.log(JSON.stringify({ ...run.plan, applied: run.applied, refusal: run.refusal ?? null }, null, 2));
+    } else {
+      for (const line of buildReconcileReport(run.plan)) info(line);
+      if (run.refusal) {
+        fail(`not applied: ${run.refusal}`);
+      } else if (run.applied) {
+        success(`Applied ${run.statements.length} statement(s); provenance and schema_log recorded.`);
+      } else if (apply) {
+        success('Nothing to apply — derived layer matches source.');
+      } else if (run.statements.length > 0) {
+        warn('Plan only — re-run with --apply to execute.');
+      }
+    }
+
+    if (run.refusal) process.exit(1);
+    if (check && checkFails(run.plan)) process.exit(1);
+  } finally {
+    await end?.();
+  }
+}

package/src/cli/derived-apply.ts

@@ -0,0 +1,162 @@
+/**
+ * derived-apply — renders a ReconcilePlan into executable SQL.
+ *
+ * Pure: no database here. The command sends the DDL batch as ONE
+ * multi-statement string (postgres simple-query protocol = one implicit
+ * transaction, so a failed create rolls back the drops that preceded it),
+ * then — after the batch commits — re-introspects the touched objects and
+ * records provenance with the fresh live def hashes, and appends the
+ * schema_log row carrying the full SQL text that was applied.
+ *
+ * The bookkeeping tables live in the `everystack` schema with no grants
+ * beyond the owner: admin-only by construction. schema_log is a memoir —
+ * the reconciler writes it and never reads it.
+ */
+
+import { normalizeSql, type SourceObject } from './derived-source.js';
+import type { ReconcilePlan } from './derived-plan.js';
+import { quoteQualified } from './pg-ident.js';
+
+/** SQL string literal with '' doubling (standard_conforming_strings). */
+export function escapeLiteral(value: string): string {
+  return `'${value.replace(/'/g, "''")}'`;
+}
+
+const nullable = (value: string | null | undefined): string =>
+  value == null ? 'NULL' : escapeLiteral(value);
+
+/** Bookkeeping DDL — idempotent, safe to send before every apply. */
+export const ENSURE_RECONCILER_SQL: string[] = [
+  `CREATE SCHEMA IF NOT EXISTS everystack`,
+  `CREATE TABLE IF NOT EXISTS everystack.derived_provenance (
+  identity text PRIMARY KEY,
+  src_hash text NOT NULL,
+  def_hash text NOT NULL,
+  applied_at timestamptz NOT NULL DEFAULT now()
+)`,
+  `CREATE TABLE IF NOT EXISTS everystack.schema_log (
+  id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+  applied_at timestamptz NOT NULL DEFAULT now(),
+  kind text NOT NULL,
+  actor text,
+  git_ref text,
+  plan_ref text,
+  from_fingerprint text,
+  to_fingerprint text,
+  sql text NOT NULL,
+  duration_ms integer,
+  outcome text NOT NULL
+)`,
+];
+
+export interface RenderedReconcile {
+  /** DDL in execution order — send as one batch (one implicit transaction). */
+  statements: string[];
+  /** Identities to (re)record in provenance after the batch commits. */
+  record: string[];
+  /** Identities whose provenance rows must be deleted (drops + prunes). */
+  remove: string[];
+}
+
+const DROP_KEYWORD: Record<string, string> = {
+  'materialized view': 'MATERIALIZED VIEW',
+  view: 'VIEW',
+  function: 'FUNCTION',
+};
+
+/** Rewrite `CREATE FUNCTION` to `CREATE OR REPLACE FUNCTION` (replace-in-place). */
+export function ensureOrReplace(sql: string): string {
+  return sql.replace(/^(\s*)CREATE\s+FUNCTION/i, '$1CREATE OR REPLACE FUNCTION');
+}
+
+const WITH_NO_DATA_RE = /WITH\s+NO\s+DATA\s*$/i;
+
+/**
+ * An object's full creation SQL: the CREATE statement plus its attachments, in
+ * source order. A matview whose source says `WITH NO DATA` gets a trailing
+ * REFRESH: the reconciler dropped a populated matview to rebuild it, and
+ * executing the source verbatim would leave it unpopulated — matching the
+ * text but regressing the database. (Found by a consumer whose extracted
+ * sources carry pg_dump's WITH NO DATA ordering.)
+ */
+function objectSql(obj: SourceObject): string[] {
+  const statements = [obj.sql, ...obj.attachments.map((a) => a.sql)];
+  if (obj.kind === 'materialized view' && WITH_NO_DATA_RE.test(normalizeSql(obj.sql))) {
+    statements.push(`REFRESH MATERIALIZED VIEW ${quoteQualified(obj.identity)}`);
+  }
+  return statements;
+}
+
+export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[]): RenderedReconcile {
+  const srcById = new Map(source.map((o) => [o.identity, o]));
+  const statements: string[] = [];
+  const record: string[] = [];
+  const remove: string[] = [];
+
+  for (const action of plan.actions) {
+    switch (action.action) {
+      case 'drop': {
+        statements.push(`DROP ${DROP_KEYWORD[action.kind]} IF EXISTS ${quoteQualified(action.identity)}`);
+        // A rebuild's drop is followed by its create; only a true removal loses provenance.
+        if (!plan.actions.some((a) => a.action === 'create' && a.identity === action.identity)) {
+          remove.push(action.identity);
+        }
+        break;
+      }
+      case 'replace': {
+        const obj = srcById.get(action.identity);
+        if (!obj) throw new Error(`replace action for ${action.identity} has no source object`);
+        statements.push(...objectSql(obj).map(ensureOrReplace));
+        record.push(action.identity);
+        break;
+      }
+      case 'create': {
+        const obj = srcById.get(action.identity);
+        if (!obj) throw new Error(`create action for ${action.identity} has no source object`);
+        statements.push(...objectSql(obj));
+        record.push(action.identity);
+        break;
+      }
+      case 'refresh': {
+        statements.push(`REFRESH MATERIALIZED VIEW ${quoteQualified(action.identity)}`);
+        break;
+      }
+      case 'baseline': {
+        record.push(action.identity);
+        break;
+      }
+      case 'prune': {
+        remove.push(action.identity);
+        break;
+      }
+    }
+  }
+
+  return { statements, record, remove };
+}
+
+export function renderProvenanceUpsert(identity: string, srcHash: string, defHash: string): string {
+  return `INSERT INTO everystack.derived_provenance (identity, src_hash, def_hash, applied_at)
+VALUES (${escapeLiteral(identity)}, ${escapeLiteral(srcHash)}, ${escapeLiteral(defHash)}, now())
+ON CONFLICT (identity) DO UPDATE SET src_hash = EXCLUDED.src_hash, def_hash = EXCLUDED.def_hash, applied_at = now()`;
+}
+
+export function renderProvenanceDelete(identity: string): string {
+  return `DELETE FROM everystack.derived_provenance WHERE identity = ${escapeLiteral(identity)}`;
+}
+
+export interface SchemaLogEntry {
+  kind: string;
+  sql: string;
+  outcome: string;
+  actor?: string | null;
+  gitRef?: string | null;
+  planRef?: string | null;
+  durationMs?: number;
+}
+
+export function renderSchemaLogInsert(entry: SchemaLogEntry): string {
+  const duration = entry.durationMs === undefined ? 'NULL' : String(Math.round(entry.durationMs));
+  return `INSERT INTO everystack.schema_log (kind, actor, git_ref, plan_ref, sql, duration_ms, outcome)
+VALUES (${escapeLiteral(entry.kind)}, ${nullable(entry.actor)}, ${nullable(entry.gitRef)}, ${nullable(entry.planRef)}, ${escapeLiteral(entry.sql)}, ${duration}, ${escapeLiteral(entry.outcome)})`;
+}