@everystack/cli 0.3.13 → 0.3.15

package/src/cli/derived-source.ts

@@ -0,0 +1,373 @@
+/**
+ * derived-source — the source side of the derived-object reconciler.
+ *
+ * Parses an app's ordered db/sql files into structured derived objects
+ * (functions, views, materialized views), attaching the statements that ride
+ * with an object (its indexes, comments, grants) and hashing each object's
+ * normalized content. The reconciler diffs these hashes against the live
+ * catalog's provenance to decide what to create, replace, or drop.
+ *
+ * Everything here is pure and database-free. The lexer respects Postgres
+ * string syntax — single quotes with '' doubling, E-strings with backslash
+ * escapes, double-quoted identifiers, tagged dollar quoting, and nested block
+ * comments — because a matview body is exactly where naive splitting dies.
+ *
+ * Identity note: functions are identified by schema-qualified NAME, not full
+ * signature. Overloads are rare in app SQL layers; when they appear we warn
+ * rather than mis-model them (the honest limitation, revisit if a consumer
+ * actually needs overloads).
+ */
+
+import { createHash } from 'node:crypto';
+
+export type DerivedKind = 'function' | 'view' | 'materialized view';
+
+/** A statement that belongs to an object: its index, comment, or grant. */
+export interface Attachment {
+  kind: 'index' | 'comment' | 'grant';
+  sql: string;
+}
+
+export interface SourceObject {
+  kind: DerivedKind;
+  /** Schema name, defaulted to `public` when unqualified. */
+  schema: string;
+  name: string;
+  /** `schema.name` — the reconciler's join key against the live catalog. */
+  identity: string;
+  /** The CREATE statement as authored (no trailing semicolon). */
+  sql: string;
+  /** Indexes / comments / grants targeting this object, in source order. */
+  attachments: Attachment[];
+  /** sha256 of the normalized CREATE + attachments — comment/whitespace edits do not change it. */
+  hash: string;
+  /** Source file this object came from. */
+  file: string;
+  /** Position in the concatenated source — a valid dependency order by convention. */
+  seq: number;
+}
+
+export interface SourceFile {
+  file: string;
+  sql: string;
+}
+
+export interface ParsedSources {
+  objects: SourceObject[];
+  /** Statements the reconciler does not manage — surfaced, never silently dropped. */
+  warnings: string[];
+}
+
+// ---------------------------------------------------------------------------
+// Lexer.
+// ---------------------------------------------------------------------------
+
+type LexState =
+  | { in: 'normal' }
+  | { in: 'single'; escaped: boolean }
+  | { in: 'double' }
+  | { in: 'dollar'; tag: string }
+  | { in: 'line-comment' }
+  | { in: 'block-comment'; depth: number };
+
+/** Matches a dollar-quote opener at position i; returns the full tag (`$$`, `$body$`) or null. */
+function dollarTagAt(text: string, i: number): string | null {
+  if (text[i] !== '$') return null;
+  const m = /^\$[A-Za-z_][A-Za-z0-9_]*\$|^\$\$/.exec(text.slice(i, i + 64));
+  return m ? m[0] : null;
+}
+
+/**
+ * Walk the text, calling `emit` for every character with the state it is IN
+ * (comments/strings/normal). The shared core under both the statement splitter
+ * and the normalizer, so they can never disagree about where a string ends.
+ */
+function lex(
+  text: string,
+  emit: (ch: string, state: LexState['in'], index: number) => void,
+): void {
+  let state: LexState = { in: 'normal' };
+  let i = 0;
+  while (i < text.length) {
+    const ch = text[i];
+    const next = text[i + 1];
+
+    if (state.in === 'normal') {
+      if (ch === '-' && next === '-') {
+        state = { in: 'line-comment' };
+        emit(ch, 'line-comment', i); i++;
+        emit(next, 'line-comment', i); i++;
+        continue;
+      }
+      if (ch === '/' && next === '*') {
+        state = { in: 'block-comment', depth: 1 };
+        emit(ch, 'block-comment', i); i++;
+        emit(next, 'block-comment', i); i++;
+        continue;
+      }
+      const tag = dollarTagAt(text, i);
+      if (tag) {
+        state = { in: 'dollar', tag };
+        for (const c of tag) { emit(c, 'dollar', i); i++; }
+        continue;
+      }
+      if (ch === "'") {
+        // E'...' / e'...' strings give backslash escape semantics.
+        const prev = text[i - 1];
+        state = { in: 'single', escaped: prev === 'E' || prev === 'e' };
+        emit(ch, 'single', i); i++;
+        continue;
+      }
+      if (ch === '"') {
+        state = { in: 'double' };
+        emit(ch, 'double', i); i++;
+        continue;
+      }
+      emit(ch, 'normal', i); i++;
+      continue;
+    }
+
+    if (state.in === 'line-comment') {
+      emit(ch, 'line-comment', i); i++;
+      if (ch === '\n') state = { in: 'normal' };
+      continue;
+    }
+
+    if (state.in === 'block-comment') {
+      if (ch === '/' && next === '*') {
+        state = { in: 'block-comment', depth: state.depth + 1 };
+        emit(ch, 'block-comment', i); i++;
+        emit(next, 'block-comment', i); i++;
+        continue;
+      }
+      if (ch === '*' && next === '/') {
+        const newDepth: number = state.depth - 1;
+        emit(ch, 'block-comment', i); i++;
+        emit(next, 'block-comment', i); i++;
+        state = newDepth === 0 ? { in: 'normal' } : { in: 'block-comment', depth: newDepth };
+        continue;
+      }
+      emit(ch, 'block-comment', i); i++;
+      continue;
+    }
+
+    if (state.in === 'single') {
+      if (state.escaped && ch === '\\') {
+        emit(ch, 'single', i); i++;
+        if (i < text.length) { emit(text[i], 'single', i); i++; }
+        continue;
+      }
+      if (ch === "'" && next === "'") {
+        emit(ch, 'single', i); i++;
+        emit(next, 'single', i); i++;
+        continue;
+      }
+      emit(ch, 'single', i); i++;
+      if (ch === "'") state = { in: 'normal' };
+      continue;
+    }
+
+    if (state.in === 'double') {
+      if (ch === '"' && next === '"') {
+        emit(ch, 'double', i); i++;
+        emit(next, 'double', i); i++;
+        continue;
+      }
+      emit(ch, 'double', i); i++;
+      if (ch === '"') state = { in: 'normal' };
+      continue;
+    }
+
+    // dollar
+    if (ch === '$') {
+      const tag = dollarTagAt(text, i);
+      if (tag === state.tag) {
+        for (const c of tag) { emit(c, 'dollar', i); i++; }
+        state = { in: 'normal' };
+        continue;
+      }
+    }
+    emit(ch, 'dollar', i); i++;
+  }
+}
+
+/** Split SQL text into statements on top-level semicolons. Statements come back trimmed. */
+export function splitSqlStatements(text: string): string[] {
+  const statements: string[] = [];
+  let current = '';
+  lex(text, (ch, state) => {
+    if (state === 'normal' && ch === ';') {
+      const s = current.trim();
+      if (s) statements.push(s);
+      current = '';
+      return;
+    }
+    current += ch;
+  });
+  const last = current.trim();
+  if (last) statements.push(last);
+  return statements;
+}
+
+const WORDISH = /[A-Za-z0-9_$]/;
+
+/**
+ * Normalize SQL for content hashing: comments stripped; outside strings,
+ * whitespace survives only as a single space between word tokens (`SELECT a`),
+ * never around punctuation (`a,b` ≡ `a, b`). A renumbered header comment or a
+ * reflowed SELECT hashes identically; any change to the actual SQL does not.
+ * The output is a canonical hashing form, not necessarily runnable SQL.
+ */
+export function normalizeSql(sql: string): string {
+  let out = '';
+  let pendingSpace = false;
+  lex(sql, (ch, state) => {
+    if (state === 'line-comment' || state === 'block-comment') return;
+    if (state === 'normal' && /\s/.test(ch)) {
+      pendingSpace = true;
+      return;
+    }
+    if (pendingSpace) {
+      if (out.length > 0 && WORDISH.test(out[out.length - 1]) && WORDISH.test(ch)) out += ' ';
+      pendingSpace = false;
+    }
+    out += ch;
+  });
+  return out.trim();
+}
+
+// ---------------------------------------------------------------------------
+// Classification.
+// ---------------------------------------------------------------------------
+
+/** Strip quotes from an identifier and split an optionally schema-qualified name. */
+function parseQualified(raw: string): { schema: string; name: string } {
+  const parts: string[] = [];
+  let current = '';
+  let quoted = false;
+  for (let i = 0; i < raw.length; i++) {
+    const ch = raw[i];
+    if (ch === '"') {
+      if (quoted && raw[i + 1] === '"') { current += '"'; i++; continue; }
+      quoted = !quoted;
+      continue;
+    }
+    if (ch === '.' && !quoted) { parts.push(current); current = ''; continue; }
+    current += ch;
+  }
+  parts.push(current);
+  if (parts.length >= 2) return { schema: parts[0], name: parts.slice(1).join('.') };
+  return { schema: 'public', name: parts[0] };
+}
+
+const FUNCTION_RE = /^CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+((?:"[^"]*"|[^\s(])+)\s*\(/i;
+const VIEW_RE = /^CREATE\s+(?:OR\s+REPLACE\s+)?VIEW\s+((?:"[^"]*"|[^\s(])+)/i;
+const MATVIEW_RE = /^CREATE\s+MATERIALIZED\s+VIEW\s+(?:IF\s+NOT\s+EXISTS\s+)?((?:"[^"]*"|[^\s(])+)/i;
+const INDEX_RE = /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?:(?:"[^"]*"|[^\s(])+\s+)?ON\s+(?:ONLY\s+)?((?:"[^"]*"|[^\s(])+)/i;
+const COMMENT_RE = /^COMMENT\s+ON\s+(MATERIALIZED\s+VIEW|VIEW|FUNCTION|COLUMN)\s+((?:"[^"]*"|[^\s(,])+)/i;
+const GRANT_RE = /^GRANT\s+[\s\S]*?\bON\s+(?:TABLE\s+)?((?:"[^"]*"|[^\s(,])+)/i;
+
+/**
+ * The first non-comment text of a statement, whitespace collapsed but token
+ * boundaries preserved (unlike the aggressive hash form — `VIEW "MyView"` must
+ * keep its space for the classification regexes).
+ */
+function statementHead(statement: string): string {
+  let out = '';
+  lex(statement, (ch, state) => {
+    if (state === 'line-comment' || state === 'block-comment') return;
+    if (state === 'normal' && /\s/.test(ch)) {
+      if (out.length > 0 && !out.endsWith(' ')) out += ' ';
+      return;
+    }
+    out += ch;
+  });
+  return out.trim().slice(0, 400);
+}
+
+/**
+ * Parse ordered source files into derived objects. Attachments (indexes,
+ * comments, grants) join the object they target; statements that are neither a
+ * derived object nor an attachment become warnings — the reconciler manages
+ * the derived layer only, and it says so out loud.
+ */
+export function parseSqlSources(files: SourceFile[]): ParsedSources {
+  const objects: SourceObject[] = [];
+  const warnings: string[] = [];
+  const byIdentity = new Map<string, SourceObject>();
+  let seq = 0;
+
+  const identityOf = (raw: string): { schema: string; name: string; identity: string } => {
+    const { schema, name } = parseQualified(raw);
+    return { schema, name, identity: `${schema}.${name}` };
+  };
+
+  for (const { file, sql } of files) {
+    for (const statement of splitSqlStatements(sql)) {
+      const head = statementHead(statement);
+
+      const asObject = (kind: DerivedKind, rawName: string): void => {
+        const { schema, name, identity } = identityOf(rawName);
+        if (byIdentity.has(identity)) {
+          warnings.push(
+            `${file}: duplicate definition for ${identity} — function overloads are not modeled; ` +
+            `the reconciler tracks one definition per name (last one wins).`,
+          );
+        }
+        const obj: SourceObject = {
+          kind, schema, name, identity,
+          sql: statement, attachments: [], hash: '', file, seq: seq++,
+        };
+        byIdentity.set(identity, obj);
+        objects.push(obj);
+      };
+
+      const attach = (kind: Attachment['kind'], rawTarget: string, stripColumn = false): boolean => {
+        let { identity } = identityOf(rawTarget);
+        if (stripColumn) {
+          // COMMENT ON COLUMN schema.obj.col — the owning object is one segment up.
+          const parts = identity.split('.');
+          identity = parts.length > 2 ? parts.slice(0, -1).join('.') : `public.${parts[0]}`;
+        }
+        const target = byIdentity.get(identity);
+        if (!target) return false;
+        target.attachments.push({ kind, sql: statement });
+        return true;
+      };
+
+      let m: RegExpExecArray | null;
+      if ((m = FUNCTION_RE.exec(head))) { asObject('function', m[1]); continue; }
+      if ((m = MATVIEW_RE.exec(head))) { asObject('materialized view', m[1]); continue; }
+      if ((m = VIEW_RE.exec(head))) { asObject('view', m[1]); continue; }
+
+      if ((m = INDEX_RE.exec(head))) {
+        if (!attach('index', m[1])) {
+          warnings.push(`${file}: CREATE INDEX targets ${m[1]}, which is not a derived object in these sources — not managed here (base-table indexes belong to the model layer).`);
+        }
+        continue;
+      }
+      if ((m = COMMENT_RE.exec(head))) {
+        const isColumn = /^COLUMN$/i.test(m[1].replace(/\s+/g, ' ').split(' ')[0]) || /^COLUMN/i.test(m[1]);
+        if (!attach('comment', m[2], isColumn)) {
+          warnings.push(`${file}: COMMENT ON targets ${m[2]}, which is not a derived object in these sources.`);
+        }
+        continue;
+      }
+      if ((m = GRANT_RE.exec(head))) {
+        if (!attach('grant', m[1])) {
+          warnings.push(`${file}: GRANT targets ${m[1]}, which is not a derived object in these sources.`);
+        }
+        continue;
+      }
+
+      warnings.push(`${file}: unmanaged statement (${head.slice(0, 60)}…) — the reconciler manages functions, views, and materialized views; move this to the appropriate layer.`);
+    }
+  }
+
+  for (const obj of objects) {
+    const content = [normalizeSql(obj.sql), ...obj.attachments.map((a) => normalizeSql(a.sql))].join('\n');
+    obj.hash = createHash('sha256').update(content).digest('hex');
+  }
+
+  return { objects, warnings };
+}

package/src/cli/index.ts

@@ -11,6 +11,7 @@ import { dbMigrateCommand, dbSeedCommand, dbResetCommand, dbPsqlCommand, dbDocto
 import { dbAuthzPullCommand, dbAuthzDiffCommand, dbAuthzTestCommand, dbAuthzOwnerCommand, dbAuthzReportCommand } from './commands/db-authz.js';
 import { dbGenerateCommand } from './commands/db-generate.js';
 import { dbPullCommand } from './commands/db-pull.js';
+import { dbReconcileCommand } from './commands/db-reconcile.js';
 import { dbSnapshotCommand, dbSnapshotsCommand } from './commands/db-snapshot.js';
 import { dbBackupProbeCommand, dbBackupCommand, dbBackupsCommand, dbRestoreCommand, dbBackupDownloadCommand } from './commands/db-backup.js';
 import { consoleCommand } from './commands/console.js';
@@ -172,6 +173,9 @@ async function main() {
     case 'db:pull':
       await dbPullCommand(flags);
       break;
+    case 'db:reconcile':
+      await dbReconcileCommand(flags);
+      break;
     case 'db:authz:pull':
       await dbAuthzPullCommand(flags);
       break;
@@ -291,13 +295,14 @@ Usage:
   everystack db:backups        [--stage <name>]  List logical backups (id, size, created)
   everystack db:restore        --from <id> [--stage <name>] --confirm  Restore a backup INTO the stage's DB (DESTRUCTIVE)
   everystack db:backup:download <id> [--stage <name>]  Presigned URL to download a backup's dump (valid 1h)
-  everystack db:generate       [--stage <name> | --database-url <url>] [--name <label>] [--models models/index.ts] [--allow-drops]  Diff models vs the live DB → write the next migration (never touches the DB; DROPs held back unless --allow-drops)
+  everystack db:generate       [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--allow-drops]  Diff models vs the live DB → write the next migration (never touches the DB; DROPs held back unless --allow-drops)
   everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise
                                Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
+  everystack db:reconcile      [--sql-dir db/sql] [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--overwrite-drift] [--json]  Reconcile the derived layer (functions/views/matviews) against db/sql sources: plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (direct connection only) and records provenance + schema_log. Hand-edits are drift (never overwritten silently); first contact with existing objects wants --baseline, once.
   everystack db:authz:pull     [--stage <name>] [--dir authz]  Introspect live authz (rls/grants/policies/secdef) → reviewable contract files
   everystack db:authz:diff     [--stage <name>] [--dir authz]  Validate the live DB against the committed contract (non-zero exit on drift)
   everystack db:authz:test     [--stage <name>] [--dir authz]  Red-team enforcement: SET ROLE + attempt per role/table/command (non-zero exit on a hole)
-  everystack db:authz:owner    [--stage <name>] [--models models/index.ts]  Red-team owner isolation: two JWT identities per owner-scoped table — catches IDOR (non-zero exit on a leak)
+  everystack db:authz:owner    [--stage <name>] [--models db/models/index.ts]  Red-team owner isolation: two JWT identities per owner-scoped table — catches IDOR (non-zero exit on a leak)
   everystack db:authz:report   [--dir authz]  Render the committed contract as a human-readable authorization review (no DB)
   everystack console           --stage <name> [--sandbox]  Interactive REPL on deployed Lambda
   everystack status            [--stage <name>] [--hours <n>]  Platform health: CDN, Lambda, rollup summary

package/src/cli/models-path.ts

@@ -0,0 +1,25 @@
+/**
+ * models-path — where the Models barrel lives.
+ *
+ * One home for schema: `db/models/` (state) beside `db/sql/` (compute). The
+ * CLI looks there first, falling back to the legacy top-level `models/` so
+ * existing consumers keep working without a flag. An explicit `--models`
+ * always wins. When neither home exists, the NEW home is returned so error
+ * messages point people at the current convention, not the legacy one.
+ */
+
+import { existsSync } from 'node:fs';
+
+/** Candidate barrels, in preference order. */
+export const MODELS_HOMES = ['db/models/index.ts', 'models/index.ts'] as const;
+
+export function resolveModelsPath(
+  flag?: string,
+  exists: (p: string) => boolean = existsSync,
+): string {
+  if (flag) return flag;
+  for (const home of MODELS_HOMES) {
+    if (exists(home)) return home;
+  }
+  return MODELS_HOMES[0];
+}

package/src/cli/runbook/detect.ts

@@ -102,10 +102,12 @@ export function detectProjectReality(root: string): ProjectReality {
       ? 'V2'
       : 'V1';
 
-  const modelsDir = path.join(abs, 'models');
+  // One home for schema: db/models is the current convention, top-level models the legacy one.
+  const modelsHome = isDir(path.join(abs, 'db', 'models')) ? 'db/models' : 'models';
+  const modelsDir = path.join(abs, modelsHome);
   const modelFiles = listTs(modelsDir).filter((f) => f !== 'index.ts');
   const models = {
-    dir: isDir(modelsDir) ? 'models' : null,
+    dir: isDir(modelsDir) ? modelsHome : null,
     files: modelFiles,
     hasIndex: isFile(path.join(modelsDir, 'index.ts')),
   };