@everystack/cli 0.3.31 → 0.4.0

package/src/cli/derived-source.ts

@@ -1,26 +1,32 @@
 /**
  * derived-source — the source side of the derived-object reconciler.
  *
- * Parses an app's ordered db/sql files into structured derived objects
- * (functions, views, materialized views), attaching the statements that ride
- * with an object (its indexes, comments, grants) and hashing each object's
- * normalized content. The reconciler diffs these hashes against the live
- * catalog's provenance to decide what to create, replace, or drop.
+ * The `SourceObject` shape every derived declaration compiles to (identity,
+ * content hash, attachments), plus the SQL infrastructure shared across the
+ * reconciler: the Postgres-aware lexer, statement splitting, hash
+ * normalization, and qualified-name parsing.
+ *
+ * History note: this file once parsed an app's raw `db/sql` files into these
+ * shapes. That parser is retired (B7, read-model-everywhere) — descriptors
+ * (defineView / defineMaterializedView / defineFunction / defineSql, compiled
+ * by derived-compile.ts) are the single home. The content-hash formula is
+ * unchanged, so provenance recorded from db/sql-era objects still matches the
+ * descriptor-compiled hash of the same SQL — the migration reconciles as a
+ * no-op, never a rebuild.
  *
  * Everything here is pure and database-free. The lexer respects Postgres
  * string syntax — single quotes with '' doubling, E-strings with backslash
  * escapes, double-quoted identifiers, tagged dollar quoting, and nested block
  * comments — because a matview body is exactly where naive splitting dies.
- *
- * Identity note: functions are identified by schema-qualified NAME, not full
- * signature. Overloads are rare in app SQL layers; when they appear we warn
- * rather than mis-model them (the honest limitation, revisit if a consumer
- * actually needs overloads).
  */
 
 import { createHash } from 'node:crypto';
 
-export type DerivedKind = 'function' | 'view' | 'materialized view';
+/**
+ * `'trigger'` and `'sql'` carry the extra fields the reconciler edges need (a trigger's
+ * table for `DROP TRIGGER … ON`, a defineSql object's explicit drop).
+ */
+export type DerivedKind = 'function' | 'view' | 'materialized view' | 'trigger' | 'sql';
 
 /** A statement that belongs to an object: its index, comment, or grant. */
 export interface Attachment {
@@ -33,7 +39,8 @@ export interface SourceObject {
   /** Schema name, defaulted to `public` when unqualified. */
   schema: string;
   name: string;
-  /** `schema.name` — the reconciler's join key against the live catalog. */
+  /** `schema.name` — the reconciler's join key against the live catalog.
+   *  (Triggers: `schema.table.name` — compound identity, joined against pg_trigger.) */
   identity: string;
   /** The CREATE statement as authored (no trailing semicolon). */
   sql: string;
@@ -41,17 +48,46 @@ export interface SourceObject {
   attachments: Attachment[];
   /** sha256 of the normalized CREATE + attachments — comment/whitespace edits do not change it. */
   hash: string;
-  /** Source file this object came from. */
+  /** Source file this object came from (descriptors: the declared-models marker). */
   file: string;
   /** Position in the concatenated source — a valid dependency order by convention. */
   seq: number;
+  /** kind 'sql' only: the declared PostgreSQL object kind (`'aggregate'`, …). */
+  objectKind?: string;
+  /** kind 'sql' only: the explicit DROP for kinds whose drop isn't derivable from the name. */
+  drop?: string;
+  /** kind 'trigger' only: the (possibly schema-qualified) table/view the trigger rides. */
+  table?: string;
+  /**
+   * Views/matviews: every declared `dependsOn` ref as an identity (models AND derived),
+   * sorted. B4's dependency drift verifies it against the live catalog's actual edges —
+   * "declared, then verified". Absent on kinds whose live edges pg_depend can't see.
+   */
+  declaredDeps?: string[];
+}
+
+/** The provenance marker descriptor-compiled objects carry in `file` — the compiler
+ *  ALWAYS declares authz (a private relation is declared-empty, not undeclared), so
+ *  B4's grant drift checks every object. */
+export const DECLARED_SOURCE_FILE = 'db/models (declared)';
+
+/** The one content-hash formula. Unchanged since the db/sql era on purpose: provenance
+ *  recorded from parser-era objects still matches the descriptor-compiled hash of the
+ *  same SQL, so the db/sql → descriptor migration reconciles as a no-op. */
+export function hashSourceContent(createSql: string, attachments: Attachment[]): string {
+  const content = [normalizeSql(createSql), ...attachments.map((a) => normalizeSql(a.sql))].join('\n');
+  return createHash('sha256').update(content).digest('hex');
 }
 
+/** A raw file the generic dir readers return (db/backfills, seed lanes). */
 export interface SourceFile {
   file: string;
   sql: string;
 }
 
+/** The reconciler's source-stream input: the declared objects plus any compile-time
+ *  warnings to surface with the plan. (Named for its parser-era producer; the only
+ *  producer now is the descriptor compiler.) */
 export interface ParsedSources {
   objects: SourceObject[];
   /** Statements the reconciler does not manage — surfaced, never silently dropped. */
@@ -242,7 +278,7 @@ export function normalizeSql(sql: string): string {
 // ---------------------------------------------------------------------------
 
 /** Strip quotes from an identifier and split an optionally schema-qualified name. */
-function parseQualified(raw: string): { schema: string; name: string } {
+export function parseQualified(raw: string): { schema: string; name: string } {
   const parts: string[] = [];
   let current = '';
   let quoted = false;
@@ -261,114 +297,6 @@ function parseQualified(raw: string): { schema: string; name: string } {
   return { schema: 'public', name: parts[0] };
 }
 
-const FUNCTION_RE = /^CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+((?:"[^"]*"|[^\s(])+)\s*\(/i;
-const VIEW_RE = /^CREATE\s+(?:OR\s+REPLACE\s+)?VIEW\s+((?:"[^"]*"|[^\s(])+)/i;
-const MATVIEW_RE = /^CREATE\s+MATERIALIZED\s+VIEW\s+(?:IF\s+NOT\s+EXISTS\s+)?((?:"[^"]*"|[^\s(])+)/i;
-const INDEX_RE = /^CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?:(?:"[^"]*"|[^\s(])+\s+)?ON\s+(?:ONLY\s+)?((?:"[^"]*"|[^\s(])+)/i;
-const COMMENT_RE = /^COMMENT\s+ON\s+(MATERIALIZED\s+VIEW|VIEW|FUNCTION|COLUMN)\s+((?:"[^"]*"|[^\s(,])+)/i;
-const GRANT_RE = /^GRANT\s+[\s\S]*?\bON\s+(?:TABLE\s+)?((?:"[^"]*"|[^\s(,])+)/i;
-
-/**
- * The first non-comment text of a statement, whitespace collapsed but token
- * boundaries preserved (unlike the aggressive hash form — `VIEW "MyView"` must
- * keep its space for the classification regexes).
- */
-function statementHead(statement: string): string {
-  let out = '';
-  lex(statement, (ch, state) => {
-    if (state === 'line-comment' || state === 'block-comment') return;
-    if (state === 'normal' && /\s/.test(ch)) {
-      if (out.length > 0 && !out.endsWith(' ')) out += ' ';
-      return;
-    }
-    out += ch;
-  });
-  return out.trim().slice(0, 400);
-}
-
-/**
- * Parse ordered source files into derived objects. Attachments (indexes,
- * comments, grants) join the object they target; statements that are neither a
- * derived object nor an attachment become warnings — the reconciler manages
- * the derived layer only, and it says so out loud.
- */
-export function parseSqlSources(files: SourceFile[]): ParsedSources {
-  const objects: SourceObject[] = [];
-  const warnings: string[] = [];
-  const byIdentity = new Map<string, SourceObject>();
-  let seq = 0;
-
-  const identityOf = (raw: string): { schema: string; name: string; identity: string } => {
-    const { schema, name } = parseQualified(raw);
-    return { schema, name, identity: `${schema}.${name}` };
-  };
-
-  for (const { file, sql } of files) {
-    for (const statement of splitSqlStatements(sql)) {
-      const head = statementHead(statement);
-
-      const asObject = (kind: DerivedKind, rawName: string): void => {
-        const { schema, name, identity } = identityOf(rawName);
-        if (byIdentity.has(identity)) {
-          warnings.push(
-            `${file}: duplicate definition for ${identity} — function overloads are not modeled; ` +
-            `the reconciler tracks one definition per name (last one wins).`,
-          );
-        }
-        const obj: SourceObject = {
-          kind, schema, name, identity,
-          sql: statement, attachments: [], hash: '', file, seq: seq++,
-        };
-        byIdentity.set(identity, obj);
-        objects.push(obj);
-      };
-
-      const attach = (kind: Attachment['kind'], rawTarget: string, stripColumn = false): boolean => {
-        let { identity } = identityOf(rawTarget);
-        if (stripColumn) {
-          // COMMENT ON COLUMN schema.obj.col — the owning object is one segment up.
-          const parts = identity.split('.');
-          identity = parts.length > 2 ? parts.slice(0, -1).join('.') : `public.${parts[0]}`;
-        }
-        const target = byIdentity.get(identity);
-        if (!target) return false;
-        target.attachments.push({ kind, sql: statement });
-        return true;
-      };
-
-      let m: RegExpExecArray | null;
-      if ((m = FUNCTION_RE.exec(head))) { asObject('function', m[1]); continue; }
-      if ((m = MATVIEW_RE.exec(head))) { asObject('materialized view', m[1]); continue; }
-      if ((m = VIEW_RE.exec(head))) { asObject('view', m[1]); continue; }
-
-      if ((m = INDEX_RE.exec(head))) {
-        if (!attach('index', m[1])) {
-          warnings.push(`${file}: CREATE INDEX targets ${m[1]}, which is not a derived object in these sources — not managed here (base-table indexes belong to the model layer).`);
-        }
-        continue;
-      }
-      if ((m = COMMENT_RE.exec(head))) {
-        const isColumn = /^COLUMN$/i.test(m[1].replace(/\s+/g, ' ').split(' ')[0]) || /^COLUMN/i.test(m[1]);
-        if (!attach('comment', m[2], isColumn)) {
-          warnings.push(`${file}: COMMENT ON targets ${m[2]}, which is not a derived object in these sources.`);
-        }
-        continue;
-      }
-      if ((m = GRANT_RE.exec(head))) {
-        if (!attach('grant', m[1])) {
-          warnings.push(`${file}: GRANT targets ${m[1]}, which is not a derived object in these sources.`);
-        }
-        continue;
-      }
-
-      warnings.push(`${file}: unmanaged statement (${head.slice(0, 60)}…) — the reconciler manages functions, views, and materialized views; move this to the appropriate layer.`);
-    }
-  }
-
-  for (const obj of objects) {
-    const content = [normalizeSql(obj.sql), ...obj.attachments.map((a) => normalizeSql(a.sql))].join('\n');
-    obj.hash = createHash('sha256').update(content).digest('hex');
-  }
-
-  return { objects, warnings };
-}
+// (The db/sql statement classifier and parseSqlSources lived here until B7 —
+// descriptors are the single home; declared-derived.ts's retiredSqlDir() is the
+// loud tombstone for checkouts still carrying the directory.)

package/src/cli/git-descent.ts

@@ -188,10 +188,23 @@ export async function verifyDescent(
     for (const candidate of candidates) {
       const dest = path.join(materializeRoot, candidate.tree);
       let predicted: string;
-      try {
+      // A swallowed error here skips the candidate — and if it was the DECLARING tree, the
+      // verdict silently falls through to 'drift'. Real compile failures are deterministic;
+      // a transient materialize/import failure (fd pressure, module-loader contention under
+      // a parallel test run or a busy CI box) is not — so a failed candidate gets exactly
+      // one retry, from a clean materialization, before it is recorded as a compile failure.
+      const attempt = async (): Promise<string> => {
         if (!fs.existsSync(dest)) materializeTree(candidate.tree, dest, repoRoot);
         const models = await loader(path.join(dest, barrel));
-        predicted = predictLiveFingerprint(models, live.snapshot, live.contract, { schema: opts.schema });
+        return predictLiveFingerprint(models, live.snapshot, live.contract, { schema: opts.schema });
+      };
+      try {
+        try {
+          predicted = await attempt();
+        } catch {
+          fs.rmSync(dest, { recursive: true, force: true });
+          predicted = await attempt();
+        }
       } catch (err: any) {
         compileFailures.push(`${short(candidate.tree)} (at ${short(candidate.commits[0])}): ${err.message}`);
         continue;

package/src/cli/index.ts

@@ -353,21 +353,21 @@ Usage:
   everystack db:backups        [--stage <name>]  List logical backups (id, size, created)
   everystack db:restore        --from <id> [--stage <name>] --confirm  Restore a backup INTO the stage's DB (DESTRUCTIVE)
   everystack db:backup:download <id> [--stage <name>]  Presigned URL to download a backup's dump (valid 1h)
-  everystack db:generate       [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--schema-out db/schema.generated.ts] [--allow-drops] [--apply]  Diff models vs the live DB → next migration file, or with --apply execute it directly (one transaction, schema_log recorded, verified by re-diff — no drizzle folder needed; direct connection only; DROPs held back unless --allow-drops)
-  everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise
+  everystack db:generate       [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--schema-out db/schema.generated.ts] [--allow-drops] [--apply] [--dry-run]  Diff models vs the live DB → next migration file, or with --apply execute it directly (one transaction, schema_log recorded, verified by re-diff — no drizzle folder needed; direct connection only; DROPs held back unless --allow-drops). --dry-run prints the edge and writes NOTHING (no migration, no journal entry, no schema refresh) — the preview verb; db:diff computes a models-vs-models edge with no database at all. The resolved --schema-out is recorded in the migration journal: later flag-less runs reuse it (flag > recorded > default), a differing flag updates the record and says so
+  everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--abilities public-read]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. Every model scaffolds its authz decision as comments (db:check fails until authored); --abilities public-read stamps the common stanza (public read, admin write) uncommented — explicit generated code, never a runtime default
                                Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
   everystack db:fingerprint    [--stage <name> | --database-url <url>] [--models <barrel>] [--json]  Content-address the live base schema (tables+constraints+authz) and compare against the models — MATCH/MISMATCH (exit 1), plus the unfingerprinted-objects report
-  everystack db:reconcile      [--sql-dir db/sql] [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--rebuild] [--overwrite-drift] [--json]  Reconcile the derived layer (functions/views/matviews) against db/sql sources: plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (direct connection only, atomic — DDL + provenance in one transaction) and records provenance + schema_log. Hand-edits are drift (never overwritten silently). First contact with existing objects: --baseline TRUSTS live == source (records provenance, verifies nothing), --rebuild GUARANTEES it (drop+create from source). They are mutually exclusive.
-  everystack db:sync           [--database-url <url>] [--models <barrel>] [--sql-dir db/sql] [--schema-out <file.ts>] [--allow-drops] [--overwrite-drift] [--baseline] [--json]  Make the database match your checkout — one verb, both layers: apply the state diff (tables+authz, one transaction, verified by re-diff), reconcile the derived layer against db/sql, report the resulting fingerprint vs the models' declared one. Dev databases only (direct connection required); DROPs held back unless --allow-drops; derived drift refuses unless --overwrite-drift; exit 1 when not converged
+  everystack db:reconcile      [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--rebuild] [--overwrite-drift] [--json]  Reconcile the derived layer (functions/views/matviews/triggers) against the DECLARED descriptors (defineView/defineMaterializedView/defineFunction/defineSql/trigger() on models, from the barrel) — the single home (db/sql is retired; leftover .sql files fail with the migration path): plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (direct connection only, atomic — DDL + provenance in one transaction) and records provenance + schema_log. Hand-edits are drift (never overwritten silently). First contact with existing objects: --baseline TRUSTS live == source (records provenance, verifies nothing), --rebuild GUARANTEES it (drop+create from source). They are mutually exclusive.
+  everystack db:sync           [--database-url <url>] [--models <barrel>] [--schema-out <file.ts>] [--allow-drops] [--overwrite-drift] [--baseline] [--json]  Make the database match your checkout — one verb, both layers: apply the state diff (tables+authz, one transaction, verified by re-diff), reconcile the derived layer against the declared descriptors, report the resulting fingerprint vs the models' declared one. Dev databases only (direct connection required); DROPs held back unless --allow-drops; derived drift refuses unless --overwrite-drift; exit 1 when not converged
   everystack db:diff           --from-models <barrel> [--to-models db/models/index.ts] [--allow-drops] [--check] [--json]  The state edge between two declared states, NO database: the SQL db:generate would produce, computed purely — CI plan previews (--check exits 1 on a non-empty edge) and computed rollbacks (swap the flags)
   everystack db:plan           [--stage <name> | --database-url <url>] [--models <barrel>] [--allow-drops] [--out db.plan.json | --out -]  Mint a verified edge against a target: asks the TARGET its fingerprint, diffs the models, writes ONE reviewable plan (edge + both endpoint fingerprints). Held drops refuse the mint (--allow-drops carries destruction explicitly). Read-only — works via the ops Lambda; plans are ephemeral, never committed
   everystack db:apply          --plan <file.plan.json> [--database-url <url>] [--stage <name>] [--models <barrel>] [--confirm] [--snapshot-ref <ref>] [--force-descent <snapshot-ref> --confirm]  Run a reviewed plan: verify the target is EXACTLY where the plan started (live fingerprint == plan.from, else refuse — the concurrency lock), verify the checkout DESCENDS from the commit declaring the target's state (the fast-forward rule, else refuse — "rebase first"), and for DESTRUCTIVE plans require --confirm always + a snapshot (automatic via db:backup with --stage, else --snapshot-ref) + the stage's approver set when declared (STS identity-verified). Every refusal is recorded in schema_log. Apply as one transaction (plan_ref stamped), verify it landed exactly on plan.to; idempotent when already there; direct connection required
-  everystack db:check          [--models <barrel>] [--sql-dir db/sql] [--schema-out <file.ts>] [--database-url <url>] [--json]  The CI gate, per PR: the merged declared state must COMPOSE (models load, no duplicate tables, db/sql parses), every exposed RLS-enabled table must declare a read path (no force-RLS-with-no-read landmine that goes dark on the superuser drop), and generated artifacts must MATCH regeneration byte-for-byte; with a scratch PostgreSQL it builds the state from scratch on an ephemeral database (created + dropped) and requires fingerprint MATCH. Exit 1 on any failure; never touches a real target
+  everystack db:check          [--models <barrel>] [--schema-out <file.ts>] [--database-url <url>] [--json]  The CI gate, per PR: the merged declared state must COMPOSE (models load, no duplicate tables, descriptors compile), every exposed RLS-enabled table must declare a read path (no force-RLS-with-no-read landmine that goes dark on the superuser drop), and generated artifacts must MATCH regeneration byte-for-byte; with a scratch PostgreSQL it builds the state from scratch on an ephemeral database (created + dropped) and requires fingerprint MATCH. Exit 1 on any failure; never touches a real target
   everystack db:approvers      --stage <name> [--set "cto,arn:..."] [--remove]  Declare who can DESTROY: the stage's destructive-approver set (SSM parameter, admin-writable). Destructive db:apply runs are then identity-verified (STS) against it; --set '' disables destructive applies; --remove returns the stage to ceremony-only
   everystack db:backfill       [--database-url <url>] [--dir db/backfills] [--apply] [--mark-applied <file.sql>] [--json]  One-shot data jobs in their own lane: plan shows applied (by CONTENT identity — renames/comment edits are no-ops) / pending (in order, unbounded-pass advisories) / blocked (a name that already ran in a different form — one-shot jobs are immutable). --apply runs each pending job as its own transaction, recorded in everystack.backfill_log (a failure rolls back alone, is recorded, stops the run); --mark-applied records without running. Never runs as a schema side effect; direct connection required
   everystack pipeline:run      [--stage <name> | --database-url <url>] [--rebuild | --curate] [--only <substr> | --from <id> | --to <id>] [--resume [--run-id <id>]] [--dry-run] [--continue-on-error] [--json]  Reproduce data from committed source: run the pipeline's stages in topological order, each idempotent and inside its own transaction, recorded in everystack.pipeline_log. Lanes: --rebuild (automated+frozen) / --curate (curated) / neither (all in order). --resume picks up the latest run, skipping applied stages. With --stage the orchestrator runs credential-free IN the ops Lambda (no URL held; heavy stages refused — run those local); --database-url is local-direct and unbounded
   everystack pipeline:list     [--rebuild | --curate] [--only <substr> | --from <id> | --to <id>]  Show the pipeline's stages in run order (topological) with their lane and dependency edges — no execution, no database
-  everystack db:template:refresh [--database-url <url>] [--models <barrel>] [--sql-dir db/sql] [--seed "<cmd>"] [--no-seed]  (Re)build the dev template <base>_tpl FROM THE DECLARED STATE (both layers, fingerprint MATCH bar) + seed-as-code (the app's db:seed script, run with DATABASE_URL pointed at the template; --seed overrides). All or nothing: a failed build/seed leaves NO template. Never a data copy
+  everystack db:template:refresh [--database-url <url>] [--models <barrel>] [--seed "<cmd>"] [--no-seed]  (Re)build the dev template <base>_tpl FROM THE DECLARED STATE (both layers, fingerprint MATCH bar) + seed-as-code (the app's db:seed script, run with DATABASE_URL pointed at the template; --seed overrides). All or nothing: a failed build/seed leaves NO template. Never a data copy
   everystack db:branch         [--list | --drop --confirm | --prune --confirm] [--database-url <url>]  Mint (or find) the current git branch's database from the template (CREATE DATABASE … TEMPLATE — schema, authz, derived layer, and seed rows inherited), print its DATABASE_URL, then db:sync evolves it with the checkout. --list maps every branch DB to its branch; --prune drops the ones whose branch is gone (never guesses: unknown mappings are kept)
   everystack db:fork           --from-stage <src> --stage <target> --confirm [--backup <id>]  Fork one DEPLOYED stage's database into another: back up the source (or reuse --backup <id>), presign the dump (the operator's credentials ARE the cross-stage authorization; expires in 1h), restore into the target via its ops Lambda. Production is never a target (that's db:restore); forking FROM production warns about PII; the branch's schema edge then lands via db:plan → db:apply (descent composes). Teardown: sst remove --stage <target>
   everystack db:authz:pull     [--stage <name>] [--dir authz]  Introspect live authz (rls/grants/policies/secdef) → reviewable contract files

package/src/cli/migration-compile.ts

@@ -19,12 +19,13 @@
  * lands with whole-DB introspection. The emission is identical either way.
  */
 
-import type { ModelDescriptor, Module } from '@everystack/model';
+import type { ModelDescriptor, Module, SequenceDescriptor } from '@everystack/model';
 import type { AuthzContract, TableContract } from './authz-contract.js';
-import { compileCreateTable, compileEnums, foreignKeySql, modelConstraintSpecs, qualifiedTable, type CompileTableOptions } from './schema-compile.js';
+import { compileCreateTable, compileEnums, compileSequences, sequenceCreateSql, foreignKeySql, modelConstraintSpecs, qualifiedTable, toSnakeCase, type CompileTableOptions } from './schema-compile.js';
 import { compileTableContract } from './authz-compile.js';
 import { emitReconcileSql } from './authz-reconcile.js';
-import { emitSchemaSql, type SchemaChange } from './schema-diff.js';
+import { emitSchemaSql, nextvalSequence, type SchemaChange } from './schema-diff.js';
+import { compileDerived } from './derived-compile.js';
 
 /** The empty (not-yet-created) authz state for a table — the greenfield baseline. */
 function emptyTable(table: string): TableContract {
@@ -36,7 +37,7 @@ function emptyTable(table: string): TableContract {
  * schema). Tables, then foreign keys, then authz — the order a fresh database must
  * apply them in.
  */
-export function compileMigration(models: ModelDescriptor[], opts: CompileTableOptions = {}): string[] {
+export function compileMigration(models: ModelDescriptor[], opts: CompileTableOptions & { sequences?: SequenceDescriptor[] } = {}): string[] {
   const sql: string[] = [];
   const schemaOf = (m: ModelDescriptor): string => m.schema ?? 'public';
 
@@ -71,9 +72,24 @@ export function compileMigration(models: ModelDescriptor[], opts: CompileTableOp
   const enumCreates: SchemaChange[] = compileEnums(models).map((e) => ({ kind: 'createType', name: e.name, values: e.values }));
   sql.push(...emitSchemaSql(enumCreates));
 
+  // 0c. Standalone sequences — before any table, so a column's defaultSql(nextval(…))
+  //     can draw on one from the first CREATE. State layer: these migrate, never reconcile.
+  sql.push(...compileSequences(opts.sequences ?? []).map(sequenceCreateSql));
+
   // 1. Tables (with columns, CHECKs, single/composite PK, uniques).
   for (const model of models) sql.push(compileCreateTable(model, opts));
 
+  // 1b. Foreign-sequence defaults — a `defaultSql` nextval draws from a sequence that
+  //     may be owned by a table created later in phase 1, so fieldColumnSql held it out
+  //     of the column line; every table (and its serial sequence) exists now.
+  for (const model of models) {
+    for (const [name, f] of Object.entries(model.fields)) {
+      const spec = f.spec;
+      if (spec.defaultKind !== 'sql' || nextvalSequence(String(spec.default)) == null) continue;
+      sql.push(`ALTER TABLE ${qualifiedTable(model)} ALTER COLUMN "${toSnakeCase(name)}" SET DEFAULT ${String(spec.default)};`);
+    }
+  }
+
   // 2. Foreign keys — after every table exists.
   for (const model of models) sql.push(...foreignKeySql(model));
 
@@ -82,7 +98,12 @@ export function compileMigration(models: ModelDescriptor[], opts: CompileTableOp
   const indexCreates: SchemaChange[] = [];
   for (const model of models) {
     for (const ix of modelConstraintSpecs(model).indexes) {
-      indexCreates.push({ kind: 'createIndex', table: qualifiedTable(model), name: ix.name, columns: ix.columns, unique: ix.unique, ...(ix.where ? { where: ix.where } : {}) });
+      indexCreates.push({
+        kind: 'createIndex', table: qualifiedTable(model), name: ix.name, columns: ix.columns, unique: ix.unique,
+        ...(ix.where ? { where: ix.where } : {}),
+        ...(ix.using ? { using: ix.using } : {}),
+        ...(ix.include?.length ? { include: ix.include } : {}),
+      });
     }
   }
   sql.push(...emitSchemaSql(indexCreates));
@@ -119,15 +140,25 @@ export function compileModuleMigration(modules: Module[], opts: CompileTableOpti
   const extensions = [...new Set(modules.flatMap((m) => m.extensions))].sort();
   sql.push(...extensions.map((e) => `CREATE EXTENSION IF NOT EXISTS "${e}";`));
 
-  // 2. Schema + authz for every modeled table.
-  sql.push(...compileMigration(modules.flatMap((m) => m.models), opts));
+  // 2. Schema + authz for every modeled table (plus the modules' standalone sequences).
+  sql.push(...compileMigration(modules.flatMap((m) => m.models), { ...opts, sequences: modules.flatMap((m) => m.sequences) }));
 
   // 3. Package SQL — functions + triggers, after the tables they reference. Each thunk's
   //    output is a self-contained multi-statement block applied as one unit.
+  //    (@deprecated — the declared derived layer below supersedes it.)
   for (const fn of modules.flatMap((m) => m.functions)) {
     const text = fn().trim();
     if (text) sql.push(text);
   }
 
+  // 4. The DECLARED derived layer — views, matviews, functions, triggers, escape-hatch
+  //    objects — compiled in topological order, after every table. Greenfield = one
+  //    complete script: state + compute; from then on the layer deploys via db:reconcile.
+  const models = modules.flatMap((m) => m.models);
+  for (const obj of compileDerived(models, modules.flatMap((m) => m.derived))) {
+    sql.push(`${obj.sql};`);
+    for (const a of obj.attachments) sql.push(`${a.sql};`);
+  }
+
   return sql;
 }

package/src/cli/migration-generate.ts

@@ -15,10 +15,10 @@
  * so the command shell (commands/db-generate.ts) is a thin layer of IO over it.
  */
 
-import type { ModelDescriptor } from '@everystack/model';
+import type { ModelDescriptor, SequenceDescriptor } from '@everystack/model';
 import type { SchemaSnapshot } from './schema-introspect.js';
 import type { AuthzContract } from './authz-contract.js';
-import { compileTableSchema, compileRenames, compileTableRenames, compileCreateTable, compileEnums } from './schema-compile.js';
+import { compileTableSchema, compileRenames, compileTableRenames, compileCreateTable, compileEnums, compileSequences } from './schema-compile.js';
 import { compileTableContract } from './authz-compile.js';
 import { emitReconcileSql } from './authz-reconcile.js';
 import { diffSchema, emitSchemaSql, type SchemaChange } from './schema-diff.js';
@@ -26,6 +26,8 @@ import { diffSchema, emitSchemaSql, type SchemaChange } from './schema-diff.js';
 export interface GenerateOptions {
   /** The Postgres schema the Models live in. Default: `public`. */
   schema?: string;
+  /** Standalone sequences the modules declare (state layer — created before tables). */
+  sequences?: SequenceDescriptor[];
   /**
    * Emit real `DROP COLUMN`/`DROP CONSTRAINT`/`DROP TABLE` for things present in the
    * database but absent from the Models. Default `false` — those are held back as
@@ -104,7 +106,7 @@ export function generateMigrationSql(models: ModelDescriptor[], current: SchemaS
   // then the call default — so one run spans schemas.
   const schemaOf = (m: ModelDescriptor): string => m.schema ?? schema;
   const desiredEnums = compileEnums(models);
-  const desired: SchemaSnapshot = { tables: models.map((m) => compileTableSchema(m, { schema })), enums: desiredEnums };
+  const desired: SchemaSnapshot = { tables: models.map((m) => compileTableSchema(m, { schema })), enums: desiredEnums, ...(opts.sequences?.length ? { sequences: compileSequences(opts.sequences) } : {}) };
   const declaredTables = new Set(desired.tables.map((t) => t.table));
   const declaredEnums = new Set(desiredEnums.map((e) => e.name));
   // Scope both tables and enums to what the Models declare — an undeclared table OR enum
@@ -116,6 +118,18 @@ export function generateMigrationSql(models: ModelDescriptor[], current: SchemaS
   // its authz contract must be read under the NEW name (the policies/grants
   // ride the rename in Postgres; without the rewrite they would re-emit and
   // collide).
+  // 0. Non-public schemas the models declare but the live side lacks — created FIRST, or
+  //    a fresh-database build (db:check's ephemeral compose, db:branch templates) fails on
+  //    its own first CREATE TABLE. The diff analog of compileMigration's phase 0a; live
+  //    databases already have their schemas (their own migrations made them), so this is
+  //    invisible in the brownfield loop. IF NOT EXISTS keeps an empty-but-existing schema
+  //    (no tables for the live side to reveal it by) from failing the create.
+  const liveSchemas = new Set(current.tables.map((t) => t.table.split('.')[0]));
+  const schemaPhase = [...new Set(desired.tables.map((t) => t.table.split('.')[0]))]
+    .filter((s) => s !== 'public' && !liveSchemas.has(s))
+    .sort()
+    .map((s) => `CREATE SCHEMA IF NOT EXISTS "${s}"`);
+
   const tableRenames = compileTableRenames(models, { schema });
   const currentNames = new Set(current.tables.map((t) => t.table));
   const pendingRenameSources = new Set(
@@ -177,7 +191,7 @@ export function generateMigrationSql(models: ModelDescriptor[], current: SchemaS
     ? emitReconcileSql({ tables: models.map((m) => compileTableContract(m, { schema })), functions: [] }, liveAuthzRenamed)
     : [];
 
-  return [...dataPhase, ...authzPhase];
+  return [...schemaPhase, ...dataPhase, ...authzPhase];
 }
 
 /** The marker drizzle migration files put between statements. */
@@ -200,6 +214,31 @@ export interface Journal {
   version: string;
   dialect: string;
   entries: JournalEntry[];
+  /**
+   * D2 — the resolved `--schema-out` path, recorded at generate time. Later runs
+   * without the flag reuse it (flag > recorded > default), so a relocated artifact
+   * never leaves an orphan at the default path. Drizzle's migrator ignores the key.
+   */
+  schemaOut?: string;
+}
+
+/** How a schema-out path was chosen, and whether the journal record needs (re)writing. */
+export interface SchemaOutResolution {
+  path: string;
+  source: 'flag' | 'recorded' | 'default';
+  updateRecord: boolean;
+}
+
+/**
+ * Resolve where the generated drizzle schema lives: an explicit flag wins, then the
+ * journal's recorded path, then the default. The record updates whenever the resolved
+ * path is not already recorded — a differing flag overwrites it (the caller says so).
+ */
+export function resolveSchemaOut(flag: string | undefined, journal: Journal | null, fallback: string): SchemaOutResolution {
+  const recorded = journal?.schemaOut;
+  if (flag) return { path: flag, source: 'flag', updateRecord: flag !== recorded };
+  if (recorded) return { path: recorded, source: 'recorded', updateRecord: false };
+  return { path: fallback, source: 'default', updateRecord: recorded !== fallback };
 }
 
 /** Sanitize a migration name into the snake_case the `NNNN_<tag>.sql` convention uses. */