@everystack/cli 0.3.31 → 0.4.0

package/src/cli/schema-introspect.ts

@@ -53,10 +53,17 @@ export interface CheckConstraint {
 /** A standalone index (NOT one backing a PK or UNIQUE constraint — those are excluded). */
 export interface IndexSchema {
   name: string;
+  /** Canonical key text per position: a plain column name, or anything richer verbatim —
+   *  `lower(email)`, `created_at DESC`, `title text_pattern_ops` (Brick E: expression
+   *  indexes, direction, and opclass are IDENTITY, not noise). Default ASC is stripped. */
   columns: string[];
   unique: boolean;
   /** Partial-index predicate (`WHERE …`), omitted when the index is total. */
   where?: string;
+  /** Access method when not btree (`'gin'`, `'gist'`, …) — btree is never spelled. */
+  using?: string;
+  /** `INCLUDE (…)` covering columns. */
+  include?: string[];
 }
 
 export interface TableSchema {
@@ -78,10 +85,25 @@ export interface EnumType {
   values: string[];
 }
 
+/** A STANDALONE sequence — one no serial column owns (those stay implicit in their
+ *  column). It holds state (the counter position), so it lives in the base schema:
+ *  migrated before tables, diffed by existence, fingerprinted. `start`/`increment`
+ *  are omitted when they are PostgreSQL's defaults (1), so a declaration that omits
+ *  them matches a live default exactly. */
+export interface SequenceSchema {
+  /** Bare for public (`order_number_seq`), qualified otherwise (`reporting.order_seq`). */
+  name: string;
+  as: 'integer' | 'bigint';
+  start?: number;
+  increment?: number;
+}
+
 export interface SchemaSnapshot {
   tables: TableSchema[];
   /** Enum types in the schema (`CREATE TYPE … AS ENUM`). Absent/empty when none. */
   enums?: EnumType[];
+  /** Standalone sequences (serial-owned ones excluded). Absent/empty when none. */
+  sequences?: SequenceSchema[];
 }
 
 // ---------------------------------------------------------------------------
@@ -163,6 +185,30 @@ GROUP BY n.nspname, t.typname
 ORDER BY n.nspname, t.typname;
 `.trim();
 
+/**
+ * STANDALONE sequences — the ones no serial column owns (a serial's implicit sequence
+ * has an 'a'/'i' pg_depend link to its column and stays represented BY that column).
+ * These hold state, so they are base-schema: migrated, diffed, fingerprinted.
+ */
+export const SEQUENCES_SQL = `
+SELECT
+  n.nspname AS schema,
+  c.relname AS name,
+  format_type(s.seqtypid, NULL) AS data_type,
+  s.seqstart AS start,
+  s.seqincrement AS increment
+FROM pg_sequence s
+JOIN pg_class c ON c.oid = s.seqrelid
+JOIN pg_namespace n ON n.oid = c.relnamespace
+WHERE n.nspname NOT IN ('pg_catalog', 'information_schema')
+  AND n.nspname NOT LIKE 'pg_%'
+  AND NOT EXISTS (
+    SELECT 1 FROM pg_depend d
+    WHERE d.objid = c.oid AND d.deptype IN ('a', 'i') AND d.refobjsubid > 0
+  )
+ORDER BY n.nspname, c.relname;
+`.trim();
+
 /**
  * Standalone indexes — `CREATE INDEX` the app declares, NOT the implicit indexes that back a
  * primary key or unique constraint (those are excluded by the `conindid` check, since they're
@@ -179,9 +225,12 @@ SELECT
   (SELECT array_agg(a.attname ORDER BY k.ord)
      FROM unnest(idx.indkey) WITH ORDINALITY k(attnum, ord)
      JOIN pg_attribute a ON a.attrelid = idx.indrelid AND a.attnum = k.attnum) AS columns,
-  pg_get_expr(idx.indpred, idx.indrelid) AS predicate
+  pg_get_expr(idx.indpred, idx.indrelid) AS predicate,
+  pg_get_indexdef(idx.indexrelid) AS definition,
+  am.amname AS method
 FROM pg_index idx
 JOIN pg_class i ON i.oid = idx.indexrelid
+JOIN pg_am am ON am.oid = i.relam
 JOIN pg_class t ON t.oid = idx.indrelid
 JOIN pg_namespace n ON n.oid = t.relnamespace
 WHERE t.relkind = 'r'
@@ -300,17 +349,107 @@ export interface IndexRow {
   is_unique: unknown;
   columns: unknown;
   predicate: unknown;
+  /** `pg_get_indexdef` — the authoritative key list (expressions, DESC, opclass, INCLUDE).
+   *  Optional: a legacy row without it falls back to the plain columns array. */
+  definition?: unknown;
+  /** `pg_am.amname`. Optional; btree folds to absent. */
+  method?: unknown;
 }
 
-export function indexRowToDescriptor(row: IndexRow): { table: string; index: IndexSchema } {
-  const columns = Array.isArray(row.columns) ? row.columns.map(String) : [];
-  const pred = row.predicate == null ? '' : String(row.predicate);
+/** Split a parenthesized list on top-level commas (expression entries contain their own). */
+function splitTopLevel(list: string): string[] {
+  const out: string[] = [];
+  let depth = 0;
+  let start = 0;
+  for (let i = 0; i < list.length; i++) {
+    const ch = list[i];
+    if (ch === '(') depth++;
+    else if (ch === ')') depth--;
+    else if (ch === ',' && depth === 0) {
+      out.push(list.slice(start, i));
+      start = i + 1;
+    }
+  }
+  out.push(list.slice(start));
+  return out.map((s) => s.trim()).filter(Boolean);
+}
+
+/** The parenthesized group starting at `open` (which must point at '('), content only. */
+function parenGroup(text: string, open: number): string | null {
+  let depth = 0;
+  for (let i = open; i < text.length; i++) {
+    if (text[i] === '(') depth++;
+    else if (text[i] === ')' && --depth === 0) return text.slice(open + 1, i);
+  }
+  return null;
+}
+
+/** Canonicalize one key entry: unquote a plain `"col"`, strip the default ` ASC`. */
+function canonicalKeyEntry(entry: string): string {
+  let s = entry.trim().replace(/\s+ASC$/i, '');
+  const quoted = s.match(/^"((?:[^"]|"")*)"$/);
+  if (quoted) s = quoted[1].replace(/""/g, '"');
+  return s;
+}
+
+/**
+ * The extended identity from a `pg_get_indexdef` text (Brick E): canonical key entries
+ * (expressions, direction, opclass — default ASC stripped, plain names unquoted), the
+ * access method from the `USING` clause (btree folds to absent), INCLUDE columns,
+ * uniqueness, and the partial predicate. Null when the text isn't index DDL.
+ */
+export function parseIndexDefinition(definition: string): Omit<IndexSchema, 'name'> | null {
+  const usingMatch = definition.match(/\sUSING\s+(\w+)\s*\(/i);
+  if (!usingMatch || usingMatch.index === undefined) return null;
+  const open = definition.indexOf('(', usingMatch.index);
+  const keyList = parenGroup(definition, open);
+  if (keyList === null) return null;
+  const columns = splitTopLevel(keyList).map(canonicalKeyEntry);
+  const method = usingMatch[1].toLowerCase();
+  const after = definition.slice(open + keyList.length + 2);
+  const includeAt = after.search(/^\s*INCLUDE\s*\(/i);
+  const include = includeAt >= 0
+    ? splitTopLevel(parenGroup(after, after.indexOf('(')) ?? '').map(canonicalKeyEntry)
+    : undefined;
+  const whereMatch = after.match(/\sWHERE\s+(.+)$/i);
+  let where = whereMatch ? whereMatch[1].trim() : '';
+  // pg_get_indexdef wraps the predicate in one pair of parens — strip it.
+  if (where.startsWith('(') && parenGroup(where, 0) === where.slice(1, -1)) where = where.slice(1, -1).trim();
   return {
-    table: `${row.schema}.${row.table}`,
-    index: { name: row.name, columns, unique: coerceBool(row.is_unique), ...(pred ? { where: pred } : {}) },
+    columns,
+    unique: /^CREATE\s+UNIQUE\s+INDEX/i.test(definition),
+    ...(where ? { where } : {}),
+    ...(method !== 'btree' ? { using: method } : {}),
+    ...(include?.length ? { include } : {}),
   };
 }
 
+/** A row's index descriptor: the definition text when present (the extended identity);
+ *  a legacy row without one keeps the old plain-columns path, so fixtures stay valid.
+ *  `is_unique`/`predicate`/`method` row fields take precedence over the parsed text. */
+export function indexRowToDescriptor(row: IndexRow): { table: string; index: IndexSchema } {
+  const pred = row.predicate == null ? '' : String(row.predicate);
+  const table = `${row.schema}.${row.table}`;
+  const base = { name: row.name, unique: coerceBool(row.is_unique), ...(pred ? { where: pred } : {}) };
+
+  const parsed = row.definition == null ? null : parseIndexDefinition(String(row.definition));
+  if (parsed) {
+    const method = row.method == null ? parsed.using : String(row.method).toLowerCase();
+    return {
+      table,
+      index: {
+        ...base,
+        columns: parsed.columns,
+        ...(method && method !== 'btree' ? { using: method } : {}),
+        ...(parsed.include?.length ? { include: parsed.include } : {}),
+      },
+    };
+  }
+
+  const columns = Array.isArray(row.columns) ? row.columns.map(String) : [];
+  return { table, index: { ...base, columns } };
+}
+
 export interface EnumRow {
   schema: string;
   name: string;
@@ -322,11 +461,32 @@ export function enumRowToDescriptor(row: EnumRow): EnumType {
   return { name: row.name, values };
 }
 
+export interface SequenceRow {
+  schema: string;
+  name: string;
+  data_type: unknown;
+  start: unknown;
+  increment: unknown;
+}
+
+export function sequenceRowToDescriptor(row: SequenceRow): SequenceSchema {
+  const start = Number(row.start ?? 1);
+  const increment = Number(row.increment ?? 1);
+  return {
+    name: row.schema === 'public' ? row.name : `${row.schema}.${row.name}`,
+    as: String(row.data_type) === 'integer' ? 'integer' : 'bigint',
+    // PostgreSQL's defaults are omitted, so declared-without matches live-default.
+    ...(start !== 1 ? { start } : {}),
+    ...(increment !== 1 ? { increment } : {}),
+  };
+}
+
 export interface SchemaRows {
   columns: ColumnRow[];
   constraints: ConstraintRow[];
   enums?: EnumRow[];
   indexes?: IndexRow[];
+  sequences?: SequenceRow[];
 }
 
 /**
@@ -400,9 +560,15 @@ export function assembleSchema(rows: SchemaRows): SchemaSnapshot {
     .map(enumRowToDescriptor)
     .sort((a, b) => a.name.localeCompare(b.name));
 
+  const sequences = (rows.sequences ?? [])
+    .filter((r) => !IGNORED_SCHEMAS.has(r.schema))
+    .map(sequenceRowToDescriptor)
+    .sort((a, b) => a.name.localeCompare(b.name));
+
   return {
     tables: [...tables.values()].sort((a, b) => a.table.localeCompare(b.table)),
     enums,
+    ...(sequences.length ? { sequences } : {}),
   };
 }
 
@@ -413,13 +579,14 @@ export function assembleSchema(rows: SchemaRows): SchemaSnapshot {
  * `assembleSchema`. This is the `current` side `db:generate` diffs the compiled Models against.
  */
 export async function introspectSchema(run: QueryRunner): Promise<SchemaSnapshot> {
-  const [columns, constraints, enums, indexes] = await Promise.all([
-    run(COLUMNS_SQL), run(CONSTRAINTS_SQL), run(ENUMS_SQL), run(INDEXES_SQL),
+  const [columns, constraints, enums, indexes, sequences] = await Promise.all([
+    run(COLUMNS_SQL), run(CONSTRAINTS_SQL), run(ENUMS_SQL), run(INDEXES_SQL), run(SEQUENCES_SQL),
   ]);
   return assembleSchema({
     columns: columns as ColumnRow[],
     constraints: constraints as ConstraintRow[],
     enums: enums as EnumRow[],
     indexes: indexes as IndexRow[],
+    sequences: sequences as SequenceRow[],
   });
 }

package/src/cli/schema-source.ts

@@ -25,7 +25,8 @@
  * Input is a `@everystack/model` ModelDescriptor (type-only — no runtime dep).
  */
 
-import type { ModelDescriptor, FieldSpec } from '@everystack/model';
+import type { ModelDescriptor, FieldSpec, DerivedDescriptor, ViewDescriptor, MaterializedViewDescriptor } from '@everystack/model';
+import { parseQualified } from './derived-source.js';
 
 /** `author_id` is the SQL identifier; `authorId` the field key. The single snake-case rule. */
 function toSnakeCase(name: string): string {
@@ -144,6 +145,7 @@ function columnModifiers(
   if (spec.defaultKind === 'now') s += '.defaultNow()';
   else if (spec.defaultKind === 'random') s += '.defaultRandom()';
   else if (spec.defaultKind === 'value') s += `.default(${defaultValueLiteral(spec.default)})`;
+  else if (spec.defaultKind === 'sql') s += `.default(sql.raw(${strLiteral(String(spec.default))}))`;
 
   if (spec.isPrimaryKey && !isComposite) s += '.primaryKey()';
 
@@ -250,6 +252,25 @@ function relationNameFragment(
 export interface CompileDrizzleSourceOptions {
   /** Reserved for parity with the DDL compiler's `schema` option. Unused in the emit. */
   schema?: string;
+  /**
+   * The declared derived layer (B6a): views/matviews that declare `fields` are emitted
+   * as typed `pgView(...)`/`pgMaterializedView(...)` `.existing()` blocks — the
+   * reconciler owns their DDL; the generated schema carries only the read shape.
+   * Relations without `fields` and non-relation kinds are skipped.
+   */
+  derived?: DerivedDescriptor[];
+}
+
+/** The derived relations that opted into typed emission, identity-sorted (deterministic). */
+function typedRelations(derived: DerivedDescriptor[]): (ViewDescriptor | MaterializedViewDescriptor)[] {
+  return derived
+    .filter((d): d is ViewDescriptor | MaterializedViewDescriptor =>
+      (d.kind === 'view' || d.kind === 'materialized view') && !!d.fields && Object.keys(d.fields).length > 0)
+    .sort((a, b) => {
+      const qa = parseQualified(a.name);
+      const qb = parseQualified(b.name);
+      return qa.schema.localeCompare(qb.schema) || qa.name.localeCompare(qb.name);
+    });
 }
 
 /**
@@ -264,14 +285,15 @@ export function compileDrizzleSource(allModels: ModelDescriptor[], _opts: Compil
   // generated schema. (The migration generator still emits its SQL + authz.) The
   // command and the drift guard both call this, so filtering here keeps them in sync.
   const models = allModels.filter((m) => !m.private);
+  const derivedRelations = typedRelations(_opts.derived ?? []);
 
   const modelsByDescriptor = new Map<ModelDescriptor, { camel: string }>();
   for (const model of models) modelsByDescriptor.set(model, { camel: toCamelCase(model.table) });
 
   // --- Collect enums (deduped by name, sorted) -----------------------------
   const enumsByName = new Map<string, readonly string[]>();
-  for (const model of models) {
-    for (const f of Object.values(model.fields)) {
+  const collectEnums = (fields: Record<string, { spec: FieldSpec }>) => {
+    for (const f of Object.values(fields)) {
       const { type, enumName, enumValues } = f.spec;
       if (type !== 'enum' || !enumName) continue;
       const values = enumValues ?? [];
@@ -281,7 +303,9 @@ export function compileDrizzleSource(allModels: ModelDescriptor[], _opts: Compil
       }
       if (!existing) enumsByName.set(enumName, values);
     }
-  }
+  };
+  for (const model of models) collectEnums(model.fields);
+  for (const rel of derivedRelations) collectEnums(rel.fields!);
   const enumNames = [...enumsByName.keys()].sort((a, b) => a.localeCompare(b));
 
   // --- Track which pg-core builders + drizzle-orm symbols are used ----------
@@ -303,6 +327,7 @@ export function compileDrizzleSource(allModels: ModelDescriptor[], _opts: Compil
       const spec = builder.spec;
       const { call, builder: pgBuilder } = baseColumnSource(toSnakeCase(key), spec);
       pgCoreBuilders.add(pgBuilder);
+      if (spec.defaultKind === 'sql') pgCoreBuilders.add('sql'); // imported from 'drizzle-orm', handled below
       const mods = columnModifiers(spec, isComposite, modelsByDescriptor);
       if (spec.isDeprecated) {
         // The contract phase (decision 13): the column stays in the database and
@@ -381,14 +406,57 @@ export function compileDrizzleSource(allModels: ModelDescriptor[], _opts: Compil
     );
   }
 
+  // --- Emit derived relation blocks (typed reads — B6a) ----------------------
+  // Views/matviews that declared `fields` become `.existing()` declarations: the
+  // reconciler owns their DDL, so drizzle never creates them — the block carries
+  // ONLY the read shape (type, nullability, array). Identity-sorted for the
+  // drift-guard byte-match; column names snake_case by the same single rule.
+  const derivedBlocks: string[] = [];
+  for (const rel of derivedRelations) {
+    const { schema, name } = parseQualified(rel.name);
+    const camel = toCamelCase(schema === 'public' ? name : `${schema}_${name}`);
+
+    const colLines: string[] = [];
+    for (const [key, builder] of Object.entries(rel.fields!)) {
+      const spec = builder.spec;
+      const { call, builder: pgBuilder } = baseColumnSource(toSnakeCase(key), spec);
+      pgCoreBuilders.add(pgBuilder);
+      // Only .array()/.notNull() can appear — defineView/defineMaterializedView
+      // reject every table-structural modifier at define time.
+      const mods = columnModifiers(spec, true, modelsByDescriptor);
+      colLines.push(`  ${key}: ${call}${mods},`);
+    }
+    const cols = `{\n${colLines.join('\n')}\n}`;
+
+    if (schema === 'public') {
+      const topFn = rel.kind === 'view' ? 'pgView' : 'pgMaterializedView';
+      pgCoreBuilders.add(topFn);
+      derivedBlocks.push(`export const ${camel} = ${topFn}(${strLiteral(name)}, ${cols}).existing();`);
+    } else {
+      const schemaFn = rel.kind === 'view' ? 'view' : 'materializedView';
+      pgCoreBuilders.add('pgSchema');
+      derivedBlocks.push(`export const ${camel} = pgSchema(${strLiteral(schema)}).${schemaFn}(${strLiteral(name)}, ${cols}).existing();`);
+    }
+  }
+
   // --- Header ----------------------------------------------------------------
-  const header = [
+  const headerLines = [
     '// GENERATED by `everystack db:generate` from models/. Do not edit.',
     '//',
     '// Cross-schema foreign keys (e.g. profiles.id -> auth.users.id) are intentionally',
     '// omitted: a Model cannot reference a table outside the model set, so the runtime',
     '// table and its relations() never carry that FK. The DB constraint stays untouched.',
-  ].join('\n');
+  ];
+  if (derivedBlocks.length) {
+    headerLines.push(
+      '//',
+      '// Derived relations (views/matviews) are declared `.existing()`: db:reconcile owns',
+      '// their DDL — these blocks carry only the read shape their descriptors declare.',
+      '// That shape is author-declared, NOT introspected: a wrong field type or',
+      '// nullability in the descriptor mistypes reads silently. Fix the descriptor.',
+    );
+  }
+  const header = headerLines.join('\n');
 
   // --- Imports ---------------------------------------------------------------
   // pg-core: sorted, enum consts are not imports (they're declared below), `sql`
@@ -422,6 +490,7 @@ export function compileDrizzleSource(allModels: ModelDescriptor[], _opts: Compil
   if (enumDecls.length) sections.push(enumDecls.join('\n'));
   for (const block of tableBlocks) sections.push(block);
   for (const block of relationBlocks) sections.push(block);
+  for (const block of derivedBlocks) sections.push(block);
 
   return sections.join('\n\n') + '\n';
 }

package/src/cli/state-apply.ts

@@ -23,6 +23,7 @@ import { introspectContract, type AuthzContract, type QueryRunner } from './auth
 import { introspectSchema, type SchemaSnapshot } from './schema-introspect.js';
 import { FUNCTIONS_SQL, contractFunctionRow } from './security-catalog.js';
 import { generateMigrationSql, HELD_DROP_PREFIX } from './migration-generate.js';
+import type { SequenceDescriptor } from '@everystack/model';
 import { fingerprintLive } from './schema-fingerprint.js';
 import { ENSURE_RECONCILER_SQL, renderSchemaLogInsert, renderSchemaLogFingerprintUpdate } from './derived-apply.js';
 
@@ -170,6 +171,8 @@ export async function applyGeneratedStatements(
 export interface StateSyncOptions extends StateApplyOptions {
   /** Passed through to the verifying re-diff so held-vs-real matches the plan. */
   allowDrops?: boolean;
+  /** Standalone sequences — the verify re-diff must see the same declared state as the plan. */
+  sequences?: SequenceDescriptor[];
 }
 
 export interface StateSyncOutcome {
@@ -228,7 +231,7 @@ export async function applyStateAndVerify(
   }
 
   const remaining = classifyGeneratedStatements(
-    generateMigrationSql(models, snapshot, { allowDrops: options.allowDrops, liveAuthz: contract }),
+    generateMigrationSql(models, snapshot, { allowDrops: options.allowDrops, liveAuthz: contract, sequences: options.sequences }),
   ).executable;
 
   return {