@everystack/cli 0.4.14 → 0.4.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.14",
+  "version": "0.4.16",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/commands/db-check.ts

@@ -78,6 +78,10 @@ export interface StaticCheckInput {
   derived?: DerivedDescriptor[] | null;
   /** Set when the derived layer failed to COMPILE (e.g. reachability) — a hard fail. */
   derivedError?: string;
+  /** The models the barrel's MODULES compose (declared-derived's stream). The verbs' state
+   *  stream reads `export const models`; when the two disagree, the state layer builds a
+   *  different database than the modules declare — the split-brain gate below names it. */
+  moduleModels?: ModelDescriptor[] | null;
 }
 
 export function runStaticChecks(input: StaticCheckInput): CheckFinding[] {
@@ -106,6 +110,31 @@ export function runStaticChecks(input: StaticCheckInput): CheckFinding[] {
     });
     return findings;
   }
+  // The split-brain gate: `export const models` (what every verb's state stream reads)
+  // vs the modules' composed models (what the derived stream reads). Divergence means the
+  // ephemeral compose builds a database missing tables the derived layer references — the
+  // cryptic "relation does not exist" this names precisely. The --matviews-as-tables
+  // splice is the common trigger: materializedTables spread into defineModule but not
+  // into the exported array.
+  if (input.moduleModels != null) {
+    const identity = (m: ModelDescriptor): string => `${m.schema}.${m.table}`;
+    const exported = new Set(input.models.map(identity));
+    const composed = new Set(input.moduleModels.map(identity));
+    const missingFromExport = [...composed].filter((t) => !exported.has(t)).sort();
+    const missingFromModules = [...exported].filter((t) => !composed.has(t)).sort();
+    if (missingFromExport.length || missingFromModules.length) {
+      const parts: string[] = [];
+      if (missingFromExport.length) parts.push(`composed by modules but MISSING from \`export const models\`: ${missingFromExport.join(', ')}`);
+      if (missingFromModules.length) parts.push(`exported but composed by NO module: ${missingFromModules.join(', ')}`);
+      findings.push({
+        level: 'fail',
+        area: 'compose',
+        message: `the barrel's exported models and its modules disagree — ${parts.join('; ')}. The verbs read the EXPORT; keep one array and use it in both places (a --matviews-as-tables pull: spread materializedTables into \`export const models\`, not only defineModule).`,
+      });
+      return findings;
+    }
+  }
+
   try {
     compileDeclaredState(input.models);
     const statements = compileMigration(input.models, { sequences: input.sequences });
@@ -269,6 +298,7 @@ export async function dbCheckCommand(flags: Record<string, string>): Promise<voi
   const findings = runStaticChecks({
     modelsPath, models, modelsError, artifactPath, artifactSource, sqlDirRetired,
     sequences: declaredDb?.sequences, derived: declaredDb?.derived, derivedError,
+    moduleModels: declaredDb?.models ?? null,
   });
   for (const f of findings) {
     (f.level === 'fail' || f.level === 'warn' ? warn : info)(`${MARK[f.level]} ${f.area}: ${f.message}`);

package/src/cli/commands/db-pull.ts

@@ -47,6 +47,37 @@ const ok = (m: string) => console.error(`  ✓ ${m}`);
 const detail = (m: string) => console.error(`  ${m}`);
 const caution = (m: string) => console.error(`  ! ${m}`);
 
+/** Types that can't (arrays, json) or won't sensibly (jsonb) serve as a key — the probe skips them. */
+const UNKEYABLE = (type: string): boolean => type.endsWith('[]') || type === 'json' || type === 'jsonb';
+
+/**
+ * The `--suggest-keys` probe for one matview: one aggregate scan measuring, per keyable
+ * column, the total row count, the non-null count, and the distinct count. Counts come back
+ * `::text` so bigints survive any runner. Null when no column is probeable. Pure — the
+ * caller runs it and feeds the row to {@link keyCandidates}.
+ */
+export function keyProbeSql(identity: string, columns: ColumnSchema[]): { sql: string; cols: string[] } | null {
+  const cols = columns.filter((c) => !UNKEYABLE(c.type)).map((c) => c.name);
+  if (cols.length === 0) return null;
+  const parts = ['count(*)::text AS n'];
+  cols.forEach((c, i) => {
+    parts.push(`count("${c}")::text AS c${i}`, `count(DISTINCT "${c}")::text AS d${i}`);
+  });
+  const [schema, name] = identity.split('.');
+  return { sql: `SELECT ${parts.join(', ')} FROM "${schema}"."${name}"`, cols };
+}
+
+/**
+ * The columns that are FUNCTIONALLY unique over the live rows: non-null count == distinct
+ * count == total, and the matview is non-empty (zero rows prove nothing). These render as
+ * COMMENTED `.primaryKey()` suggestions — a surfaced decision, never a guessed constraint.
+ */
+export function keyCandidates(row: Record<string, unknown>, cols: string[]): string[] {
+  const n = Number(row.n);
+  if (!Number.isFinite(n) || n === 0) return [];
+  return cols.filter((_, i) => Number(row[`c${i}`]) === n && Number(row[`d${i}`]) === n);
+}
+
 /** A QueryRunner backed by the ops Lambda `db:query` action (read-only). */
 function lambdaRunner(region: string, fn: string): QueryRunner {
   return async (sql: string) => {
@@ -85,10 +116,16 @@ export async function dbPullCommand(flags: Record<string, string>): Promise<void
   }
 
   const matviewsAsTables = flags['matviews-as-tables'] === 'true';
+  const suggestKeys = flags['suggest-keys'] === 'true';
+  if (suggestKeys && !matviewsAsTables) {
+    fail('--suggest-keys is a --matviews-as-tables companion (it suggests .primaryKey() candidates for the flipped tables) — pass both.');
+    process.exit(1);
+  }
 
   let current;
   let derivedCatalog: DerivedCatalog | undefined;
   let matviewColumns: Map<string, ColumnSchema[]> | undefined;
+  let candidatesByIdentity: Map<string, string[]> | undefined;
   try {
     let runner: QueryRunner;
     let end: (() => Promise<void>) | undefined;
@@ -110,6 +147,19 @@ export async function dbPullCommand(flags: Record<string, string>): Promise<void
     // matview columns the derived layer (definition-only) doesn't carry.
     if (matviewsAsTables) {
       matviewColumns = matviewColumnsByIdentity((await runner(MATVIEW_COLUMNS_SQL)) as ColumnRow[]);
+      // --suggest-keys: one aggregate scan PER MATVIEW measuring functional uniqueness —
+      // the natural-key evidence a matview without a unique index can't otherwise give.
+      if (suggestKeys && matviewColumns.size) {
+        note(`Probing ${matviewColumns.size} matview(s) for key candidates (one full scan each)...`);
+        candidatesByIdentity = new Map();
+        for (const [identity, cols] of matviewColumns) {
+          const probe = keyProbeSql(identity, cols);
+          if (!probe) continue;
+          const [row] = await runner(probe.sql);
+          const found = keyCandidates((row ?? {}) as Record<string, unknown>, probe.cols);
+          if (found.length) candidatesByIdentity.set(identity, found);
+        }
+      }
     }
     await end?.();
   } catch (err: any) {
@@ -143,6 +193,7 @@ export async function dbPullCommand(flags: Record<string, string>): Promise<void
       matviewsAsTables: {
         columns: matviewColumns ?? new Map(),
         enums: new Map((current.enums ?? []).map((e) => [e.name, e.values])),
+        ...(candidatesByIdentity ? { keyCandidates: candidatesByIdentity } : {}),
       },
     } : {}),
   });
@@ -152,7 +203,7 @@ export async function dbPullCommand(flags: Record<string, string>): Promise<void
     detail(`Derived layer: ${derived.names.length} descriptor(s) + ${derived.sequenceNames.length} sequence(s) rendered (adopt with db:reconcile --baseline after committing).`);
   }
   if (derived.materializedTableNames.length) {
-    detail(`${derived.materializedTableNames.length} matview(s) rendered as defineMaterializedTable — these are MODELS: wire them into your module's models array (spread \`materializedTables\` from the derived file).`);
+    detail(`${derived.materializedTableNames.length} matview(s) rendered as defineMaterializedTable — these are MODELS: spread \`materializedTables\` into your barrel's \`export const models\` (the array the verbs read) so defineModule and the export stay ONE array. db:check fails on any divergence.`);
     caution('Matviews carry no PK/NOT NULL — the rendered fields are nullable and unkeyed; tighten (.primaryKey()/.notNull()) as you review. Unique matview indexes render as index().unique() constraints.');
   }
 

package/src/cli/declared-derived.ts

@@ -11,7 +11,7 @@
 import path from 'node:path';
 import fs from 'node:fs/promises';
 import { pathToFileURL } from 'node:url';
-import type { Module, SequenceDescriptor, DerivedDescriptor } from '@everystack/model';
+import type { Module, ModelDescriptor, SequenceDescriptor, DerivedDescriptor } from '@everystack/model';
 import { compileDerived } from './derived-compile.js';
 import { compileTableRenames } from './schema-compile.js';
 import type { SourceObject } from './derived-source.js';
@@ -27,6 +27,9 @@ export interface DeclaredDerived {
   sequences: SequenceDescriptor[];
   /** Pending table renames (`new qualified` → `old qualified`) from the models' markers. */
   renamedTables: Record<string, string>;
+  /** The models the modules compose — db:check's split-brain gate compares these against
+   *  the barrel's `export const models` (the array every verb's state stream reads). */
+  models: ModelDescriptor[];
 }
 
 /**
@@ -166,6 +169,7 @@ export function composeDeclaredDerived(modules: Module[], modelsPath: string): D
       derived,
       sequences: modules.flatMap((m) => m.sequences),
       renamedTables: compileTableRenames(models, {}),
+      models,
     };
   } catch (err) {
     throw asModelComposeError(modelsPath, err);

package/src/cli/derived-render.ts

@@ -52,6 +52,10 @@ export interface DerivedRenderOptions {
     columns: Map<string, ColumnSchema[]>;
     /** Enum name → values, for enum-typed matview columns (same map the model render uses). */
     enums?: Map<string, string[]>;
+    /** --suggest-keys: matview identity → columns measured functionally unique over the live
+     *  rows. Rendered as a COMMENTED .primaryKey() suggestion on the field line — a surfaced
+     *  decision (the authz-scaffold idiom), never applied code. */
+    keyCandidates?: Map<string, string[]>;
   };
 }
 
@@ -276,7 +280,12 @@ export function renderDerivedSource(catalog: DerivedCatalog, opts: DerivedRender
             table: o.identity, columns: cols,
             primaryKey: [], uniques: [], checks: [], foreignKeys: [], indexes: [],
           };
-          const fieldLines = renderFieldLines(shape, new Set(), opts.matviewsAsTables.enums ?? new Map());
+          // Field lines align with `cols` by index — append the --suggest-keys comment in place.
+          const suggested = new Set(opts.matviewsAsTables.keyCandidates?.get(o.identity) ?? []);
+          const fieldLines = renderFieldLines(shape, new Set(), opts.matviewsAsTables.enums ?? new Map())
+            .map((line, i) => suggested.has(cols[i].name)
+              ? `${line} // PK candidate: non-null + all-distinct across live rows — add .primaryKey() if this is the natural key`
+              : line);
           const builders = o.indexes.map((def) => {
             const b = renderTableIndexBuilder(def);
             if (b === null) warnings.push(`${o.identity}: index not renderable: ${def}`);
@@ -287,10 +296,15 @@ export function renderDerivedSource(catalog: DerivedCatalog, opts: DerivedRender
           need('defineMaterializedTable');
           need('field');
           need('sql');
+          // MODEL naming idiom: bare table + a schema prop. The derived layer's
+          // qualified-first-arg spelling must NOT leak here — defineModel stores the
+          // table name verbatim, so 'stats.x' would become a dot-containing identifier
+          // in public and 3-part-qualify (public.stats.x) downstream.
           lines.push(
             ...fixmes,
-            `export const ${varName} = defineMaterializedTable(${tsString(declaredName(o))}, {`,
+            `export const ${varName} = defineMaterializedTable(${tsString(o.name)}, {`,
             `  ${abilities.length ? `abilities: [${abilities.join(', ')}],` : 'private: true,'}`,
+            ...(o.schema !== 'public' ? [`  schema: ${tsString(o.schema)},`] : []),
             '  fields: {',
             ...fieldLines,
             '  },',
@@ -439,8 +453,10 @@ export function renderDerivedFile(result: DerivedRenderResult, knownTables: Map<
   if (result.sequenceNames.length) {
     lines.push(`export const sequences = [\n${result.sequenceNames.map((n) => `  ${n},`).join('\n')}\n];`);
   }
-  // Materialized tables (--matviews-as-tables) are MODELS: the consumer spreads this array
-  // into their module's models — defineModule({ models: [...models, ...materializedTables] }).
+  // Materialized tables (--matviews-as-tables) are MODELS: spread this array into the
+  // barrel's `export const models` — the array the verbs' state stream reads — and let
+  // defineModule take that same array. Spreading into defineModule ONLY leaves the export
+  // without these tables and the ephemeral compose fails (db:check's split-brain gate).
   if (result.materializedTableNames.length) {
     lines.push(`export const materializedTables = [\n${result.materializedTableNames.map((n) => `  ${n},`).join('\n')}\n];`);
   }

package/src/cli/index.ts

@@ -364,7 +364,7 @@ Usage:
   everystack db:export         --schema <name> [--stage <name> | --database-url <url> [--out <file.dump>]] [--models <barrel>]  Schema-scoped pg_dump artifact, stamped with the DECLARED schema fingerprint (the canonical-sync export; db:swap gates on that stamp). --stage dumps the stage's private DB via the ops Lambda → S3; --database-url (explicit flag, never the env) dumps a reachable DB to a local .dump + .meta.json — the build-locally → publish → swap on-ramp
   everystack db:swap           --schema <name> --database-url <url> --from <artifact.dump> [--fingerprint <hash>]  Land a schema artifact atomically: fingerprint gate → restore into <schema>_incoming (COPY-safe rewrite) → one txn (drop+rename+recreate app→schema FKs, re-apply authz) → verify → drop retiring. Refresh-free; app.* untouched. DESTRUCTIVE (--stage/--direct ops venue rides stage-write-lanes)
   everystack db:generate       [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--schema-out db/schema.generated.ts] [--allow-drops] [--apply] [--dry-run]  Diff models vs the live DB → next migration file, or with --apply execute it directly (one transaction, schema_log recorded, verified by re-diff — no drizzle folder needed; direct connection only; DROPs held back unless --allow-drops). --dry-run prints the edge and writes NOTHING (no migration, no journal entry, no schema refresh) — the preview verb; db:diff computes a models-vs-models edge with no database at all. The resolved --schema-out is recorded in the migration journal: later flag-less runs reuse it (flag > recorded > default), a differing flag updates the record and says so
-  everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--derived-out <file.ts>] [--abilities public-read]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. --derived-out <file.ts> extracts the derived layer (descriptors + sequences) as its own self-contained module — alone it leaves the models untouched (the hand-maintained-barrel splice); with --out the models render omits the now-external derived layer. Every model scaffolds its authz decision as comments (db:check fails until authored); --abilities public-read stamps the common stanza (public read, admin write) uncommented — explicit generated code, never a runtime default. --matviews-as-tables renders every matview as defineMaterializedTable with INTROSPECTED fields (the canonical-sync flip: a pipeline-owned table everystack migrates but never refreshes) — names land in an exported materializedTables array to spread into your models; fields come back nullable/unkeyed (matviews carry no PK/NOT NULL) — tighten on review
+  everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--derived-out <file.ts>] [--abilities public-read]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. --derived-out <file.ts> extracts the derived layer (descriptors + sequences) as its own self-contained module — alone it leaves the models untouched (the hand-maintained-barrel splice); with --out the models render omits the now-external derived layer. Every model scaffolds its authz decision as comments (db:check fails until authored); --abilities public-read stamps the common stanza (public read, admin write) uncommented — explicit generated code, never a runtime default. --matviews-as-tables renders every matview as defineMaterializedTable with INTROSPECTED fields (the canonical-sync flip: a pipeline-owned table everystack migrates but never refreshes) — names land in an exported materializedTables array to spread into your models; fields come back nullable/unkeyed (matviews carry no PK/NOT NULL) — tighten on review; add --suggest-keys to probe the LIVE rows for functionally-unique columns (one scan per matview) and surface each as a commented .primaryKey() suggestion. docs/derived-objects.md#flipping-a-matview-to-a-materialized-table---matviews-as-tables
                                Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
   everystack db:fingerprint    [--stage <name> | --database-url <url>] [--models <barrel>] [--json]  Content-address the live base schema (tables+constraints+authz) and compare against the models — MATCH/MISMATCH (exit 1), plus the unfingerprinted-objects report
   everystack db:reconcile      [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--rebuild] [--overwrite-drift] [--json]  Reconcile the derived layer (functions/views/matviews/triggers) against the DECLARED descriptors (defineView/defineMaterializedView/defineFunction/defineSql/trigger() on models, from the barrel) — the single home (db/sql is retired; leftover .sql files fail with the migration path): plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (direct connection only, atomic — DDL + provenance in one transaction) and records provenance + schema_log. Hand-edits are drift (never overwritten silently). First contact with existing objects: --baseline TRUSTS live == source (records provenance, verifies nothing), --rebuild GUARANTEES it (drop+create from source). They are mutually exclusive.