@everystack/cli 0.3.14 → 0.3.16

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.3.14",
+  "version": "0.3.16",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/commands/db-authz.ts

@@ -44,6 +44,7 @@ import { writeContract, loadContract } from '../authz-contract-io.js';
 import { renderContractMarkdown } from '../authz-render.js';
 import { FUNCTIONS_SQL, catalogFunctionToDescriptor } from '../security-catalog.js';
 import { resolveConfig, opsFunction } from '../config.js';
+import { resolveModelsPath } from '../models-path.js';
 import { invokeAction } from '../aws.js';
 import { step, success, fail, info, warn } from '../output.js';
 
@@ -239,7 +240,7 @@ async function loadModels(modelsPath: string): Promise<ModelDescriptor[]> {
  * exit on any leak or vacuous policy; an `unprobed` table (one identity) is a warning, not a fail.
  */
 export async function dbAuthzOwnerCommand(flags: Record<string, string>): Promise<void> {
-  const modelsPath = flags.models || 'models/index.ts';
+  const modelsPath = resolveModelsPath(flags.models);
   const schema = flags.schema || 'public';
 
   let probes: OwnerProbe[];

package/src/cli/commands/db-generate.ts

@@ -28,11 +28,12 @@ import { generateMigrationSql, unmodeledTables, formatMigrationFile, planMigrati
 import { introspectContract, type QueryRunner } from '../authz-contract.js';
 import { FUNCTIONS_SQL, catalogFunctionToDescriptor } from '../security-catalog.js';
 import { resolveDbSource, createUrlRunner, type DbSource } from '../db-source.js';
+import { resolveModelsPath } from '../models-path.js';
+import { applyGeneratedStatements, classifyGeneratedStatements, currentGitRef } from '../state-apply.js';
 import { resolveConfig, opsFunction } from '../config.js';
 import { invokeAction } from '../aws.js';
 import { step, success, fail, info, warn } from '../output.js';
 
-const DEFAULT_MODELS = 'models/index.ts';
 const DEFAULT_MIGRATIONS = 'drizzle';
 const DEFAULT_SCHEMA_OUT = 'db/schema.generated.ts';
 
@@ -123,7 +124,7 @@ async function dbGenerateInit(modelsPath: string, migrationsDir: string, schemaO
 }
 
 export async function dbGenerateCommand(flags: Record<string, string>): Promise<void> {
-  const modelsPath = flags.models || DEFAULT_MODELS;
+  const modelsPath = resolveModelsPath(flags.models);
   const migrationsDir = path.resolve(flags.dir || DEFAULT_MIGRATIONS);
   const schemaOut = path.resolve(flags['schema-out'] || DEFAULT_SCHEMA_OUT);
   const name = flags.name || 'generated';
@@ -141,9 +142,12 @@ export async function dbGenerateCommand(flags: Record<string, string>): Promise<
     return;
   }
 
+  const apply = flags.apply === 'true';
   let models: ModelDescriptor[];
   let current;
   let liveAuthz;
+  let runner!: QueryRunner;
+  let end: (() => Promise<void>) | undefined;
   try {
     step(`Loading models from ${modelsPath}...`);
     models = await loadModels(modelsPath);
@@ -154,8 +158,10 @@ export async function dbGenerateCommand(flags: Record<string, string>): Promise<
     step(`Writing the drizzle schema → ${path.relative(process.cwd(), schemaOut)}...`);
     await fs.writeFile(schemaOut, compileDrizzleSource(models), 'utf8');
     const dbSource: DbSource = resolveDbSource(flags);
-    let runner: QueryRunner;
-    let end: (() => Promise<void>) | undefined;
+    if (apply && dbSource.kind !== 'url') {
+      fail('--apply needs a direct connection (--database-url or DATABASE_URL) — the ops Lambda query path is read-only by design.');
+      process.exit(1);
+    }
     if (dbSource.kind === 'url') {
       step(dbSource.from === 'flag' ? 'Connecting via --database-url...' : 'Connecting via DATABASE_URL...');
       ({ runner, end } = await createUrlRunner(dbSource.url));
@@ -169,7 +175,7 @@ export async function dbGenerateCommand(flags: Record<string, string>): Promise<
     current = await introspectSchema(runner);
     step('Introspecting authorization (rls + grants + policies)...');
     liveAuthz = await introspectContract(runner, mapFunctionRow, FUNCTIONS_SQL);
-    await end?.();
+    if (!apply) await end?.();
   } catch (err: any) {
     fail(err.message);
     info('Ensure your IAM user/role has lambda:InvokeFunction and the stage is deployed.');
@@ -190,6 +196,45 @@ export async function dbGenerateCommand(flags: Record<string, string>): Promise<
     process.exit(0);
   }
 
+  // --apply: no tape. Print the plan, execute the executable subset as one
+  // transaction, record it in schema_log, then VERIFY by re-diffing — done
+  // means db:generate is a no-op again, not "a file was written".
+  if (apply) {
+    const classified = classifyGeneratedStatements(statements);
+    info(`Plan — ${statements.length} statement(s):`);
+    console.log('');
+    for (const s of statements) console.log(s.endsWith(';') ? s : `${s};`);
+    console.log('');
+    if (classified.heldDrops.length) warn(`${classified.heldDrops.length} destructive change(s) held back — NOT executed (re-run with --allow-drops to include them).`);
+    if (classified.notices.length) info(`${classified.notices.length} notice(s) — manual follow-ups, nothing executed for them.`);
+    if (classified.executable.length === 0) {
+      success('Nothing executable — only held/notice items differ.');
+      process.exit(0);
+    }
+    step(`Applying ${classified.executable.length} statement(s) as one transaction...`);
+    try {
+      await applyGeneratedStatements(runner, statements, { actor: process.env.USER ?? null, gitRef: currentGitRef() });
+    } catch (err: any) {
+      fail(`Apply failed and rolled back: ${err.message}`);
+      process.exit(1);
+    }
+    step('Verifying — re-introspecting and re-diffing...');
+    const after = await introspectSchema(runner);
+    const afterAuthz = await introspectContract(runner, mapFunctionRow, FUNCTIONS_SQL);
+    const remaining = classifyGeneratedStatements(
+      generateMigrationSql(models, after, { allowDrops, liveAuthz: afterAuthz }),
+    ).executable;
+    await end?.();
+    if (remaining.length > 0) {
+      fail(`Verify FAILED — ${remaining.length} statement(s) still differ after apply:`);
+      for (const s of remaining) console.log(s);
+      process.exit(1);
+    }
+    const heldNote = classified.heldDrops.length ? ` (${classified.heldDrops.length} held drop(s) remain, by design)` : '';
+    success(`Applied and verified — db:generate is a no-op again${heldNote}. Recorded in everystack.schema_log.`);
+    process.exit(0);
+  }
+
   const journalPath = path.join(migrationsDir, 'meta', '_journal.json');
   let journal: Journal;
   try {

package/src/cli/commands/db-reconcile.ts

@@ -24,7 +24,6 @@
 
 import fs from 'node:fs/promises';
 import path from 'node:path';
-import { execSync } from 'node:child_process';
 import type { QueryRunner } from '../authz-contract.js';
 import { parseSqlSources, type SourceFile } from '../derived-source.js';
 import { introspectDerived } from '../derived-introspect.js';
@@ -37,6 +36,7 @@ import {
   ENSURE_RECONCILER_SQL,
 } from '../derived-apply.js';
 import { resolveDbSource, createUrlRunner, type DbSource } from '../db-source.js';
+import { currentGitRef } from '../state-apply.js';
 import { resolveConfig, opsFunction } from '../config.js';
 import { invokeAction } from '../aws.js';
 import { step, success, fail, info, warn } from '../output.js';
@@ -223,14 +223,6 @@ async function readSqlDir(dir: string): Promise<SourceFile[]> {
   })));
 }
 
-function currentGitRef(): string | null {
-  try {
-    return execSync('git rev-parse HEAD', { stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim();
-  } catch {
-    return null;
-  }
-}
-
 export async function dbReconcileCommand(flags: Record<string, string>): Promise<void> {
   const sqlDir = flags['sql-dir'] || DEFAULT_SQL_DIR;
   const apply = flags.apply === 'true';

package/src/cli/derived-apply.ts

@@ -13,7 +13,7 @@
  * the reconciler writes it and never reads it.
  */
 
-import type { SourceObject } from './derived-source.js';
+import { normalizeSql, type SourceObject } from './derived-source.js';
 import type { ReconcilePlan } from './derived-plan.js';
 import { quoteQualified } from './pg-ident.js';
 
@@ -69,9 +69,22 @@ export function ensureOrReplace(sql: string): string {
   return sql.replace(/^(\s*)CREATE\s+FUNCTION/i, '$1CREATE OR REPLACE FUNCTION');
 }
 
-/** An object's full creation SQL: the CREATE statement plus its attachments, in source order. */
+const WITH_NO_DATA_RE = /WITH\s+NO\s+DATA\s*$/i;
+
+/**
+ * An object's full creation SQL: the CREATE statement plus its attachments, in
+ * source order. A matview whose source says `WITH NO DATA` gets a trailing
+ * REFRESH: the reconciler dropped a populated matview to rebuild it, and
+ * executing the source verbatim would leave it unpopulated — matching the
+ * text but regressing the database. (Found by a consumer whose extracted
+ * sources carry pg_dump's WITH NO DATA ordering.)
+ */
 function objectSql(obj: SourceObject): string[] {
-  return [obj.sql, ...obj.attachments.map((a) => a.sql)];
+  const statements = [obj.sql, ...obj.attachments.map((a) => a.sql)];
+  if (obj.kind === 'materialized view' && WITH_NO_DATA_RE.test(normalizeSql(obj.sql))) {
+    statements.push(`REFRESH MATERIALIZED VIEW ${quoteQualified(obj.identity)}`);
+  }
+  return statements;
 }
 
 export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[]): RenderedReconcile {

package/src/cli/index.ts

@@ -295,14 +295,14 @@ Usage:
   everystack db:backups        [--stage <name>]  List logical backups (id, size, created)
   everystack db:restore        --from <id> [--stage <name>] --confirm  Restore a backup INTO the stage's DB (DESTRUCTIVE)
   everystack db:backup:download <id> [--stage <name>]  Presigned URL to download a backup's dump (valid 1h)
-  everystack db:generate       [--stage <name> | --database-url <url>] [--name <label>] [--models models/index.ts] [--allow-drops]  Diff models vs the live DB → write the next migration (never touches the DB; DROPs held back unless --allow-drops)
+  everystack db:generate       [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--allow-drops] [--apply]  Diff models vs the live DB → next migration file, or with --apply execute it directly (one transaction, schema_log recorded, verified by re-diff — no drizzle folder needed; direct connection only; DROPs held back unless --allow-drops)
   everystack db:pull           [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>]  Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise
                                Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
   everystack db:reconcile      [--sql-dir db/sql] [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--overwrite-drift] [--json]  Reconcile the derived layer (functions/views/matviews) against db/sql sources: plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (direct connection only) and records provenance + schema_log. Hand-edits are drift (never overwritten silently); first contact with existing objects wants --baseline, once.
   everystack db:authz:pull     [--stage <name>] [--dir authz]  Introspect live authz (rls/grants/policies/secdef) → reviewable contract files
   everystack db:authz:diff     [--stage <name>] [--dir authz]  Validate the live DB against the committed contract (non-zero exit on drift)
   everystack db:authz:test     [--stage <name>] [--dir authz]  Red-team enforcement: SET ROLE + attempt per role/table/command (non-zero exit on a hole)
-  everystack db:authz:owner    [--stage <name>] [--models models/index.ts]  Red-team owner isolation: two JWT identities per owner-scoped table — catches IDOR (non-zero exit on a leak)
+  everystack db:authz:owner    [--stage <name>] [--models db/models/index.ts]  Red-team owner isolation: two JWT identities per owner-scoped table — catches IDOR (non-zero exit on a leak)
   everystack db:authz:report   [--dir authz]  Render the committed contract as a human-readable authorization review (no DB)
   everystack console           --stage <name> [--sandbox]  Interactive REPL on deployed Lambda
   everystack status            [--stage <name>] [--hours <n>]  Platform health: CDN, Lambda, rollup summary

package/src/cli/models-path.ts

@@ -0,0 +1,25 @@
+/**
+ * models-path — where the Models barrel lives.
+ *
+ * One home for schema: `db/models/` (state) beside `db/sql/` (compute). The
+ * CLI looks there first, falling back to the legacy top-level `models/` so
+ * existing consumers keep working without a flag. An explicit `--models`
+ * always wins. When neither home exists, the NEW home is returned so error
+ * messages point people at the current convention, not the legacy one.
+ */
+
+import { existsSync } from 'node:fs';
+
+/** Candidate barrels, in preference order. */
+export const MODELS_HOMES = ['db/models/index.ts', 'models/index.ts'] as const;
+
+export function resolveModelsPath(
+  flag?: string,
+  exists: (p: string) => boolean = existsSync,
+): string {
+  if (flag) return flag;
+  for (const home of MODELS_HOMES) {
+    if (exists(home)) return home;
+  }
+  return MODELS_HOMES[0];
+}

package/src/cli/runbook/detect.ts

@@ -102,10 +102,12 @@ export function detectProjectReality(root: string): ProjectReality {
       ? 'V2'
       : 'V1';
 
-  const modelsDir = path.join(abs, 'models');
+  // One home for schema: db/models is the current convention, top-level models the legacy one.
+  const modelsHome = isDir(path.join(abs, 'db', 'models')) ? 'db/models' : 'models';
+  const modelsDir = path.join(abs, modelsHome);
   const modelFiles = listTs(modelsDir).filter((f) => f !== 'index.ts');
   const models = {
-    dir: isDir(modelsDir) ? 'models' : null,
+    dir: isDir(modelsDir) ? modelsHome : null,
     files: modelFiles,
     hasIndex: isFile(path.join(modelsDir, 'index.ts')),
   };

package/src/cli/state-apply.ts

@@ -0,0 +1,123 @@
+/**
+ * state-apply — apply db:generate's live-computed diff directly, no tape.
+ *
+ * The state layer's analog of the derived reconciler's apply path: the diff
+ * is computed fresh against the live catalog (it cannot be stale, cannot be
+ * skipped, and has nothing numbered to collide), executed as ONE batch (one
+ * implicit transaction), and recorded in `everystack.schema_log` with the
+ * full SQL text. Held drops and notices are comments by construction and are
+ * never executed — the --allow-drops posture decides what is real DDL long
+ * before this module sees it.
+ *
+ * This retires the drizzle folder as the transport for table changes. The
+ * review discipline the tape file used to provide moves to the printed plan:
+ * the command shows every statement (including what it is NOT running)
+ * before anything executes. Fingerprint pinning (the edge runner) later
+ * upgrades "computed against the database just now" to "verified against a
+ * named state at both ends" — but nothing here needs a folder.
+ */
+
+import { execSync } from 'node:child_process';
+import type { QueryRunner } from './authz-contract.js';
+import { HELD_DROP_PREFIX } from './migration-generate.js';
+import { ENSURE_RECONCILER_SQL, renderSchemaLogInsert } from './derived-apply.js';
+
+const NOTICE_PREFIX = '-- NOTICE';
+
+/** Every line is a comment (or blank) — nothing executable inside. */
+function isPureComment(statement: string): boolean {
+  return statement
+    .split('\n')
+    .every((line) => line.trim() === '' || line.trim().startsWith('--'));
+}
+
+export interface ClassifiedStatements {
+  /** Real DDL (possibly carrying leading -- WARNING lines) — what apply runs. */
+  executable: string[];
+  /** Destructive changes held back as comments (re-run with --allow-drops). */
+  heldDrops: string[];
+  /** Manual follow-ups / rename notices — comments, never executed. */
+  notices: string[];
+}
+
+export function classifyGeneratedStatements(statements: string[]): ClassifiedStatements {
+  const executable: string[] = [];
+  const heldDrops: string[] = [];
+  const notices: string[] = [];
+  for (const statement of statements) {
+    if (statement.startsWith(HELD_DROP_PREFIX)) heldDrops.push(statement);
+    else if (statement.startsWith(NOTICE_PREFIX) || isPureComment(statement)) notices.push(statement);
+    else executable.push(statement);
+  }
+  return { executable, heldDrops, notices };
+}
+
+/** The current git HEAD, for schema_log's intent pointer. Null outside a repo. */
+export function currentGitRef(): string | null {
+  try {
+    return execSync('git rev-parse HEAD', { stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim();
+  } catch {
+    return null;
+  }
+}
+
+export interface StateApplyOptions {
+  actor?: string | null;
+  gitRef?: string | null;
+  planRef?: string | null;
+  /** Injectable clock for tests. */
+  now?: () => number;
+}
+
+export interface StateApplyResult {
+  /** True when DDL actually executed. */
+  applied: boolean;
+  classified: ClassifiedStatements;
+  /** The batch that was sent (empty when nothing was executable). */
+  batchSql: string;
+}
+
+/**
+ * Execute the executable subset of a generated diff as one batch and record
+ * it in schema_log. A failed batch rolls back (one implicit transaction),
+ * logs the failure best-effort, and rethrows.
+ */
+export async function applyGeneratedStatements(
+  runner: QueryRunner,
+  statements: string[],
+  options: StateApplyOptions = {},
+): Promise<StateApplyResult> {
+  const classified = classifyGeneratedStatements(statements);
+  if (classified.executable.length === 0) {
+    return { applied: false, classified, batchSql: '' };
+  }
+
+  const now = options.now ?? Date.now;
+  await runner(ENSURE_RECONCILER_SQL.join(';\n'));
+
+  // Semicolon on its own line: a statement whose last line is a comment
+  // cannot swallow the separator.
+  const batchSql = classified.executable.join('\n;\n');
+  const started = now();
+  try {
+    await runner(batchSql);
+  } catch (err: any) {
+    try {
+      await runner(renderSchemaLogInsert({
+        kind: 'state apply', sql: batchSql,
+        outcome: `failed: ${String(err?.message ?? err).slice(0, 500)}`,
+        actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
+        durationMs: now() - started,
+      }));
+    } catch { /* the memoir is best-effort on failure */ }
+    throw err;
+  }
+
+  await runner(renderSchemaLogInsert({
+    kind: 'state apply', sql: batchSql, outcome: 'applied',
+    actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
+    durationMs: now() - started,
+  }));
+
+  return { applied: true, classified, batchSql };
+}