@everystack/cli 0.3.28 → 0.4.0

package/README.md

@@ -216,7 +216,7 @@ everystack db:psql --stage dev [-c SQL] # interactive psql / one-off query via L
 # Model-driven schema (@everystack/model)
 everystack db:generate                  # diff Models vs the live DB → next migration
 everystack db:pull                      # introspect a live DB → field() Models (brownfield on-ramp)
-everystack db:reconcile                 # deploy the derived layer (functions/views/matviews) from db/sql — no migrations
+everystack db:reconcile                 # deploy the derived layer (functions/views/matviews/triggers) from the declared descriptors in db/models — no migrations
 everystack db:fingerprint               # content-address the live base schema vs the Models — MATCH/MISMATCH
 everystack db:sync                      # make the database match your checkout — state + compute, one verb (dev DBs)
 everystack db:diff                      # the state edge between two declared states — no DB, CI-pure; reverse = computed rollback

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.3.28",
+  "version": "0.4.0",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -33,6 +33,10 @@
       "types": "./src/cli/audit-api.ts",
       "default": "./src/cli/audit-api.ts"
     },
+    "./apply": {
+      "types": "./src/cli/apply-execute.ts",
+      "default": "./src/cli/apply-execute.ts"
+    },
     "./audit/source": {
       "types": "./src/cli/audit-source-api.ts",
       "default": "./src/cli/audit-source-api.ts"
@@ -78,7 +82,7 @@
     "structured-headers": "1.0.1",
     "tsx": "4.21.0",
     "typescript": "5.9.3",
-    "@everystack/model": "0.3.6"
+    "@everystack/model": "0.4.0"
   },
   "peerDependencies": {
     "@everystack/server": ">=0.1.0",

package/src/cli/apply-execute.ts

@@ -0,0 +1,185 @@
+/**
+ * apply-execute — the apply core: verify → apply → verify, as one function
+ * over an injected QueryRunner.
+ *
+ * This is the transactional heart of `db:apply`, lifted out of the command
+ * shell so it can run in either place the write must live:
+ *   - the CLI, over a direct `--database-url` connection (local dev DBs), and
+ *   - the ops Lambda, over its resource-linked operator connection (a deployed
+ *     stage, reached credential-free via IAM — the operator never holds a URL).
+ *
+ * The three moves, in order, none skippable:
+ *   1. VERIFY the target is exactly where the plan started — live
+ *      fingerprint == plan.from, or REFUSE. This is the concurrency lock:
+ *      two applies racing the same stage serialize on it (the second's
+ *      parent no longer matches; it re-mints against the new reality). A
+ *      hand-edited target refuses the same way. A target already at
+ *      plan.to is done — re-apply is idempotent, not an error.
+ *   2. APPLY the edge as one transaction, recorded in schema_log with the
+ *      plan's content address (`plan_ref`) and both fingerprints.
+ *   3. VERIFY the target landed exactly on plan.to. The plan is
+ *      self-contained — no models needed at apply time — and the endpoint
+ *      is exact even on brownfield targets (mint predicted it with the
+ *      unmodeled tables merged in). A mismatch here is a real fault, loud.
+ *
+ * Pure orchestration: the connection, the git-descent verdict, and the
+ * destructive ceremony (authority, snapshot) are all injected. The checkout
+ * and the caller's identity live outside the Lambda, so descent and authority
+ * are computed by the caller and passed in as verdicts; the write, the lock,
+ * and the memoir stay here, atomic with the connection.
+ */
+
+import { introspectContract, type QueryRunner, type AuthzContract } from './authz-contract.js';
+import { introspectSchema, type SchemaSnapshot } from './schema-introspect.js';
+import { FUNCTIONS_SQL, contractFunctionRow } from './security-catalog.js';
+import { fingerprintLive } from './schema-fingerprint.js';
+import { applyGeneratedStatements } from './state-apply.js';
+import { ENSURE_RECONCILER_SQL, renderSchemaLogInsert, renderSchemaLogFingerprintUpdate } from './derived-apply.js';
+import { checkPlanPrecondition, planHash, type EdgePlan } from './edge-plan.js';
+
+export interface ApplyPlanOptions {
+  actor?: string | null;
+  /** Defaults to the plan's own gitRef (the intent that minted it). */
+  gitRef?: string | null;
+  /**
+   * The fast-forward rule (brick 7): called with the target's live state
+   * after the concurrency lock passes, BEFORE anything executes. Return
+   * `ok: false` to refuse — the checkout does not descend from the commit
+   * declaring the target's state. Omitted = no descent enforcement (tests,
+   * or an explicit, confirmed, snapshotted force).
+   */
+  verifyDescent?: (live: {
+    snapshot: SchemaSnapshot;
+    contract: AuthzContract;
+    fingerprint: string;
+  }) => Promise<{ ok: true } | { ok: false; reason: string }>;
+  /**
+   * Destructive authority (brick 9, decisions 11+12): called only when the
+   * plan is destructive, after the lock and descent checks pass. Return
+   * `ok: false` to refuse — the caller is not in the stage's declared
+   * approver set. Omitted = no authority gate (non-staged targets keep the
+   * ceremony the command shell enforces).
+   */
+  verifyAuthority?: () => Promise<{ ok: true } | { ok: false; reason: string }>;
+  /**
+   * Runs after every verification passes and BEFORE the edge executes, only
+   * for destructive plans — the auto-snapshot seam ("losing data should be
+   * hard": confirmed, approved, snapshotted, in that order). A throw here
+   * aborts the apply with nothing executed.
+   */
+  beforeDestructive?: () => Promise<void>;
+  /** Injectable clock for tests. */
+  now?: () => number;
+}
+
+export interface ApplyPlanResult {
+  status: 'applied' | 'already-applied' | 'refused' | 'descent-refused' | 'authority-refused' | 'verify-failed';
+  liveFingerprint: string;
+  reason?: string;
+  logId?: number;
+}
+
+/**
+ * Decision 7, made whole: every database records every refusal, not just
+ * every apply. Best-effort by construction — a memoir INSERT that cannot be
+ * written must never mask the refusal itself (a target that rejects the
+ * bookkeeping write would have failed the apply's ENSURE identically).
+ */
+async function recordRefusal(
+  runner: QueryRunner,
+  plan: EdgePlan,
+  liveFingerprint: string,
+  gate: string,
+  reason: string,
+  options: ApplyPlanOptions,
+): Promise<void> {
+  try {
+    await runner(ENSURE_RECONCILER_SQL.join(';\n'));
+    await runner(renderSchemaLogInsert({
+      kind: 'state refusal',
+      outcome: 'refused',
+      actor: options.actor ?? null,
+      gitRef: options.gitRef ?? plan.gitRef,
+      planRef: planHash(plan),
+      fromFingerprint: liveFingerprint,
+      toFingerprint: plan.toFingerprint,
+      sql: `-- REFUSED (${gate}): ${reason}\n${plan.statements.join('\n;\n')}`,
+    }));
+  } catch {
+    // Swallowed deliberately — see the docblock.
+  }
+}
+
+/** Verify → apply → verify. Pure orchestration over an injected QueryRunner. */
+export async function executeApplyPlan(
+  runner: QueryRunner,
+  plan: EdgePlan,
+  options: ApplyPlanOptions = {},
+): Promise<ApplyPlanResult> {
+  const before = await introspectSchema(runner);
+  const beforeAuthz = await introspectContract(runner, contractFunctionRow, FUNCTIONS_SQL);
+  const live = fingerprintLive(before, beforeAuthz).hash;
+
+  if (live === plan.toFingerprint) {
+    return { status: 'already-applied', liveFingerprint: live };
+  }
+  const pre = checkPlanPrecondition(plan, live);
+  if (!pre.ok) {
+    await recordRefusal(runner, plan, live, 'concurrency lock', pre.reason, options);
+    return { status: 'refused', liveFingerprint: live, reason: pre.reason };
+  }
+
+  if (options.verifyDescent) {
+    const descent = await options.verifyDescent({ snapshot: before, contract: beforeAuthz, fingerprint: live });
+    if (!descent.ok) {
+      await recordRefusal(runner, plan, live, 'fast-forward rule', descent.reason, options);
+      return { status: 'descent-refused', liveFingerprint: live, reason: descent.reason };
+    }
+  }
+
+  if (plan.destructive > 0 && options.verifyAuthority) {
+    const authority = await options.verifyAuthority();
+    if (!authority.ok) {
+      await recordRefusal(runner, plan, live, 'destructive authority', authority.reason, options);
+      return { status: 'authority-refused', liveFingerprint: live, reason: authority.reason };
+    }
+  }
+
+  if (plan.destructive > 0 && options.beforeDestructive) {
+    await options.beforeDestructive();
+  }
+
+  const result = await applyGeneratedStatements(runner, plan.statements, {
+    actor: options.actor,
+    gitRef: options.gitRef ?? plan.gitRef,
+    planRef: planHash(plan),
+    fromFingerprint: live,
+    now: options.now,
+  });
+
+  const after = await introspectSchema(runner);
+  const afterAuthz = await introspectContract(runner, contractFunctionRow, FUNCTIONS_SQL);
+  const landed = fingerprintLive(after, afterAuthz).hash;
+  if (result.logId !== undefined) {
+    await runner(renderSchemaLogFingerprintUpdate(result.logId, landed));
+  }
+
+  if (landed !== plan.toFingerprint) {
+    return {
+      status: 'verify-failed',
+      liveFingerprint: landed,
+      reason: `applied, but the target landed on ${landed.slice(0, 12)} — the plan predicted ${plan.toFingerprint.slice(0, 12)}. Investigate before touching this database again (db:fingerprint, db:generate).`,
+      ...(result.logId !== undefined ? { logId: result.logId } : {}),
+    };
+  }
+  return {
+    status: 'applied',
+    liveFingerprint: landed,
+    ...(result.logId !== undefined ? { logId: result.logId } : {}),
+  };
+}
+
+// Re-exported so a single import (`@everystack/cli/apply`) gives a consumer —
+// notably the ops Lambda's db:apply action — everything it needs to validate
+// and run a plan without reaching into the CLI's internal module layout.
+export { PLAN_VERSION, planHash, checkPlanPrecondition, type EdgePlan } from './edge-plan.js';

package/src/cli/authz-compile.ts

@@ -308,8 +308,10 @@ export function compileTableContract(model: ModelDescriptor, opts: CompileOption
 /** The four DML privileges, sorted — the expansion of a `manage` ability. */
 const CRUD = ['DELETE', 'INSERT', 'SELECT', 'UPDATE'];
 
-/** The SQL privilege a single CRUD verb grants. `manage` expands to all of CRUD. */
-const VERB: Record<Exclude<Ability['action'], 'manage'>, string> = {
+/** The SQL privilege a single CRUD verb grants. `manage` expands to all of CRUD.
+ *  `execute` is a FUNCTION ability — defineModel rejects it on tables, so it never
+ *  reaches the table grant compiler (the throw below keeps that contract loud). */
+const VERB: Record<Exclude<Ability['action'], 'manage' | 'execute'>, string> = {
   read: 'SELECT',
   create: 'INSERT',
   update: 'UPDATE',
@@ -318,6 +320,7 @@ const VERB: Record<Exclude<Ability['action'], 'manage'>, string> = {
 
 /** The privileges an ability's action grants (a single verb, or all of CRUD for `manage`). */
 function verbsFor(action: Ability['action']): string[] {
+  if (action === 'execute') throw new Error(`can('execute') is a function ability — it has no table privilege (defineModel rejects it; this is unreachable).`);
   return action === 'manage' ? [...CRUD] : [VERB[action]];
 }
 

package/src/cli/authz-lint.ts

@@ -0,0 +1,48 @@
+/**
+ * authz-lint — the "force-RLS with no read authz" gate.
+ *
+ * Every modeled table gets RLS enabled (`compileTableContract` emits `rls.enabled: true`), and the
+ * model is default-deny: no matching `can()` means no policy. So an EXPOSED table that declares no
+ * read ability is a superuser-drop landmine — it reads fine while a bypassing role (a superuser
+ * api) is in front, then returns empty for every app role the instant that role becomes
+ * RLS-subject. This is exactly how ~200 tables shipped RLS with zero policies/grants and would have
+ * gone dark on the credential split. The gate catches it statically, over the models, at CI time —
+ * long before the RLS is applied and superuser is dropped.
+ */
+
+import type { ModelDescriptor } from '@everystack/model';
+
+export interface ReadAuthzGap {
+  schema: string;
+  table: string;
+  message: string;
+}
+
+/** A model has a read path if it declares any `read` ability (public/owner/role/column) or `manage`
+ *  (which subsumes read, e.g. admin). Anything less means no app role can read the table. */
+function hasReadPath(model: ModelDescriptor): boolean {
+  return model.abilities.some((a) => a.action === 'read' || a.action === 'manage');
+}
+
+/**
+ * Find the exposed, RLS-enabled tables that declare no read ability — the tables that will read
+ * empty for every app role once it is RLS-subject. Private (non-exposed) tables are skipped: they
+ * are operational, not part of the data API, and are meant to be unreachable by app roles.
+ */
+export function findReadAuthzGaps(models: readonly ModelDescriptor[]): ReadAuthzGap[] {
+  const gaps: ReadAuthzGap[] = [];
+  for (const m of models) {
+    if (m.private) continue;
+    if (hasReadPath(m)) continue;
+    gaps.push({
+      schema: m.schema,
+      table: m.table,
+      message:
+        `${m.schema}.${m.table} is exposed and RLS-enabled but declares no read ability — every app role ` +
+        `reads empty once it is RLS-subject (e.g. after dropping a superuser api). Declare a read: ` +
+        `can('read') for public data, can('read', { owner: '<col>' }) for private, or mark the model ` +
+        `private() if it is not part of the data API.`,
+    });
+  }
+  return gaps;
+}

package/src/cli/aws.ts

@@ -5,6 +5,8 @@
  * Authentication uses the developer's IAM credentials (default credential chain).
  */
 
+import { isOpsHandlerCrash, extractCrashDiagnosis, formatOpsCrashReport } from './ops-diagnostics.js';
+
 let s3Client: InstanceType<typeof import('@aws-sdk/client-s3').S3Client> | null = null;
 let lambdaClient: InstanceType<typeof import('@aws-sdk/client-lambda').LambdaClient> | null = null;
 let kvsClient: InstanceType<typeof import('@aws-sdk/client-cloudfront-keyvaluestore').CloudFrontKeyValueStoreClient> | null = null;
@@ -26,9 +28,43 @@ async function getLambda(region: string) {
   return lambdaClient;
 }
 
+/**
+ * Lazily load an optional-peer AWS SDK client, turning a missing dependency into a one-line
+ * install remedy instead of a raw `ERR_MODULE_NOT_FOUND` at the call site. The `@aws-sdk/*`
+ * clients are optional peers of the CLI (not every consumer needs RDS, backup, …), so a consumer
+ * that never installed the one a given command needs gets guidance, not a cryptic crash. Takes a
+ * thunk so the `import()` stays a literal — statically analyzable and fully typed.
+ */
+export async function importOptionalAws<T>(
+  load: () => Promise<T>,
+  pkg: string,
+  feature: string,
+): Promise<T> {
+  try {
+    return await load();
+  } catch (err: unknown) {
+    const code = (err as { code?: string })?.code;
+    const message = String((err as { message?: string })?.message ?? err);
+    if (
+      code === 'ERR_MODULE_NOT_FOUND' ||
+      code === 'MODULE_NOT_FOUND' ||
+      /Cannot find (package|module)/.test(message)
+    ) {
+      throw new Error(
+        `${feature} needs ${pkg}, which isn't installed (it's an optional peer dependency). Install it:\n  pnpm add ${pkg}`,
+      );
+    }
+    throw err;
+  }
+}
+
 async function getRds(region: string) {
   if (!rdsClient) {
-    const { RDSClient } = await import('@aws-sdk/client-rds');
+    const { RDSClient } = await importOptionalAws(
+      () => import('@aws-sdk/client-rds'),
+      '@aws-sdk/client-rds',
+      'db:snapshot',
+    );
     rdsClient = new RDSClient({ region });
   }
   return rdsClient;
@@ -192,6 +228,24 @@ export async function resolveFunctionName(
   return candidates[0].name;
 }
 
+/**
+ * A read-only QueryRunner backed by the ops Lambda's `db:query` action — the
+ * credential-free introspection path. Every deployed-stage read verb
+ * (db:plan, db:apply --stage's descent probe, …) shares it: catalog SQL runs
+ * inside AWS on the operator connection, and the row set comes back over IAM.
+ * `db:query` refuses writes, so this can only read.
+ */
+export function lambdaQueryRunner(
+  region: string,
+  functionName: string,
+): (sql: string) => Promise<any[]> {
+  return async (sql: string) => {
+    const result: any = await invokeAction(region, functionName, 'db:query', { sql });
+    if (result?.error) throw new Error(`Query failed: ${result.error}`);
+    return result?.rows ?? [];
+  };
+}
+
 export async function invokeAction(
   region: string,
   functionName: string,
@@ -209,6 +263,7 @@ export async function invokeAction(
     client.send(new InvokeCommand({ FunctionName: fnName, Payload: encoded }));
 
   let response;
+  let invokedName = functionName;
   try {
     response = await invoke(functionName);
   } catch (err: unknown) {
@@ -224,6 +279,7 @@ export async function invokeAction(
       } catch {
         // Cache invalidation is best-effort
       }
+      invokedName = live;
       response = await invoke(live);
     } else {
       throw err;
@@ -234,13 +290,47 @@ export async function invokeAction(
     const errorBody = response.Payload
       ? JSON.parse(new TextDecoder().decode(response.Payload))
       : { errorMessage: 'Unknown Lambda error' };
-    throw new Error(errorBody.errorMessage || response.FunctionError);
+    const base = String(errorBody.errorMessage || response.FunctionError);
+    // A FunctionError means the handler threw uncaught. When the message is the opaque
+    // runtime/init crash (a broken top-level import — every action dies the same way), surface the
+    // real cause from CloudWatch instead of re-throwing "exit status 1".
+    if (isOpsHandlerCrash(base)) {
+      throw new Error(await diagnoseOpsCrash(region, invokedName, base));
+    }
+    throw new Error(base);
   }
 
   if (!response.Payload) return null;
   return JSON.parse(new TextDecoder().decode(response.Payload));
 }
 
+/** Recent CloudWatch log text for a function's log group (last `windowMs`, capped). */
+async function fetchRecentLogText(region: string, logGroupName: string, windowMs = 5 * 60_000): Promise<string> {
+  const { CloudWatchLogsClient, FilterLogEventsCommand } = await import('@aws-sdk/client-cloudwatch-logs');
+  const client = new CloudWatchLogsClient({ region });
+  const res = await client.send(new FilterLogEventsCommand({
+    logGroupName,
+    startTime: Date.now() - windowMs,
+    limit: 200,
+  }));
+  return (res.events ?? []).map((e) => e.message ?? '').join('\n');
+}
+
+/**
+ * Turn an opaque ops-handler runtime crash into a report naming the real cause: resolve the ops
+ * function's log group, read the recent lines, and extract the failing import. Best-effort — any
+ * failure (no log access, no group) falls back to the where-to-look hint.
+ */
+export async function diagnoseOpsCrash(region: string, functionName: string, baseMessage: string): Promise<string> {
+  try {
+    const logGroup = await resolveLogGroup(region, functionName);
+    const text = await fetchRecentLogText(region, logGroup);
+    return formatOpsCrashReport(baseMessage, logGroup, extractCrashDiagnosis(text));
+  } catch {
+    return formatOpsCrashReport(baseMessage);
+  }
+}
+
 /**
  * Resolve the CloudWatch log group for a Lambda function.
  *
@@ -421,7 +511,11 @@ export async function createRdsSnapshot(
   snapshotId: string,
 ): Promise<RdsSnapshotSummary> {
   const client = await getRds(region);
-  const { CreateDBSnapshotCommand } = await import('@aws-sdk/client-rds');
+  const { CreateDBSnapshotCommand } = await importOptionalAws(
+    () => import('@aws-sdk/client-rds'),
+    '@aws-sdk/client-rds',
+    'db:snapshot',
+  );
   const res = await client.send(new CreateDBSnapshotCommand({
     DBInstanceIdentifier: dbInstanceId,
     DBSnapshotIdentifier: snapshotId,
@@ -447,7 +541,11 @@ export async function describeRdsSnapshots(
   dbInstanceId: string,
 ): Promise<RdsSnapshotSummary[]> {
   const client = await getRds(region);
-  const { DescribeDBSnapshotsCommand } = await import('@aws-sdk/client-rds');
+  const { DescribeDBSnapshotsCommand } = await importOptionalAws(
+    () => import('@aws-sdk/client-rds'),
+    '@aws-sdk/client-rds',
+    'db:snapshots',
+  );
   const summaries: RdsSnapshotSummary[] = [];
   let marker: string | undefined;
   do {

package/src/cli/backfill.ts

@@ -8,7 +8,7 @@
  * unlike `schema_log`, which stays a write-only memoir. The distrust that
  * killed the migration tape doesn't transfer: nothing is numbered, nothing
  * is watermarked, and identity is the NORMALIZED CONTENT HASH (the same
- * comment-insensitive, dollar-quote-aware lexing as db/sql) — renaming a
+ * comment-insensitive, dollar-quote-aware lexing as the reconciler) — renaming a
  * file or editing a comment is a no-op, never a re-run.
  *
  * One-shot jobs are IMMUTABLE. A file whose name already ran on this