@everystack/cli 0.3.22 → 0.3.25

package/README.md

@@ -220,8 +220,12 @@ everystack db:reconcile                 # deploy the derived layer (functions/vi
 everystack db:fingerprint               # content-address the live base schema vs the Models — MATCH/MISMATCH
 everystack db:sync                      # make the database match your checkout — state + compute, one verb (dev DBs)
 everystack db:diff                      # the state edge between two declared states — no DB, CI-pure; reverse = computed rollback
-everystack db:plan | db:apply           # mint a reviewable plan against a target, apply it fingerprint-verified at both ends + the fast-forward rule (checkout must descend from the commit declaring the target's state)
+everystack db:plan | db:apply           # mint a reviewable plan against a target, apply it fingerprint-verified at both ends + the fast-forward rule (checkout must descend from the commit declaring the target's state); destructive plans: --confirm + snapshot + approver set when declared
 everystack db:check                     # the CI gate per PR: merged declared state composes + artifacts match regeneration (+ ephemeral from-scratch build w/ a scratch PG)
+everystack db:approvers                 # declare who can DESTROY: the stage's destructive-approver set (SSM, STS-verified at apply)
+everystack db:backfill                  # one-shot data jobs in db/backfills/*.sql — content-hash identity, own per-DB record, never a schema side effect
+everystack db:template:refresh          # (re)build the dev template from the declared state + your db:seed script — never a data copy
+everystack db:branch                    # per-branch dev DB minted from the template, keyed on the git branch (--list / --prune --confirm)
 
 # Security
 everystack db:doctor                    # is the DB least-privilege + RLS-subject?

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.3.22",
+  "version": "0.3.25",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -78,7 +78,7 @@
     "structured-headers": "1.0.1",
     "tsx": "4.21.0",
     "typescript": "5.9.3",
-    "@everystack/model": "0.3.5"
+    "@everystack/model": "0.3.6"
   },
   "peerDependencies": {
     "@everystack/server": ">=0.1.0",
@@ -93,7 +93,8 @@
     "expo-updates": "55.0.21",
     "postgres": "3.4.9",
     "react": "19.2.0",
-    "react-native": "0.83.6"
+    "react-native": "0.83.6",
+    "@aws-sdk/client-sts": "3.1053.0"
   },
   "peerDependenciesMeta": {
     "postgres": {
@@ -131,6 +132,9 @@
     },
     "@aws-sdk/s3-request-presigner": {
       "optional": true
+    },
+    "@aws-sdk/client-sts": {
+      "optional": true
     }
   },
   "devDependencies": {
@@ -148,7 +152,8 @@
     "drizzle-orm": "0.41.0",
     "jest": "29.7.0",
     "react": "19.2.0",
-    "ts-jest": "29.4.9"
+    "ts-jest": "29.4.9",
+    "@aws-sdk/client-sts": "3.1053.0"
   },
   "scripts": {
     "test": "jest",

package/src/cli/apply-authority.ts

@@ -0,0 +1,150 @@
+/**
+ * apply-authority — destructive applies are role-gated to a declared
+ * approver set, enforced against the caller's REAL identity (brick 9 of the
+ * migration-authority design, decisions 11 + 12).
+ *
+ * "CTO drops tables; everyone else creates functions and views." The
+ * approver set lives STAGE-SIDE — an SSM parameter in the stage's namespace,
+ * admin-writable only — never in the repo, where a checkout edit would
+ * defeat it (a repo-declared list is a convention wearing a gate's clothes).
+ * The chokepoint resolves the caller's real identity via STS — the same
+ * pattern as the `db:psql` break-glass, where the IAM read IS the gate.
+ *
+ * The gate engages on DECLARATION: a stage that never declared approvers
+ * keeps the ceremony (explicit confirmation + snapshot, always) but is not
+ * identity-gated — the migration-friendly posture, stated honestly. A
+ * declared-but-EMPTY set means "no one": destructive applies are disabled
+ * on that stage until the set says otherwise.
+ *
+ * The refusal is the two-person flow's handoff: it names who CAN approve,
+ * and `plan_ref` already pins the exact reviewed artifact — a dev mints the
+ * destructive plan, an approver applies that same file.
+ *
+ * The decision core is pure; the AWS edge (SSM + STS) is thin and injected
+ * by the command shell.
+ */
+
+/** The SSM parameter holding the approver set — a JSON array of entries. */
+export function approverParamName(appName: string, stage: string): string {
+  return `/everystack/${appName}/${stage}/db-approvers`;
+}
+
+/**
+ * Does an approver entry match a caller ARN? Exact ARN match, or a match on
+ * any path segment after the first slash — so the set can hold plain names:
+ * `ty` matches `arn:aws:iam::123:user/ty` and the session name in
+ * `arn:aws:sts::123:assumed-role/Admin/ty`. Segment (not suffix) matching:
+ * `ty` never matches `user/notty`.
+ */
+export function approverMatches(entry: string, identityArn: string): boolean {
+  if (entry === identityArn) return true;
+  const segments = identityArn.split('/').slice(1);
+  return segments.includes(entry);
+}
+
+export interface AuthorityInputs {
+  /** The declared approver set; null = the stage never declared one. */
+  approvers: string[] | null;
+  /** The caller's resolved identity (STS ARN); null = unresolvable. */
+  identity: string | null;
+}
+
+export type AuthorityVerdict =
+  | { ok: true; reason: 'approved' | 'undeclared' }
+  | { ok: false; reason: string };
+
+/** The pure decision core for a DESTRUCTIVE apply. Non-destructive plans never reach it. */
+export function checkDestructiveAuthority({ approvers, identity }: AuthorityInputs): AuthorityVerdict {
+  if (approvers === null) return { ok: true, reason: 'undeclared' };
+  if (approvers.length === 0) {
+    return {
+      ok: false,
+      reason:
+        'the declared approver set is EMPTY — destructive applies are disabled on this stage. ' +
+        'Declare approvers with `everystack db:approvers --set <arn-or-name,...>` (admin-writable parameter).',
+    };
+  }
+  if (identity === null) {
+    return {
+      ok: false,
+      reason:
+        'this stage declares a destructive-approver set, so the apply must be identity-verified — ' +
+        'and your AWS identity could not be resolved (credentials?). Fix the credentials or hand the plan to an approver.',
+    };
+  }
+  if (approvers.some((entry) => approverMatches(entry, identity))) {
+    return { ok: true, reason: 'approved' };
+  }
+  return {
+    ok: false,
+    reason:
+      `you are ${identity}, and the declared approvers are: ${approvers.join(', ')}. ` +
+      'Hand the plan file to an approver — they review and apply the exact same artifact (its plan_ref pins it).',
+  };
+}
+
+// ---------------------------------------------------------------------------
+// The thin AWS edge (lazy imports, injected by the command shell).
+// ---------------------------------------------------------------------------
+
+/** Read the approver set from SSM. Null when the parameter does not exist. */
+export async function readApproverParam(region: string, appName: string, stage: string): Promise<string[] | null> {
+  const { SSMClient, GetParameterCommand } = await import('@aws-sdk/client-ssm');
+  const client = new SSMClient({ region });
+  try {
+    const result = await client.send(new GetParameterCommand({ Name: approverParamName(appName, stage) }));
+    return parseApproverValue(result.Parameter?.Value ?? '');
+  } catch (err: any) {
+    if (err?.name === 'ParameterNotFound') return null;
+    throw err;
+  }
+}
+
+/** The stored form is a JSON array; a bare comma-separated string is accepted on read. */
+export function parseApproverValue(value: string): string[] {
+  const trimmed = value.trim();
+  if (trimmed === '') return [];
+  if (trimmed.startsWith('[')) {
+    const parsed = JSON.parse(trimmed);
+    if (!Array.isArray(parsed) || parsed.some((e) => typeof e !== 'string')) {
+      throw new Error('db-approvers parameter must be a JSON array of strings');
+    }
+    return parsed.map((e: string) => e.trim()).filter(Boolean);
+  }
+  return trimmed.split(',').map((e) => e.trim()).filter(Boolean);
+}
+
+/** Declare (or replace) the approver set. Stored as a JSON array. */
+export async function writeApproverParam(region: string, appName: string, stage: string, approvers: string[]): Promise<void> {
+  const { SSMClient, PutParameterCommand } = await import('@aws-sdk/client-ssm');
+  const client = new SSMClient({ region });
+  await client.send(new PutParameterCommand({
+    Name: approverParamName(appName, stage),
+    Value: JSON.stringify(approvers),
+    Type: 'String',
+    Overwrite: true,
+  }));
+}
+
+/** Remove the declaration entirely — the stage returns to the undeclared (ceremony-only) posture. */
+export async function removeApproverParam(region: string, appName: string, stage: string): Promise<void> {
+  const { SSMClient, DeleteParameterCommand } = await import('@aws-sdk/client-ssm');
+  const client = new SSMClient({ region });
+  try {
+    await client.send(new DeleteParameterCommand({ Name: approverParamName(appName, stage) }));
+  } catch (err: any) {
+    if (err?.name !== 'ParameterNotFound') throw err;
+  }
+}
+
+/** The caller's real identity — the STS ARN. Null when credentials can't answer. */
+export async function resolveCallerIdentity(region: string): Promise<string | null> {
+  const { STSClient, GetCallerIdentityCommand } = await import('@aws-sdk/client-sts');
+  const client = new STSClient({ region });
+  try {
+    const result = await client.send(new GetCallerIdentityCommand({}));
+    return result.Arn ?? null;
+  } catch {
+    return null;
+  }
+}

package/src/cli/backfill.ts

@@ -0,0 +1,231 @@
+/**
+ * backfill — one-shot data jobs in their own lane (brick 8 of the
+ * migration-authority design, decision 15).
+ *
+ * Schema is fingerprint-governed; DATA has no live authority to fingerprint,
+ * so the backfill lane is the one place a per-database ledger is correct:
+ * `everystack.backfill_log` records what ran here, and the runner READS it —
+ * unlike `schema_log`, which stays a write-only memoir. The distrust that
+ * killed the migration tape doesn't transfer: nothing is numbered, nothing
+ * is watermarked, and identity is the NORMALIZED CONTENT HASH (the same
+ * comment-insensitive, dollar-quote-aware lexing as db/sql) — renaming a
+ * file or editing a comment is a no-op, never a re-run.
+ *
+ * One-shot jobs are IMMUTABLE. A file whose name already ran on this
+ * database in a different form is BLOCKED, loudly, with the fix named: a
+ * genuinely new job gets a new file; a cosmetic-but-not-comment edit gets
+ * `--mark-applied`. Never silently re-run, never silently skipped. A prior
+ * FAILED run is a retry, not an edit — it stays pending.
+ *
+ * Backfills never run as a side effect of a schema apply (expand →
+ * backfill → contract is a sequence of deliberate acts); the schema verbs
+ * only SURFACE pending backfills as notices. Long-running smells get an
+ * advisory naming the jobs lane — advisory only, taste is not enforced.
+ */
+
+import { createHash } from 'node:crypto';
+import { lex, normalizeSql, splitSqlStatements, type SourceFile } from './derived-source.js';
+import { escapeLiteral } from './derived-apply.js';
+import type { QueryRunner } from './authz-contract.js';
+
+// ---------------------------------------------------------------------------
+// Identity + advisories (pure).
+// ---------------------------------------------------------------------------
+
+/** The job's identity: sha256 over the comment-insensitive canonical form. */
+export function backfillIdentity(sql: string): string {
+  return createHash('sha256').update(normalizeSql(sql)).digest('hex');
+}
+
+/**
+ * The statement with comments dropped and quoted content blanked — a shape
+ * regexes can interrogate without being fooled by `'WHERE it hurts'`.
+ */
+function sqlSkeleton(sql: string): string {
+  let out = '';
+  lex(sql, (ch, state) => {
+    if (state === 'line-comment' || state === 'block-comment') return;
+    if (state === 'single' || state === 'dollar') return; // quoted content blanked
+    out += ch;
+  });
+  return out;
+}
+
+/** Advisory only — an unbounded data pass belongs in the jobs lane, but taste never blocks. */
+export function backfillAdvisories(sql: string): string[] {
+  const advisories: string[] = [];
+  for (const statement of splitSqlStatements(sql)) {
+    const skeleton = sqlSkeleton(statement);
+    const verb = skeleton.match(/^\s*(UPDATE|DELETE)\b/i)?.[1];
+    if (verb && !/\bWHERE\b/i.test(skeleton)) {
+      advisories.push(
+        `${verb.toUpperCase()} without WHERE — an unbounded pass over the table. Fine for small tables; for big ones, batch it or run it as a job (@everystack/jobs).`,
+      );
+    }
+  }
+  return advisories;
+}
+
+// ---------------------------------------------------------------------------
+// Planning (pure).
+// ---------------------------------------------------------------------------
+
+export interface BackfillRecord {
+  name: string;
+  identity: string;
+  outcome: string; // 'applied' | 'marked-applied' | 'failed'
+}
+
+export interface BackfillPlan {
+  applied: { file: string; identity: string }[];
+  /** To run, in file-name order. */
+  pending: { file: string; identity: string; sql: string; advisories: string[] }[];
+  /** Edited-after-run — refused with the fix named. */
+  blocked: { file: string; identity: string; reason: string }[];
+}
+
+const baseName = (file: string): string => file.split('/').pop() ?? file;
+const counts = (outcome: string): boolean => outcome === 'applied' || outcome === 'marked-applied';
+
+export function planBackfills(files: SourceFile[], log: BackfillRecord[]): BackfillPlan {
+  const appliedIdentities = new Set(log.filter((r) => counts(r.outcome)).map((r) => r.identity));
+  const appliedByName = new Map<string, string>(); // name → identity that ran under it
+  for (const r of log) {
+    if (counts(r.outcome)) appliedByName.set(r.name, r.identity);
+  }
+
+  const plan: BackfillPlan = { applied: [], pending: [], blocked: [] };
+  for (const file of [...files].sort((a, b) => baseName(a.file).localeCompare(baseName(b.file)))) {
+    const identity = backfillIdentity(file.sql);
+    if (appliedIdentities.has(identity)) {
+      plan.applied.push({ file: file.file, identity });
+      continue;
+    }
+    const ranAs = appliedByName.get(baseName(file.file));
+    if (ranAs !== undefined && ranAs !== identity) {
+      plan.blocked.push({
+        file: file.file,
+        identity,
+        reason:
+          `${baseName(file.file)} already ran on this database in a different form — one-shot jobs are immutable. ` +
+          'A genuinely new job gets a NEW file name; if this edit changed nothing the job does, record it: --mark-applied.',
+      });
+      continue;
+    }
+    plan.pending.push({ file: file.file, identity, sql: file.sql, advisories: backfillAdvisories(file.sql) });
+  }
+  return plan;
+}
+
+// ---------------------------------------------------------------------------
+// The record + the runner.
+// ---------------------------------------------------------------------------
+
+export const ENSURE_BACKFILL_SQL: string[] = [
+  'CREATE SCHEMA IF NOT EXISTS everystack',
+  `CREATE TABLE IF NOT EXISTS everystack.backfill_log (
+  id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+  applied_at timestamptz NOT NULL DEFAULT now(),
+  identity text NOT NULL,
+  name text NOT NULL,
+  actor text,
+  git_ref text,
+  sql text NOT NULL,
+  duration_ms integer,
+  outcome text NOT NULL
+)`,
+];
+
+export interface BackfillLogEntry {
+  identity: string;
+  name: string;
+  sql: string;
+  outcome: string;
+  actor?: string | null;
+  gitRef?: string | null;
+  durationMs?: number;
+}
+
+export function renderBackfillLogInsert(entry: BackfillLogEntry): string {
+  const opt = (v: string | null | undefined): string => (v == null ? 'NULL' : escapeLiteral(v));
+  const duration = entry.durationMs === undefined ? 'NULL' : String(Math.round(entry.durationMs));
+  return `INSERT INTO everystack.backfill_log (identity, name, actor, git_ref, sql, duration_ms, outcome)
+VALUES (${escapeLiteral(entry.identity)}, ${escapeLiteral(entry.name)}, ${opt(entry.actor)}, ${opt(entry.gitRef)}, ${escapeLiteral(entry.sql)}, ${duration}, ${escapeLiteral(entry.outcome)})`;
+}
+
+/** The applied record for this database. Empty when the lane has never run here. */
+export async function readBackfillLog(runner: QueryRunner): Promise<BackfillRecord[]> {
+  const rows = await runner(`
+    SELECT name, identity, outcome FROM everystack.backfill_log ORDER BY id
+  `).catch((err: any) => {
+    if (/relation .*backfill_log.* does not exist|schema .* does not exist/i.test(err.message ?? '')) return [];
+    throw err;
+  });
+  return rows as BackfillRecord[];
+}
+
+export interface BackfillRunOptions {
+  actor?: string | null;
+  gitRef?: string | null;
+  /** Injectable clock for tests. */
+  now?: () => number;
+}
+
+export interface BackfillRunResult {
+  ran: { file: string; identity: string; durationMs: number }[];
+  /** The first failure, if any — the run STOPS there (later files may depend on it). */
+  failed?: { file: string; identity: string; error: string };
+}
+
+/**
+ * Run the pending jobs in order, each as its own single batch (one implicit
+ * transaction per job — a failed job rolls back alone and is recorded as
+ * `failed`, staying pending for the retry). Blocked files never execute.
+ */
+export async function executeBackfills(
+  runner: QueryRunner,
+  plan: BackfillPlan,
+  options: BackfillRunOptions = {},
+): Promise<BackfillRunResult> {
+  const now = options.now ?? Date.now;
+  const result: BackfillRunResult = { ran: [] };
+  if (plan.pending.length === 0) return result;
+
+  await runner(ENSURE_BACKFILL_SQL.join(';\n'));
+  for (const job of plan.pending) {
+    const name = baseName(job.file);
+    const started = now();
+    try {
+      await runner(job.sql);
+      const durationMs = now() - started;
+      await runner(renderBackfillLogInsert({
+        identity: job.identity, name, sql: job.sql, outcome: 'applied',
+        actor: options.actor, gitRef: options.gitRef, durationMs,
+      }));
+      result.ran.push({ file: job.file, identity: job.identity, durationMs });
+    } catch (err: any) {
+      await runner(renderBackfillLogInsert({
+        identity: job.identity, name, sql: `-- FAILED: ${err.message}\n${job.sql}`, outcome: 'failed',
+        actor: options.actor, gitRef: options.gitRef, durationMs: now() - started,
+      }));
+      result.failed = { file: job.file, identity: job.identity, error: err.message };
+      return result;
+    }
+  }
+  return result;
+}
+
+/** The adopt hatch: record a job as applied WITHOUT running it (mirrors --baseline). */
+export async function markBackfillApplied(
+  runner: QueryRunner,
+  file: SourceFile,
+  options: BackfillRunOptions = {},
+): Promise<string> {
+  const identity = backfillIdentity(file.sql);
+  await runner(ENSURE_BACKFILL_SQL.join(';\n'));
+  await runner(renderBackfillLogInsert({
+    identity, name: baseName(file.file), sql: file.sql, outcome: 'marked-applied',
+    actor: options.actor, gitRef: options.gitRef,
+  }));
+  return identity;
+}