@everystack/cli 0.3.25 → 0.3.28

package/README.md

@@ -226,6 +226,7 @@ everystack db:approvers                 # declare who can DESTROY: the stage's d
 everystack db:backfill                  # one-shot data jobs in db/backfills/*.sql — content-hash identity, own per-DB record, never a schema side effect
 everystack db:template:refresh          # (re)build the dev template from the declared state + your db:seed script — never a data copy
 everystack db:branch                    # per-branch dev DB minted from the template, keyed on the git branch (--list / --prune --confirm)
+everystack db:fork                      # fork a deployed stage's DB into a feature stage (backup → presigned hand-off → restore); production is never a target
 
 # Security
 everystack db:doctor                    # is the DB least-privilege + RLS-subject?

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.3.25",
+  "version": "0.3.28",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",

package/src/cli/commands/db-fork.ts

@@ -0,0 +1,139 @@
+/**
+ * `everystack db:fork` — fork one stage's database into another
+ * (branch-stages plan, brick 2; docs/plans/branch-stages.md).
+ *
+ *   db:fork --from-stage staging --stage feat-foo --confirm [--backup <id>]
+ *
+ * Three moves, all existing surfaces: back up the source (or reuse a named
+ * backup), presign the dump with the OPERATOR's credentials — the presigned
+ * URL is the whole cross-stage hand-off, no bucket policies, no IAM
+ * coupling, expires in an hour — and invoke the target's `db:restore` with
+ * the URL. The target stage must already be deployed (`sst deploy --stage
+ * feat-foo`); the fork replaces its database with the source's data, and
+ * the branch's own schema changes then land through the front door
+ * (`db:plan` → `db:apply` — the forked state is one a main commit declares,
+ * so the fast-forward rule composes for free).
+ *
+ * Production is never a fork target, flat — that is disaster recovery
+ * (`db:restore`), not a fork. Forking FROM production carries the PII
+ * data-governance warning. Teardown is the stage's own: `sst remove`.
+ */
+
+import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
+import { invokeAction, presignGet } from '../aws.js';
+import { keyForId, parseBackupRef, isProductionTier, crossStageGuard } from '../backup.js';
+import { step, success, fail, info, warn } from '../output.js';
+
+export type ForkGuardVerdict =
+  | { ok: true; piiWarning: string | null }
+  | { ok: false; reason: string };
+
+/** The pure gate — see the module docblock for the decisions it enforces. */
+export function checkForkGuards(input: { from?: string; stage?: string; confirm: boolean }): ForkGuardVerdict {
+  const { from, stage, confirm } = input;
+  if (!from || from === 'true') {
+    return { ok: false, reason: 'db:fork needs the source: --from-stage <name> (the stage whose data you are forking).' };
+  }
+  if (!stage || stage === 'true') {
+    return { ok: false, reason: 'db:fork needs the target: --stage <name> (an already-deployed feature stage; `sst deploy --stage <name>` first).' };
+  }
+  if (from === stage) {
+    return { ok: false, reason: 'source and target are the same stage — restoring a stage onto itself is db:restore, not a fork.' };
+  }
+  if (isProductionTier(stage)) {
+    return { ok: false, reason: `"${stage}" is production-tier — a fork never overwrites production. Restoring production is disaster recovery: db:restore, with its own ceremony.` };
+  }
+  if (!confirm) {
+    return { ok: false, reason: `db:fork REPLACES ${stage}'s database with ${from}'s data — confirm with --confirm.` };
+  }
+  const pii = crossStageGuard(from, stage);
+  return { ok: true, piiWarning: pii.requiresConfirm ? (pii.warning ?? null) : null };
+}
+
+export async function dbForkCommand(flags: Record<string, string>): Promise<void> {
+  const verdict = checkForkGuards({
+    from: flags['from-stage'],
+    stage: flags.stage,
+    confirm: flags.confirm === 'true',
+  });
+  if (!verdict.ok) {
+    fail(verdict.reason);
+    process.exit(1);
+  }
+  if (verdict.piiWarning) warn(verdict.piiWarning.replace(/pass --confirm to proceed\.?$/i, 'confirmed.'));
+
+  const from = flags['from-stage'];
+  const target = flags.stage;
+
+  step(`Resolving source stage ${from}...`);
+  let source: CliConfig;
+  try {
+    source = await resolveConfig(from);
+  } catch (err: any) {
+    fail(err.message);
+    process.exit(1);
+  }
+  if (!source.backupsBucket) {
+    fail(`Stage ${from} has no backupsBucket in its deployed config — logical backups are the fork's transport (docs/backups.md).`);
+    process.exit(1);
+  }
+
+  // Move 1: a backup of the source — fresh by default, or a named one.
+  let backupId = flags.backup;
+  if (backupId && backupId !== 'true') {
+    const ref = parseBackupRef(backupId);
+    if (!ref || !keyForId(backupId)) {
+      fail(`Malformed backup id: ${backupId}`);
+      process.exit(1);
+    }
+    if (ref.stage !== from) {
+      fail(`Backup ${backupId} belongs to stage "${ref.stage}", not --from-stage ${from} — a fork's data names its source honestly.`);
+      process.exit(1);
+    }
+    info(`Reusing backup ${backupId}.`);
+  } else {
+    step(`Backing up ${from} (pg_dump → S3; may take a while for large databases)...`);
+    const result: any = await invokeAction(source.region, opsFunction(source), 'db:backup', { stage: from });
+    if (result?.error) {
+      fail(`Source backup failed: ${result.error}`);
+      process.exit(1);
+    }
+    backupId = result.id;
+    info(`Backup ${backupId}.`);
+  }
+
+  // Move 2: the presigned hand-off — the operator's credentials ARE the authorization.
+  step('Presigning the dump (1 hour)...');
+  let url: string;
+  try {
+    url = await presignGet(source.region, source.backupsBucket, keyForId(backupId)!, 3600);
+  } catch (err: any) {
+    fail(`Presign failed: ${err.message}`);
+    process.exit(1);
+  }
+
+  // Move 3: restore into the target over the URL.
+  step(`Resolving target stage ${target}...`);
+  let targetConfig: CliConfig;
+  try {
+    targetConfig = await resolveConfig(target);
+  } catch (err: any) {
+    fail(`${err.message}\nDeploy the feature stage first: sst deploy --stage ${target}`);
+    process.exit(1);
+  }
+  step(`Restoring into ${target} (streamed; replaces its database)...`);
+  const restored: any = await invokeAction(targetConfig.region, opsFunction(targetConfig), 'db:restore', {
+    url,
+    confirm: true,
+  });
+  if (restored?.error) {
+    fail(`Restore failed: ${restored.error}`);
+    process.exit(1);
+  }
+
+  success(`Forked ${from} → ${target} (backup ${backupId}).`);
+  info('Next:');
+  info(`  everystack db:fingerprint --stage ${target}     # where the fork is (a state some main commit declares)`);
+  info(`  everystack db:plan --stage ${target}            # your branch's edge on top — descent composes for free`);
+  info(`  sst remove --stage ${target}                    # teardown takes the database with it`);
+}

package/src/cli/commands/db-sync.ts

@@ -169,6 +169,16 @@ export function buildSyncReport(run: SyncRun): string[] {
     lines.push(`· ${state.classified.notices.length} notice(s) — manual follow-ups, nothing executed for them`);
   }
 
+  // The base fingerprint is a STATE-layer fact: both endpoints are sampled around
+  // the state apply, before compute runs. Print it here, adjacent to the state
+  // section, so an empty→full move never reads as if the compute step caused it —
+  // the derived layer (functions/views/matviews) is not fingerprinted at all.
+  lines.push(
+    state.toFingerprint === state.fromFingerprint
+      ? `base fingerprint: ${short(state.toFingerprint)} (unchanged)`
+      : `base fingerprint: ${short(state.fromFingerprint)} → ${short(state.toFingerprint)} (across the state apply)`,
+  );
+
   // Compute.
   if (compute === null) {
     lines.push('· compute: no db/sql directory — derived layer not declared, skipped');
@@ -181,12 +191,7 @@ export function buildSyncReport(run: SyncRun): string[] {
     }
   }
 
-  // Fingerprint.
-  lines.push(
-    state.toFingerprint === state.fromFingerprint
-      ? `fingerprint: ${short(state.toFingerprint)} (unchanged)`
-      : `fingerprint: ${short(state.fromFingerprint)} → ${short(state.toFingerprint)}`,
-  );
+  // Verdict.
   if (run.fingerprintMatch) {
     lines.push('MATCH — the database is the state your checkout declares');
   } else if (run.stateConverged) {

package/src/cli/commands/runbook.ts

@@ -36,9 +36,10 @@ function cliVersion(): string {
 }
 
 /** Import the app's Model barrel for the data-model chapter. Absent or unloadable → honest warning, no chapter. */
-async function loadDataModel(root: string, hasIndex: boolean): Promise<{ dataModel: string | null; warning: string | null }> {
-  if (!hasIndex) return { dataModel: null, warning: null };
-  const barrel = path.join(root, 'models', 'index.ts');
+async function loadDataModel(root: string, models: { dir: string | null; hasIndex: boolean }): Promise<{ dataModel: string | null; warning: string | null }> {
+  if (!models.hasIndex || !models.dir) return { dataModel: null, warning: null };
+  // The detected home (db/models preferred, legacy models/ fallback) — never hardcoded.
+  const barrel = path.join(root, models.dir, 'index.ts');
   try {
     const mod = await import(pathToFileURL(barrel).href);
     const models = mod.models ?? mod.default;
@@ -64,7 +65,7 @@ export interface RunbookRun {
 /** Result-producing core, shared by the command, `--check`, and the `audit` capstone. */
 export async function runRunbook(root: string): Promise<RunbookRun> {
   const reality = detectProjectReality(root);
-  const { dataModel, warning } = await loadDataModel(reality.root, reality.models.hasIndex);
+  const { dataModel, warning } = await loadDataModel(reality.root, reality.models);
   const outPath = path.join(reality.root, OUT_REL);
   const oldText = fs.existsSync(outPath) ? fs.readFileSync(outPath, 'utf8') : null;
   const result = generateRunbook(reality, { cliVersion: cliVersion(), dataModel }, oldText ?? undefined);

package/src/cli/index.ts

@@ -21,6 +21,7 @@ import { dbCheckCommand } from './commands/db-check.js';
 import { dbApproversCommand } from './commands/db-approvers.js';
 import { dbBackfillCommand } from './commands/db-backfill.js';
 import { dbTemplateRefreshCommand, dbBranchCommand } from './commands/db-branch.js';
+import { dbForkCommand } from './commands/db-fork.js';
 import { dbSnapshotCommand, dbSnapshotsCommand } from './commands/db-snapshot.js';
 import { dbBackupProbeCommand, dbBackupCommand, dbBackupsCommand, dbRestoreCommand, dbBackupDownloadCommand } from './commands/db-backup.js';
 import { consoleCommand } from './commands/console.js';
@@ -215,6 +216,9 @@ async function main() {
     case 'db:branch':
       await dbBranchCommand(flags);
       break;
+    case 'db:fork':
+      await dbForkCommand(flags);
+      break;
     case 'db:authz:pull':
       await dbAuthzPullCommand(flags);
       break;
@@ -348,6 +352,7 @@ Usage:
   everystack db:backfill       [--database-url <url>] [--dir db/backfills] [--apply] [--mark-applied <file.sql>] [--json]  One-shot data jobs in their own lane: plan shows applied (by CONTENT identity — renames/comment edits are no-ops) / pending (in order, unbounded-pass advisories) / blocked (a name that already ran in a different form — one-shot jobs are immutable). --apply runs each pending job as its own transaction, recorded in everystack.backfill_log (a failure rolls back alone, is recorded, stops the run); --mark-applied records without running. Never runs as a schema side effect; direct connection required
   everystack db:template:refresh [--database-url <url>] [--models <barrel>] [--sql-dir db/sql] [--seed "<cmd>"] [--no-seed]  (Re)build the dev template <base>_tpl FROM THE DECLARED STATE (both layers, fingerprint MATCH bar) + seed-as-code (the app's db:seed script, run with DATABASE_URL pointed at the template; --seed overrides). All or nothing: a failed build/seed leaves NO template. Never a data copy
   everystack db:branch         [--list | --drop --confirm | --prune --confirm] [--database-url <url>]  Mint (or find) the current git branch's database from the template (CREATE DATABASE … TEMPLATE — schema, authz, derived layer, and seed rows inherited), print its DATABASE_URL, then db:sync evolves it with the checkout. --list maps every branch DB to its branch; --prune drops the ones whose branch is gone (never guesses: unknown mappings are kept)
+  everystack db:fork           --from-stage <src> --stage <target> --confirm [--backup <id>]  Fork one DEPLOYED stage's database into another: back up the source (or reuse --backup <id>), presign the dump (the operator's credentials ARE the cross-stage authorization; expires in 1h), restore into the target via its ops Lambda. Production is never a target (that's db:restore); forking FROM production warns about PII; the branch's schema edge then lands via db:plan → db:apply (descent composes). Teardown: sst remove --stage <target>
   everystack db:authz:pull     [--stage <name>] [--dir authz]  Introspect live authz (rls/grants/policies/secdef) → reviewable contract files
   everystack db:authz:diff     [--stage <name>] [--dir authz]  Validate the live DB against the committed contract (non-zero exit on drift)
   everystack db:authz:test     [--stage <name>] [--dir authz]  Red-team enforcement: SET ROLE + attempt per role/table/command (non-zero exit on a hole)

package/src/cli/runbook/sections.ts

@@ -150,9 +150,14 @@ function extend(r: ProjectReality): Block {
   ];
   if (r.models.dir) {
     lines.push(
-      '- **Add a table or field:** edit the Model in `models/`, run `everystack db:generate`,',
-      '  review the emitted migration, then `everystack db:migrate`. Never hand-write a SQL',
-      '  migration, never edit `db/schema.ts` — after any change `db:generate` must be a clean no-op.',
+      `- **Add a table or field:** edit the Model in \`${r.models.dir}/\`, run \`everystack db:sync\` —`,
+      '  the dev database follows your checkout (state + authz + derived, verified by fingerprint).',
+      '  Never hand-write a SQL migration, never edit generated schema — CI runs `everystack db:check`',
+      '  and refuses artifacts that do not match regeneration.',
+      '- **Ship a schema change:** protected stages take plans, not syncs: `everystack db:plan`',
+      '  (a reviewable edge, fingerprint-pinned at both ends) → review → `everystack db:apply`.',
+      '- **Move data:** one-shot jobs in `db/backfills/*.sql`, run deliberately via',
+      '  `everystack db:backfill --apply` — never as a schema side effect.',
     );
     if (r.migrationMode === 'transitional') {
       lines.push(
@@ -165,8 +170,9 @@ function extend(r: ProjectReality): Block {
     );
   } else if (r.hasDatabase) {
     lines.push(
-      '- **Schema changes:** migrate to Models (`everystack db:pull` writes `models/` from the',
-      '  live database), then the flow is: edit Model → `everystack db:generate` → `db:migrate`.',
+      '- **Schema changes:** migrate to Models (`everystack db:pull` writes `db/models/` from the',
+      '  live database — COMMIT that state before evolving it), then the loop is: edit Model →',
+      '  `everystack db:sync` (dev) / `db:plan` → `db:apply` (protected stages).',
     );
   }
   if (r.handlers.api) {
@@ -193,17 +199,24 @@ function extend(r: ProjectReality): Block {
 function database(r: ProjectReality): Block {
   const flow =
     r.migrationMode === 'everystack'
-      ? ['- `everystack db:generate` — compile Models → next migration (schema + RLS + grants)']
+      ? [
+          '- `everystack db:sync` — dev edit loop: the database follows your checkout (state + authz + derived)',
+          '- `everystack db:check` — the CI gate: merged Models compose, generated artifacts match regeneration',
+          '- `everystack db:plan` → `everystack db:apply` — protected stages: reviewable, fingerprint-verified edges',
+          '- `everystack db:branch` / `db:template:refresh` — per-git-branch dev databases from a seeded template',
+          '- `everystack db:backfill` — one-shot data jobs (`db/backfills/*.sql`), own record, run deliberately',
+        ]
       : r.migrationMode === 'transitional'
         ? [
             '- **Mid-migration to Models:** the journal is still hand-authored. Target flow is',
-            '  `everystack db:generate`; it must produce a clean, empty data diff before the',
-            '  generated flow takes over. Until then, review hand-written SQL like the liability it is.',
+            '  `everystack db:sync` (dev) / `db:plan` → `db:apply` (protected); `everystack db:generate`',
+            '  must produce a clean, empty data diff before the generated flow takes over. Until then,',
+            '  review hand-written SQL like the liability it is.',
           ]
         : [
             '- Migrations are hand-authored (drizzle-kit). The Models on-ramp: `everystack db:pull`',
-            '  writes `models/` from the live database; from there `everystack db:generate` compiles',
-            '  migrations (schema + RLS + grants) and hand-written SQL retires.',
+            '  writes `db/models/` from the live database (COMMIT that state before evolving it); from',
+            '  there `everystack db:sync` / `db:plan` → `db:apply` take over and hand-written SQL retires.',
           ];
   return gen('database', [
     '## Database operations',

package/src/cli/schema-fingerprint.ts

@@ -27,6 +27,13 @@
  *  - FUNCTIONS are excluded — they belong to the derived layer, whose own
  *    content hashes (derived-introspect) cover them. Base fingerprint + derived
  *    manifest together are the full schema state.
+ *  - Expressions (column defaults, check predicates, partial-index WHERE) hash
+ *    through the SAME normalizers the diff uses (`normalizeDefault` /
+ *    `normalizeCheck`), applied to both sides in `canonicalizeState`. Without
+ *    this the model side hashes its source form (`'primary'`, `scope = ANY(...)`)
+ *    while the live side hashes pg's deparse (`'primary'::text`,
+ *    `(scope = ANY(...))`) — the identity below breaks on any schema with a text
+ *    default, a check, or a partial index even when the diff is a no-op.
  *  - Deparse stability caveat, stated once: policy/check predicates hash as
  *    Postgres deparses them; a major-version deparser change can shift
  *    fingerprints. The version prefix plus the re-diff path keep that honest.
@@ -43,8 +50,12 @@ import type { SchemaSnapshot, TableSchema } from './schema-introspect.js';
 import type { AuthzContract, TableContract } from './authz-contract.js';
 import { compileTableSchema, compileEnums } from './schema-compile.js';
 import { compileTableContract } from './authz-compile.js';
+import { normalizeDefault, normalizeCheck } from './schema-diff.js';
 
-export const FINGERPRINT_VERSION = 1;
+// v2: expression fields (defaults, checks, partial-index WHERE) normalize through
+// the diff's pg-deparse normalizers before hashing, so the model form and the live
+// deparsed form agree — the identity `fingerprintModels === fingerprintLive` holds.
+export const FINGERPRINT_VERSION = 2;
 
 // ---------------------------------------------------------------------------
 // Canonical form.
@@ -70,13 +81,17 @@ function canonicalTable(table: TableSchema): Record<string, unknown> {
   return {
     table: table.table,
     columns: byKey(
-      table.columns.map((c) => ({ name: c.name, type: c.type, notNull: c.notNull, default: c.default })),
+      // Defaults hash SEMANTICALLY, through the same normalizer the diff uses: the Model
+      // emits `'primary'`, pg deparses it to `'primary'::text` — the same state, one hash.
+      table.columns.map((c) => ({ name: c.name, type: c.type, notNull: c.notNull, default: normalizeDefault(c.default) })),
       (c) => c.name,
     ),
     primaryKey: table.primaryKey,
     // Content, not names: a pg-default `_key` and a model-named unique are the same state.
     uniques: byKey(table.uniques.map((u) => ({ columns: u.columns })), (u) => u.columns.join(',')),
-    checks: byKey(table.checks.map((c) => ({ expr: c.expr })), (c) => c.expr),
+    // Predicates hash as the diff compares them — one layer of pg's wrapping parens dropped
+    // (`(scope = ANY (...))` == `scope = ANY (...)`), whitespace collapsed.
+    checks: byKey(table.checks.map((c) => ({ expr: normalizeCheck(c.expr) })), (c) => c.expr),
     foreignKeys: byKey(
       table.foreignKeys.map((f) => ({
         columns: f.columns, refTable: f.refTable, refColumns: f.refColumns,
@@ -86,7 +101,8 @@ function canonicalTable(table: TableSchema): Record<string, unknown> {
       (f) => stableStringify(f),
     ),
     indexes: byKey(
-      table.indexes.map((i) => ({ columns: i.columns, unique: i.unique, ...(i.where ? { where: i.where } : {}) })),
+      // Partial-index WHERE is a predicate too — normalize it the same way as a check.
+      table.indexes.map((i) => ({ columns: i.columns, unique: i.unique, ...(i.where ? { where: normalizeCheck(i.where) } : {}) })),
       (i) => stableStringify(i),
     ),
   };