@everystack/cli 0.4.59 → 0.4.60

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@everystack/cli",
-  "version": "0.4.59",
+  "version": "0.4.60",
   "description": "CLI and OTA updates for Expo apps on everystack",
   "license": "AGPL-3.0-only",
   "author": "Scalable Technology, Inc. <licensing@scalable.technology>",
@@ -160,7 +160,7 @@
     "jest": "29.7.0",
     "react": "19.2.0",
     "ts-jest": "29.4.9",
-    "@everystack/server": "0.4.20"
+    "@everystack/server": "0.4.21"
   },
   "scripts": {
     "test": "jest",

package/src/cli/commands/db-backup.ts

@@ -21,6 +21,7 @@ import { pipeline } from 'node:stream/promises';
 import { randomBytes } from 'node:crypto';
 import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
 import { invokeAction, presignGet, getS3 } from '../aws.js';
+import { runStageBackup, stageBackupDeps } from '../stage-backup.js';
 import {
   parseBackupRef, keyForId, crossStageGuard, restoreTargetGuard,
   backupKey, backupId, metaKey, utcStamp,
@@ -226,39 +227,27 @@ export async function dbBackupCommand(flags: Record<string, string>): Promise<vo
   info(`Region: ${config.region}, Function: ${opsFunction(config)}`);
   const fn = opsFunction(config);
 
-  step('Dispatching the backup task (credential-free, via the ops Lambda)...');
-  let dispatched: any;
-  try {
-    dispatched = await invokeAction(config.region, fn, 'db:backup', { stage: flags.stage, actor: process.env.USER ?? null });
-  } catch (err: any) {
-    fail(`Backup failed to dispatch: ${err.message}`);
-    process.exit(1);
-  }
-  if (dispatched?.error) {
-    fail(`Backup failed: ${dispatched.error}`);
-    process.exit(1);
-  }
-  const { runId, taskArn, id, warning } = dispatched as { runId: string; taskArn: string; id: string; warning?: string };
-  info(`backup ${id} — task ${taskArn}`);
-  info(`run id: ${runId} (everystack.task_log)`);
-  if (warning) info(`note: ${warning}`);
+  const result = await runStageBackup(stageBackupDeps(config, fn, flags.stage, {
+    onDispatch: (d) => {
+      info(`backup ${d.id} — task ${d.taskArn}`);
+      info(`run id: ${d.runId} (everystack.task_log)`);
+      // The dispatch warning is folded into result.warning and printed at the end, so printing it
+      // here too says the same thing twice.
+    },
+  }));
 
-  step('Running pg_dump → S3 (this may take a while for large databases)...');
-  const poll = await pollTaskUntilStopped(config.region, fn, { runId, taskArn });
-  if (poll.outcome === 'timeout') {
-    fail(`db:backup timed out (last status: ${poll.lastStatus}). Run id ${runId} — reconcile via ECS/everystack.task_log.`);
-    process.exit(1);
-  }
-  if (poll.outcome === 'error') {
-    fail(`task:status failed repeatedly: ${poll.status.error}. The backup may still be running — reconcile run id ${runId}.`);
-    process.exit(1);
-  }
-  if (poll.status.exitCode !== 0) {
-    fail(`Backup failed (exit ${poll.status.exitCode ?? 'unknown'})${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}. Read the task logs (CloudWatch).`);
+  if (!result.ok) {
+    fail(`Backup failed: ${result.reason}`);
+    if (result.orphanKey) {
+      info(`The artifact is still in the bucket: s3://${config.backupsBucket ?? '<backups-bucket>'}/${result.orphanKey}`);
+    }
     process.exit(1);
   }
-  const bytes = (poll.status.result?.bytes as number | undefined) ?? undefined;
-  success(`Backup ${id} (${fmtBytes(bytes)}). Restore with: everystack db:restore --from ${id} --confirm`);
+
+  // Render it. The helper produced this and nothing printed it, so the one signal that an artifact
+  // was not verified against what the operator sent was invisible at the only moment it mattered.
+  if (result.warning) warn(result.warning);
+  success(`Backup ${result.id} (${fmtBytes(result.bytes)}). Restore with: everystack db:restore --from ${result.id} --confirm`);
 }
 
 /** db:backups — list a stage's logical backups. */

package/src/cli/commands/db-fork.ts

@@ -23,6 +23,7 @@ import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
 import { invokeAction, presignGet } from '../aws.js';
 import { keyForId, parseBackupRef, isProductionTier, crossStageGuard } from '../backup.js';
 import { pollTaskUntilStopped } from '../task-poll.js';
+import { runStageBackup, stageBackupDeps } from '../stage-backup.js';
 import { step, success, fail, info, warn } from '../output.js';
 
 export type ForkGuardVerdict =
@@ -94,18 +95,17 @@ export async function dbForkCommand(flags: Record<string, string>): Promise<void
     info(`Reusing backup ${backupId}.`);
   } else {
     step(`Backing up ${from} (pg_dump → S3 in the Task; may take a while for large databases)...`);
-    const dispatched: any = await invokeAction(source.region, opsFunction(source), 'db:backup', { stage: from, actor: process.env.USER ?? null });
-    if (dispatched?.error) {
-      fail(`Source backup failed: ${dispatched.error}`);
-      process.exit(1);
-    }
-    backupId = dispatched.id;
-    const poll = await pollTaskUntilStopped(source.region, opsFunction(source), { runId: dispatched.runId, taskArn: dispatched.taskArn });
-    if (poll.outcome !== 'stopped' || poll.status.exitCode !== 0) {
-      const why = poll.outcome === 'stopped' ? `exit ${poll.status.exitCode}${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}` : poll.outcome;
-      fail(`Source backup failed (${why}). Reconcile run ${dispatched.runId}.`);
+    // The shared sandwich: a fork built from a dump that spans a migration copies a schema that
+    // never existed. Same helper db:backup and db:swap use, so all three agree on what a proven
+    // artifact is.
+    const backup = await runStageBackup(stageBackupDeps(source, opsFunction(source), from, { onStep: step }));
+    if (!backup.ok) {
+      fail(`Source backup failed: ${backup.reason} The fork was NOT created.`);
+      if (backup.orphanKey) info(`The artifact is still in the bucket: s3://${source.backupsBucket}/${backup.orphanKey}`);
       process.exit(1);
     }
+    backupId = backup.id;
+    if (backup.warning) warn(backup.warning);
     info(`Backup ${backupId}.`);
   }
 

package/src/cli/commands/db-swap.ts

@@ -45,6 +45,7 @@ import { keyForArtifactId, metaKey, utcStamp } from '../backup.js';
 import { rdsSnapshotIdentifier } from '../rds-snapshot.js';
 import { pollTaskUntilStopped } from '../task-poll.js';
 import { decideSnapshotMode, confirmPhysicalSnapshot, interpretBackupPoll, type SnapshotModeRequest } from '../swap-snapshot.js';
+import { runStageBackup, stageBackupDeps } from '../stage-backup.js';
 import { pgEnvFromUrl, pgKeepaliveConninfo } from './db.js';
 import { step, success, fail, warn, info } from '../output.js';
 
@@ -475,19 +476,30 @@ async function takePreSwapSnapshot(
   // Logical: dispatch the Task and WAIT. The dispatch returning is not the backup existing — that
   // conflation is what let a pg_dump run concurrently with the restore it was supposed to precede.
   step('Snapshotting the stage before the swap (db:backup — waiting for the dump to finish)...');
-  const dispatched: any = await invokeAction(ctx.region!, ctx.opsFn!, 'db:backup', {
-    stage: ctx.stage,
-    actor: process.env.USER ?? null,
+  // The shared sandwich. A rollback point taken across a concurrent migration holds a schema that
+  // matches neither endpoint — restoring from it would not return the stage to where it was, which
+  // is the one thing a pre-swap snapshot has to guarantee. An unproven artifact fails the swap.
+  const backup = await runStageBackup({
+    ...stageBackupDeps({ region: ctx.region! }, ctx.opsFn!, ctx.stage, {
+      onStep: step,
+      onDispatch: (d: { id: string; runId: string }) => info(`backup ${d.id} dispatched (run ${d.runId}) — waiting for the task to stop before the restore starts.`),
+    }),
+    // Swap keeps its OWN reading of a finished poll: it is the only caller that can tell the
+    // operator to re-run with `--snapshot-ref <id>` once the dump lands, and each outcome needs
+    // different advice. The shared helper takes injected deps for exactly this.
+    awaitTask: async (ids) => {
+      const poll = await pollTaskUntilStopped(ctx.region!, ctx.opsFn!, { runId: ids.runId, taskArn: ids.taskArn });
+      const verdict = interpretBackupPoll(poll, { runId: ids.runId, id: ids.id });
+      if (!verdict.ok) return verdict;
+      // Carry the stamp through, or this caller silently opts out of the image-skew check.
+      return { ok: true, stampedFingerprint: (poll as any).status?.result?.fingerprint as string | undefined };
+    },
   });
-  if (dispatched?.error) throw new Error(`the pre-swap backup would not dispatch, so the swap was NOT applied: ${dispatched.error}`);
-  const { runId, taskArn, id } = dispatched as { runId: string; taskArn: string; id: string };
-  info(`backup ${id} dispatched (run ${runId}) — waiting for the task to stop before the restore starts.`);
-  const verdict = interpretBackupPoll(
-    await pollTaskUntilStopped(ctx.region!, ctx.opsFn!, { runId, taskArn }),
-    { runId, id },
-  );
-  if (!verdict.ok) throw new Error(verdict.reason);
-  info(`rollback point CONFIRMED: backup ${id} complete — restore with db:restore --from ${id} --confirm.`);
+  if (!backup.ok) {
+    throw new Error(`the pre-swap backup is not a usable rollback point, so the swap was NOT applied and live is untouched: ${backup.reason}`);
+  }
+  if (backup.warning) warn(backup.warning);
+  info(`rollback point CONFIRMED: backup ${backup.id} complete — restore with db:restore --from ${backup.id} --confirm.`);
 }
 
 export async function dbSwapCommand(flags: Record<string, string>): Promise<void> {

package/src/cli/stage-backup.ts

@@ -0,0 +1,229 @@
+/**
+ * The fingerprint sandwich around a stage backup — ONE implementation, three callers.
+ *
+ * A `db:backup` artifact carries a live base-schema fingerprint in its `.meta.json`, and the
+ * destructive-apply gate compares it against `plan.from`. That comparison is only meaningful if the
+ * fingerprint describes the schema the dump ACTUALLY holds. A dump taken across a concurrent
+ * migration holds a schema matching NEITHER endpoint, so its stamped fingerprint is a lie the gate
+ * would later trust. The sandwich reads the schema before the dump and again after it, and keeps
+ * the artifact only if the two agree.
+ *
+ * WHY THIS LIVES IN THE CLI: it used to run inside the task image, which imported
+ * `@everystack/cli/apply` to do it. The image is published and digest-pinned, so that froze a copy
+ * of the very CLI internals the gate depends on while the operator's checkout kept moving — a
+ * silent version skew on a safety-critical input. Here it runs from the operator's own checkout,
+ * the same code `db:apply` uses, so the two can never disagree about what a fingerprint means.
+ *
+ * WHY IT IS SHARED: `fingerprint` is a REQUIRED field on the backup job. Three commands dispatch
+ * backups (db:backup, db:fork, db:swap --snapshot logical), and when the field was added to one of
+ * them the other two silently broke. One implementation, one contract.
+ *
+ * The dispatch/poll/retract wiring is dependency-injected so the whole decision table is testable
+ * without AWS.
+ */
+
+import { invokeAction, lambdaSessionRunner } from './aws.js';
+import { pollTaskUntilStopped } from './task-poll.js';
+
+/** The verdict on a completed dump's fingerprint pair. Pure — the heart of the sandwich. */
+export type SandwichVerdict = { ok: true } | { ok: false; reason: string };
+
+/**
+ * Did the schema hold still across the dump window?
+ *
+ * Agreement proves no DDL landed ANYWHERE between the two reads, so the fingerprint genuinely
+ * describes the dump's schema. Disagreement means the artifact is unusable as a safety net —
+ * better a refused backup than a false one.
+ */
+export function judgeSandwich(fpBefore: string, fpAfter: string): SandwichVerdict {
+  if (fpBefore === fpAfter) return { ok: true };
+  return {
+    ok: false,
+    reason:
+      `the schema changed during the dump (${fpBefore.slice(0, 12)} → ${fpAfter.slice(0, 12)}), `
+      + 'so the backup spans a migration and its fingerprint cannot be proven. Re-run when no migration is concurrent.',
+  };
+}
+
+export interface StageBackupDeps {
+  /** Read the live base-schema fingerprint. Called twice, bracketing the dump. */
+  readFingerprint: () => Promise<string>;
+  /** Dispatch `db:backup` with the operator-computed fingerprint. */
+  dispatch: (fingerprint: string) => Promise<{ runId: string; taskArn: string; id: string; key: string; warning?: string }>;
+  /**
+   * Wait for the task to stop; resolve ok, or a human reason it did not.
+   *
+   * `stampedFingerprint` is what the task reported writing into the artifact's `.meta.json`. It is
+   * the evidence the image honored the job — see the skew check in runStageBackup.
+   */
+  awaitTask: (ids: { runId: string; taskArn: string; id: string }) => Promise<{ ok: true; bytes?: number; stampedFingerprint?: string } | { ok: false; reason: string }>;
+  /** Delete a retracted artifact (the ops Lambda's db:backup:retract). */
+  retract: (key: string) => Promise<void>;
+  /** Progress output. Optional so tests stay quiet. */
+  onStep?: (message: string) => void;
+}
+
+export type StageBackupResult =
+  | { ok: true; id: string; key: string; bytes?: number; fingerprint: string; warning?: string }
+  | { ok: false; reason: string; retracted?: boolean; orphanKey?: string };
+
+/**
+ * Take a stage backup whose fingerprint is PROVEN stable, or refuse.
+ *
+ * Never throws for an expected failure — every outcome is a verdict the caller renders, so the
+ * three commands can word their own consequences ("the swap was NOT applied", "the fork stopped")
+ * without re-deriving the logic.
+ */
+export async function runStageBackup(deps: StageBackupDeps): Promise<StageBackupResult> {
+  const step = deps.onStep ?? (() => {});
+
+  step('Reading the live schema fingerprint (before the dump)...');
+  let fpBefore: string;
+  try {
+    fpBefore = await deps.readFingerprint();
+  } catch (err: any) {
+    // Refuse BEFORE dispatching: a dump whose fingerprint cannot be proven is not a safety net, and
+    // taking one anyway would burn the dump window and still leave nothing trustworthy.
+    return { ok: false, reason: `the schema fingerprint could not be read, so no backup was dispatched: ${err?.message ?? String(err)}` };
+  }
+
+  let dispatched: { runId: string; taskArn: string; id: string; key: string; warning?: string };
+  try {
+    dispatched = await deps.dispatch(fpBefore);
+  } catch (err: any) {
+    return { ok: false, reason: `the backup would not dispatch: ${err?.message ?? String(err)}` };
+  }
+
+  const finished = await deps.awaitTask(dispatched);
+  if (!finished.ok) return { ok: false, reason: finished.reason };
+
+  // Did the image actually USE the fingerprint we sent?
+  //
+  // This is the version-skew guard, and it deliberately does not rely on the image cooperating. An
+  // older task image ignores the job's `fingerprint` field entirely and computes its own, stamping
+  // the manifest with a value this CLI never produced — the exact stale-internals problem the
+  // operator-side move exists to remove, arriving through a pinned digest nobody re-published. A
+  // protocol handshake cannot catch that, because an old image does not know to check it. Comparing
+  // what came back against what we sent works against ANY image, old or new.
+  if (finished.stampedFingerprint !== undefined && finished.stampedFingerprint !== fpBefore) {
+    step('Retracting the backup (the task stamped a fingerprint we did not send)...');
+    const mismatch =
+      `the task stamped fingerprint ${finished.stampedFingerprint.slice(0, 12)} but the operator sent `
+      + `${fpBefore.slice(0, 12)} — the task image computed its own instead of using the job's, which means it `
+      + 'predates the operator-side fingerprint contract. Re-publish the Task image (or pin a newer digest).';
+    try {
+      await deps.retract(dispatched.key);
+      return { ok: false, reason: `${mismatch} Backup ${dispatched.id} was retracted.`, retracted: true };
+    } catch (err: any) {
+      return {
+        ok: false,
+        reason: `${mismatch} Backup ${dispatched.id} could ALSO not be retracted (${err?.message ?? String(err)}) — delete it by hand.`,
+        retracted: false,
+        orphanKey: dispatched.key,
+      };
+    }
+  }
+
+  step('Re-reading the schema fingerprint (after the dump)...');
+  let fpAfter: string;
+  try {
+    fpAfter = await deps.readFingerprint();
+  } catch (err: any) {
+    // The dump exists but is unproven. Do NOT retract — an unreadable fingerprint is not evidence
+    // the schema moved, and silently destroying a good backup is the worse error.
+    return {
+      ok: false,
+      reason: `the dump finished but the schema fingerprint could not be re-read (${err?.message ?? String(err)}), so backup ${dispatched.id} is NOT proven stable. Verify no migration ran during the dump, or re-run.`,
+      orphanKey: dispatched.key,
+    };
+  }
+
+  const verdict = judgeSandwich(fpBefore, fpAfter);
+  if (verdict.ok) {
+    // A task that reported no fingerprint at all is not the skew case above — the result-ledger
+    // write is explicitly best-effort, so a blip there loses the whole result row. Failing the
+    // backup for that would turn a cosmetic outage into a lost safety net, so say it and continue.
+    const unverified = finished.stampedFingerprint === undefined
+      ? 'the task did not report the fingerprint it stamped, so the artifact could not be verified against what the operator sent (the result-ledger write is best-effort)'
+      : undefined;
+    const warning = [dispatched.warning, unverified].filter(Boolean).join('; ') || undefined;
+    return { ok: true, id: dispatched.id, key: dispatched.key, bytes: finished.bytes, fingerprint: fpBefore, warning };
+  }
+
+  step('Retracting the backup (its fingerprint cannot be proven)...');
+  try {
+    await deps.retract(dispatched.key);
+    return { ok: false, reason: `${verdict.reason} Backup ${dispatched.id} was retracted.`, retracted: true };
+  } catch (err: any) {
+    // The artifact outlived the failure. Say so loudly with the key — an unproven backup left in
+    // the bucket is exactly what the destructive-apply gate must never pick up.
+    return {
+      ok: false,
+      reason: `${verdict.reason} Backup ${dispatched.id} could ALSO not be retracted (${err?.message ?? String(err)}) — delete it by hand so the destructive-apply gate never trusts it.`,
+      retracted: false,
+      orphanKey: dispatched.key,
+    };
+  }
+}
+
+/**
+ * Bind the sandwich to a real stage: the ops Lambda supplies the session the fingerprint is read
+ * over, dispatches the task, reports status, and performs the retraction — so the operator never
+ * holds a database URL or an S3 credential for any of it.
+ *
+ * The ONE place the wiring is written. A caller that hand-rolled this is how `fingerprint` became
+ * required in db:backup while db:fork and db:swap kept dispatching without it.
+ */
+export function stageBackupDeps(
+  config: { region: string },
+  opsFn: string,
+  stage: string | undefined,
+  hooks: { onStep?: (message: string) => void; onDispatch?: (d: { runId: string; taskArn: string; id: string; warning?: string }) => void } = {},
+): StageBackupDeps {
+  return {
+    // BOTH ends of the sandwich read over this ONE closure, so they cannot land on different lanes.
+    // That matters: a before-read over a direct session and an after-read over the two-invoke
+    // ops-Lambda blend could manufacture a false mismatch — or, worse, a false match. The symmetry
+    // is structural here rather than asserted; if a caller ever needs to override one end, it has to
+    // override both.
+    readFingerprint: async () => {
+      // The SAME helper db:apply's concurrency lock uses — that shared definition is the point.
+      const { liveBaseFingerprint } = await import('./apply-execute.js');
+      return (await liveBaseFingerprint(lambdaSessionRunner(config.region, opsFn))).hash;
+    },
+    dispatch: async (fingerprint) => {
+      const dispatched: any = await invokeAction(config.region, opsFn, 'db:backup', {
+        stage, actor: process.env.USER ?? null, fingerprint,
+      });
+      if (dispatched?.error) throw new Error(dispatched.error);
+      hooks.onDispatch?.(dispatched);
+      return dispatched;
+    },
+    awaitTask: async (ids) => {
+      const poll = await pollTaskUntilStopped(config.region, opsFn, { runId: ids.runId, taskArn: ids.taskArn });
+      if (poll.outcome === 'timeout') {
+        return { ok: false, reason: `the dump did not finish in time (last status: ${poll.lastStatus}). Run id ${ids.runId} — reconcile via ECS / everystack.task_log.` };
+      }
+      if (poll.outcome === 'error') {
+        return { ok: false, reason: `the task status could not be read (${poll.status.error}). The dump may still be running — reconcile run id ${ids.runId}.` };
+      }
+      if (poll.status.exitCode !== 0) {
+        return { ok: false, reason: `the dump exited ${poll.status.exitCode ?? 'unknown'}${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}. Read the task logs (CloudWatch).` };
+      }
+      return {
+        ok: true,
+        bytes: poll.status.result?.bytes as number | undefined,
+        // What the task says it wrote into the artifact's .meta.json — the skew evidence.
+        stampedFingerprint: poll.status.result?.fingerprint as string | undefined,
+      };
+    },
+    retract: async (key) => {
+      // `stage` scopes what the action is willing to delete — it refuses any key outside this
+      // stage's backup prefix, so a retraction cannot reach an export artifact or a restore intent.
+      const res: any = await invokeAction(config.region, opsFn, 'db:backup:retract', { key, stage });
+      if (res?.error) throw new Error(res.error);
+      if (!res?.retracted) throw new Error(`the ops function did not confirm the retraction (returned ${JSON.stringify(res)})`);
+    },
+    onStep: hooks.onStep,
+  };
+}