@everystack/cli 0.4.35 → 0.4.38

package/src/cli/swap-snapshot.ts

@@ -0,0 +1,260 @@
+/**
+ * db:swap's pre-swap rollback point — which snapshot to take, and proof that it exists.
+ *
+ * The swap's step 2 is "SNAPSHOT — the rollback point, before anything destructive". It was taking a
+ * LOGICAL backup (`db:backup` via the ops Lambda) and that turned out to be wrong twice over:
+ *
+ *  - It never waited. The ops action DISPATCHES a Fargate task and returns the run id; the swap
+ *    printed "snapshot on record" and started restoring immediately. So the pg_dump ran alongside
+ *    the restore — a consumer measured the restore blocked ~5 minutes on
+ *    `Lock/relation HELD BY pid [pg_dump] COPY <schema>.<table>`, the swap contending with its own
+ *    backup. On a small instance that makes the swap's duration a coin flip unrelated to data volume.
+ *  - Worse than slow: unproven. A task that failed to start, or died on its credential, left the
+ *    swap proceeding into a destructive rename believing it had a rollback point it did not have.
+ *
+ * A PHYSICAL RDS snapshot is the better rollback point on every axis that matters here: it is a
+ * control-plane call, so it holds no relation locks, reads nothing through the buffer cache, and
+ * needs no client connection. It is also RDS-only, hence the mode selection below rather than a
+ * straight replacement.
+ *
+ * The decision is pure and the confirmation takes its IO injected, so both are provable without an
+ * AWS account. See docs/schema-swap.md.
+ */
+
+/** The values `--snapshot` accepts. `none` is the explicit opt-out; there is no implicit one. */
+export const SNAPSHOT_MODES = ['physical', 'logical', 'none'] as const;
+export type SnapshotModeRequest = (typeof SNAPSHOT_MODES)[number];
+
+/** What db:swap should do about a rollback point before it touches anything. */
+export type SnapshotDecision =
+  /** Take an RDS snapshot of this instance (no locks, no pg_dump, no connection). */
+  | { mode: 'physical'; instanceId: string }
+  /** Dispatch db:backup and WAIT for the task to finish before restoring. */
+  | { mode: 'logical' }
+  /** The operator already took one and named it — nothing to do but record it. */
+  | { mode: 'attested'; ref: string }
+  /** The operator explicitly accepted no rollback point. */
+  | { mode: 'none' }
+  /** Nothing safe is available and no consent was given — refuse before anything moves. */
+  | { mode: 'refuse'; reason: string };
+
+export interface SnapshotDecisionInput {
+  /**
+   * `stage` = `--stage <name> --direct`, which has an ops Lambda and deployed outputs behind it.
+   * `url` = a bare `--database-url`, which has neither: no stage config to read an instance id from
+   * and no ops function to dispatch a backup to.
+   */
+  venue: 'stage' | 'url';
+  /** `config.databaseInstanceId`. `placeholder` is what a non-RDS or dev-mode stage carries. */
+  instanceId?: string;
+  /** `--snapshot-ref <id>` — a snapshot the operator took themselves (db:apply's precedent). */
+  snapshotRef?: string;
+  /** `--snapshot <physical|logical|none>`. */
+  requested?: SnapshotModeRequest;
+}
+
+/** Is this instance id something we can actually snapshot, or an absent/placeholder output? */
+function usableInstanceId(id?: string): id is string {
+  return !!id && id !== 'placeholder';
+}
+
+/**
+ * Choose the rollback point.
+ *
+ * Precedence, and the reasoning for it:
+ *   1. `--snapshot-ref` — the operator attests one exists. Taking a second is waste, and this is the
+ *      shape db:apply's direct lane already uses, so the two destructive verbs read the same.
+ *   2. `--snapshot none` — informed consent to have no rollback point. Explicit only.
+ *   3. `--snapshot physical|logical` — an explicit choice, honoured or refused with the reason.
+ *   4. The default: physical on an RDS stage, logical on any other stage, refuse on a bare URL.
+ *
+ * The bare-URL default is a REFUSAL rather than a warning. It used to warn and carry on, which is
+ * the same class of hole as the `--stage` bypass 0.4.36 closed: a destructive verb whose safety step
+ * is optional in practice.
+ */
+export function decideSnapshotMode(input: SnapshotDecisionInput): SnapshotDecision {
+  const { venue, instanceId, snapshotRef, requested } = input;
+
+  if (requested !== undefined && !SNAPSHOT_MODES.includes(requested)) {
+    return {
+      mode: 'refuse',
+      reason: `--snapshot ${requested} is not a snapshot mode. Pass one of: ${SNAPSHOT_MODES.join(', ')} `
+        + `(physical = an RDS snapshot, logical = a db:backup pg_dump, none = explicitly no rollback point).`,
+    };
+  }
+
+  if (snapshotRef) return { mode: 'attested', ref: snapshotRef };
+  if (requested === 'none') return { mode: 'none' };
+
+  if (requested === 'physical') {
+    // A physical snapshot is an RDS control-plane call, which needs a REGION as well as an instance
+    // id — and the only source of a region here is the stage's deployed config. A bare
+    // --database-url has none, so honouring the request would mean calling AWS with an undefined
+    // region and reporting a confusing SDK error instead of the real problem.
+    if (venue === 'url') {
+      return {
+        mode: 'refuse',
+        reason: 'A physical snapshot needs the stage\'s region and instance id, and a bare --database-url carries neither. '
+          + 'Take one against the stage (everystack db:snapshot --stage <name>) and attest it here: --snapshot-ref <id>. '
+          + 'Or accept the risk explicitly with --snapshot none.',
+      };
+    }
+    if (!usableInstanceId(instanceId)) {
+      return {
+        mode: 'refuse',
+        reason: 'A physical snapshot needs the RDS instance id, and this stage does not expose one. '
+          + 'Add `databaseInstanceId: database.id` to the outputs return block in sst.config.ts and redeploy '
+          + '(run db:swap from the app directory so .sst/outputs.json is readable), or pass --snapshot logical '
+          + 'for a pg_dump rollback point instead.',
+      };
+    }
+    return { mode: 'physical', instanceId };
+  }
+
+  if (requested === 'logical') {
+    if (venue === 'url') {
+      return {
+        mode: 'refuse',
+        reason: 'A logical snapshot runs in the stage\'s Task lane via the ops Lambda, and a bare --database-url has no stage behind it. '
+          + 'Take one yourself and name it: --snapshot-ref <id> (everystack db:backup --database-url … or db:snapshot), '
+          + 'or accept the risk explicitly with --snapshot none.',
+      };
+    }
+    return { mode: 'logical' };
+  }
+
+  // No explicit request — the default per venue.
+  if (venue === 'stage') {
+    return usableInstanceId(instanceId) ? { mode: 'physical', instanceId } : { mode: 'logical' };
+  }
+  return {
+    mode: 'refuse',
+    reason: 'db:swap over a bare --database-url does not take a snapshot for you, and it will not run destructively without one. '
+      + 'Take a rollback point and name it: --snapshot-ref <id> (everystack db:snapshot, or db:backup --database-url …). '
+      + 'If you genuinely want no rollback point, say so: --snapshot none.',
+  };
+}
+
+/**
+ * A finished `pollTaskUntilStopped` result, read as "is there a rollback point or not".
+ *
+ * Kept pure and separate because this is the judgement the old code never made: it treated the
+ * DISPATCH as the answer. Every non-success outcome here has to abort the swap, and each one needs
+ * different advice, so the wording is worth pinning in a test.
+ */
+export function interpretBackupPoll(
+  poll:
+    | { outcome: 'timeout'; lastStatus: string }
+    | { outcome: 'error'; status: { error?: string } }
+    | { outcome: 'stopped'; status: { exitCode?: number | null; stoppedReason?: string | null } },
+  ids: { runId: string; id: string },
+): { ok: true } | { ok: false; reason: string } {
+  const untouched = 'so the swap was NOT applied and live is untouched.';
+  if (poll.outcome === 'timeout') {
+    return {
+      ok: false,
+      reason: `the pre-swap backup did not finish in time (last status: ${poll.lastStatus}), ${untouched} `
+        + `Run id ${ids.runId} — check everystack.task_log / ECS, then re-run with --snapshot-ref ${ids.id} once the backup is on record.`,
+    };
+  }
+  if (poll.outcome === 'error') {
+    return {
+      ok: false,
+      reason: `the pre-swap backup's status could not be read (${poll.status.error}), so the swap was NOT applied. `
+        + `It may still be running — reconcile run id ${ids.runId} before retrying.`,
+    };
+  }
+  if (poll.status.exitCode !== 0) {
+    return {
+      ok: false,
+      reason: `the pre-swap backup FAILED (exit ${poll.status.exitCode ?? 'unknown'})`
+        + `${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}, ${untouched} `
+        + `Read the task logs (CloudWatch) before retrying.`,
+    };
+  }
+  return { ok: true };
+}
+
+/** The RDS control-plane calls confirmPhysicalSnapshot needs, injected so the wait is testable. */
+export interface PhysicalSnapshotIO {
+  /** CreateDBSnapshot — returns the new snapshot's identifier and initial status. */
+  create: (snapshotId: string) => Promise<{ identifier: string; status: string }>;
+  /** DescribeDBSnapshots for the instance (manual snapshots). */
+  describe: () => Promise<Array<{ identifier: string; status: string }>>;
+  log: (msg: string) => void;
+  sleep: (ms: number) => Promise<void>;
+  now: () => number;
+}
+
+export interface ConfirmPhysicalOptions {
+  instanceId: string;
+  snapshotId: string;
+  /** How long to wait for `available` before refusing. */
+  deadlineMs?: number;
+  /** How often to re-read the status. */
+  pollIntervalMs?: number;
+}
+
+/**
+ * A snapshot's status is polled to a TERMINAL state before the swap is allowed to continue. 15
+ * minutes is generous for a dev-sized instance and short enough that a stuck snapshot surfaces as a
+ * refusal rather than an hour of silence.
+ */
+const DEFAULT_SNAPSHOT_DEADLINE_MS = 15 * 60_000;
+const DEFAULT_SNAPSHOT_POLL_MS = 10_000;
+
+/**
+ * Take an RDS snapshot and do not return until RDS says it is `available`.
+ *
+ * Why wait at all, when the snapshot's consistency point is fixed the moment CreateDBSnapshot is
+ * accepted: because "accepted" is not "exists". A snapshot can go to `failed` (instance state,
+ * storage), and the entire value of this step is that the operator can get back. Proceeding into a
+ * destructive rename on an unconfirmed rollback point is the defect this module was written to
+ * remove — waiting is the only thing that turns the printed id into a fact.
+ *
+ * On the deadline it THROWS naming the snapshot id, because the re-run is one flag: the snapshot is
+ * still coming, so `--snapshot-ref <id>` reuses it instead of starting another.
+ */
+export async function confirmPhysicalSnapshot(
+  io: PhysicalSnapshotIO,
+  opts: ConfirmPhysicalOptions,
+): Promise<{ id: string }> {
+  const deadlineMs = opts.deadlineMs ?? DEFAULT_SNAPSHOT_DEADLINE_MS;
+  const pollMs = opts.pollIntervalMs ?? DEFAULT_SNAPSHOT_POLL_MS;
+  const created = await io.create(opts.snapshotId);
+  const id = created.identifier;
+  io.log(`physical snapshot ${id} of ${opts.instanceId} — status ${created.status}.`);
+  if (created.status === 'available') return { id };
+  if (created.status === 'failed') {
+    throw new Error(`the pre-swap RDS snapshot ${id} failed immediately, so the swap was NOT applied and live is untouched.`);
+  }
+
+  const start = io.now();
+  let lastStatus = created.status;
+  for (;;) {
+    if (io.now() - start >= deadlineMs) {
+      throw new Error(
+        `the pre-swap RDS snapshot ${id} is still "${lastStatus}" after ${Math.round(deadlineMs / 60_000)} minutes, so the swap was NOT applied and live is untouched. `
+        + `The snapshot is still being taken — it is a real rollback point once it reaches available. `
+        + `Watch it with \`everystack db:snapshots\`, then re-run this swap with --snapshot-ref ${id} to reuse it instead of taking another.`,
+      );
+    }
+    await io.sleep(pollMs);
+    const snaps = await io.describe();
+    const mine = snaps.find((s) => s.identifier === id);
+    if (!mine) {
+      throw new Error(
+        `the pre-swap RDS snapshot ${id} is no longer listed on ${opts.instanceId} — it was deleted or never registered, so there is no rollback point. `
+        + `The swap was NOT applied and live is untouched.`,
+      );
+    }
+    if (mine.status !== lastStatus) {
+      lastStatus = mine.status;
+      io.log(`physical snapshot ${id}: ${lastStatus}`);
+    }
+    if (mine.status === 'available') return { id };
+    if (mine.status === 'failed') {
+      throw new Error(`the pre-swap RDS snapshot ${id} FAILED, so the swap was NOT applied and live is untouched. Check the instance's state and storage, then retry.`);
+    }
+  }
+}