@everystack/cli 0.4.36 → 0.4.39

package/src/cli/pg-argtypes.ts

@@ -0,0 +1,52 @@
+/**
+ * pg-argtypes — function identity is name + ARGUMENT TYPES.
+ *
+ * PostgreSQL identifies a function by `schema.name(argtypes)`, not by name. The derived
+ * layer keyed on `schema.name` alone, so two overloads collapsed onto one identity: the
+ * generated `DROP FUNCTION IF EXISTS api.get_user` errored ("function name is not
+ * unique"), `derived_provenance` (identity is its PRIMARY KEY) held one row for two live
+ * objects so drift could never see the second, and db:pull rendered duplicate consts.
+ *
+ * The identity spelling is the CATALOG's: `format_type(oid, NULL)` over `proargtypes` —
+ * `integer`, `character varying`, `double precision`, `text[]`. The live side reads it
+ * straight from pg_proc; the DECLARED side has only what the author typed, so
+ * `@everystack/model`'s `normalizeArgType` maps the SQL aliases (`int`, `varchar`,
+ * `timestamptz`) onto the same spelling and drops typmods (a function argument carries
+ * none — PostgreSQL discards length/precision, so `varchar(20)` and `varchar` are ONE
+ * type here).
+ *
+ * Known limit: a custom type outside the search_path prints QUALIFIED (`api.my_enum`)
+ * while a hand-written `arg('x', 'my_enum')` does not, and the two read as different
+ * functions. db:pull-rendered args are already catalog-spelled, so this only reaches a
+ * hand-authored declaration; the reconciler names the pair loudly rather than diffing it
+ * in silence (see planReconcile's spelling-mismatch warning).
+ *
+ * Pure and database-free — the identity must be computable in db:diff's CI-pure path.
+ */
+
+import { normalizeArgType } from '@everystack/model';
+
+// The spelling itself lives in @everystack/model, next to `arg()` — it is part of the
+// DECLARED vocabulary, and defineModule needs it to tell an overload from a duplicate.
+export { normalizeArgType };
+
+/** `api.get_user(integer, text)` — the identity, from the DECLARED argument types. */
+export function functionIdentity(schema: string, name: string, argTypes: readonly string[]): string {
+  return `${schema}.${name}(${argTypes.map(normalizeArgType).join(', ')})`;
+}
+
+/**
+ * Split a function identity into its qualified name and its argument list.
+ * `args` is null for a LEGACY identity that carries no signature (a pre-signature
+ * provenance row) — callers keep the old rendering for those.
+ */
+export function splitFunctionIdentity(identity: string): { qualified: string; args: string | null } {
+  const open = identity.indexOf('(');
+  if (open === -1 || !identity.endsWith(')')) return { qualified: identity, args: null };
+  return { qualified: identity.slice(0, open), args: identity.slice(open + 1, -1) };
+}
+
+/** The pre-signature identity of a function — `schema.name`. The provenance upgrade's lookup key. */
+export function legacyFunctionIdentity(identity: string): string {
+  return splitFunctionIdentity(identity).qualified;
+}

package/src/cli/swap-heartbeat.ts

@@ -78,6 +78,24 @@ export interface HeartbeatSample {
   blockedBy: string | null;
 }
 
+/**
+ * What the CLIENT has sent, to sit alongside what the server reports.
+ *
+ * The server side alone cannot tell a stalled load from a finished one. Both look like
+ * `Client/ClientRead` with flat progress: the server is waiting for the next command. The difference
+ * is whether there is anything left to send, and only the client knows that. Without this the
+ * detector called the deadlock signature on a run that had landed all 5,984,956 rows and was one
+ * second from completing.
+ */
+export interface ClientFeed {
+  /** Bytes handed to psql's stdin so far. */
+  fedBytes: number;
+  /** Bytes the restore has to feed in total (the rewritten SQL file's size). */
+  totalBytes: number;
+  /** The local reader has written everything — nothing more is coming from our side. */
+  done: boolean;
+}
+
 export type Liveness =
   /** Rows or COPY bytes grew since the last poll. */
   | 'progressing'
@@ -85,6 +103,12 @@ export type Liveness =
   | 'blocked'
   /** Flat progress AND the server is waiting on the client: the deadlock signature. */
   | 'client-stall'
+  /**
+   * The client has sent everything and the server is waiting on it. That is not a stall — it is the
+   * tail of a successful load (the final commit, the connection winding down). Distinguishing this
+   * from `client-stall` is the whole reason ClientFeed exists.
+   */
+  | 'finishing'
   /** Connected and working, but nothing measurable moved this poll (DDL, index build). */
   | 'busy'
   /** No psql backend connected — not started yet, or already gone. */
@@ -218,9 +242,14 @@ function movedForward(prev: HeartbeatSample, cur: HeartbeatSample): boolean {
  *   3. no predecessor        → busy (cannot claim progress OR a stall on the first poll)
  *   4. rows/bytes grew       → progressing, EVEN in ClientRead (the healthy high-latency case)
  *   5. schema not created    → busy (DDL phase, nothing to measure)
- *   6. flat + ClientRead     → client-stall (the deadlock signature)
+ *   6. flat + ClientRead + the client still has bytes to send → client-stall (the deadlock signature)
+ *   7. flat + ClientRead + the client has sent everything     → finishing (the successful tail)
+ *
+ * Step 7 is the B5 fix. "Nothing is moving" was treated as sufficient evidence of a stall, and it is
+ * not: at the end of a healthy load nothing moves either. A stall claim now requires UNLANDED WORK —
+ * bytes the client still owes the server — which is why `feed` is threaded this far down.
  */
-export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample): Liveness {
+export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample, feed?: ClientFeed): Liveness {
   if (cur.state === null) return 'absent';
   if (!prev) return 'busy';
   // PROGRESS OUTRANKS EVERY SCARY WAIT STATE, including Lock. A restore takes relation locks
@@ -231,7 +260,7 @@ export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample): Li
   if (movedForward(prev, cur)) return 'progressing';
   if (cur.waitEventType === 'Lock') return 'blocked';
   if (!cur.schemaExists) return 'busy';
-  if (cur.waitEvent === 'ClientRead') return 'client-stall';
+  if (cur.waitEvent === 'ClientRead') return feed?.done ? 'finishing' : 'client-stall';
   return 'busy';
 }
 
@@ -251,16 +280,70 @@ function groupNum(n: number): string {
   return n.toLocaleString('en-US');
 }
 
-/** One operator-readable line per poll. Says what moved, what it is waiting on, and how long in. */
-export function formatSample(prev: HeartbeatSample | null, cur: HeartbeatSample, liveness: Liveness): string {
+/** `1.9 GB`, `512 KB` — bytes at poll-line resolution. */
+function fmtBytes(n: number): string {
+  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+  let v = n;
+  let u = 0;
+  while (v >= 1024 && u < units.length - 1) { v /= 1024; u += 1; }
+  return `${u === 0 ? v : v.toFixed(1)} ${units[u]}`;
+}
+
+/**
+ * WHICH signal moved between two samples, named honestly.
+ *
+ * The bug (B6): the progressing line said "COPY advancing" whenever the row delta was zero,
+ * regardless of what had actually moved. So a restore in its index phase — relations growing, no
+ * COPY anywhere — printed "COPY advancing", and so did the poll right after the final COPY finished.
+ * An idle tail read as a live load. Each branch below reports only what it actually observed.
+ */
+export function progressLabel(prev: HeartbeatSample, cur: HeartbeatSample): string {
+  const dRows = (cur.rows ?? 0) - (prev.rows ?? 0);
+  if (dRows > 0) {
+    const secs = Math.max(1, (cur.elapsedMs - prev.elapsedMs) / 1000);
+    return `+${groupNum(dRows)} row(s) (${groupNum(Math.round(dRows / secs))} rows/s)`;
+  }
+  const copyMoved = cur.copyBytes > prev.copyBytes || cur.copyRows > prev.copyRows;
+  // "Advancing" is a claim about NOW, so it needs a COPY running now — not merely one that moved.
+  if (copyMoved && cur.copies > 0) return 'COPY advancing';
+  const dRel = cur.relations - prev.relations;
+  if (dRel > 0) return `+${groupNum(dRel)} relation(s) created`;
+  if (copyMoved) return 'a COPY completed since the last poll';
+  return 'something moved';
+}
+
+/** The client-side position clause — `client fed 1.9 GB/2.0 GB (95%)`. Empty when unknown. */
+function clientClause(feed?: ClientFeed): string {
+  if (!feed) return '';
+  const pct = feed.totalBytes > 0 ? Math.floor((feed.fedBytes / feed.totalBytes) * 100) : 0;
+  const sent = feed.done ? 'client has fed ALL' : 'client fed';
+  return `, ${sent} ${fmtBytes(feed.fedBytes)}/${fmtBytes(feed.totalBytes)} (${pct}%)`;
+}
+
+/**
+ * One operator-readable line per poll. Says what moved, what it is waiting on, how long in — and, when
+ * known, what the CLIENT has sent.
+ *
+ * That last part is B7. A `Client/ClientRead` wait with the client 20% through the file and the same
+ * wait with the file fully fed are completely different situations, and they used to print
+ * identically: two debugging sessions went down the wrong path because "the client is not sending"
+ * and "the instance has no I/O left" were indistinguishable in the log.
+ */
+export function formatSample(
+  prev: HeartbeatSample | null,
+  cur: HeartbeatSample,
+  liveness: Liveness,
+  feed?: ClientFeed,
+): string {
   const at = `t+${humanElapsed(cur.elapsedMs)}`;
   const wait = cur.waitEventType ? `${cur.waitEventType}/${cur.waitEvent ?? '?'}` : (cur.state ?? 'idle');
+  const client = clientClause(feed);
 
   if (liveness === 'absent') {
     return `restore: no psql backend connected to the target (${at}) — the loader has not started yet, or has already exited.`;
   }
   if (!cur.schemaExists) {
-    return `restore: schema not created yet, no tables yet — DDL phase, ${wait} (${at}).`;
+    return `restore: schema not created yet, no tables yet — DDL phase, ${wait}${client} (${at}).`;
   }
 
   const rows = cur.rows === null ? 'unknown' : groupNum(cur.rows);
@@ -275,20 +358,18 @@ export function formatSample(prev: HeartbeatSample | null, cur: HeartbeatSample,
   if (liveness === 'blocked') {
     // Naming the holder is the difference between an actionable report and a shrug.
     const by = cur.blockedBy ? ` HELD BY ${cur.blockedBy}` : ' (holder not visible — it may belong to another role)';
-    return `restore: waiting on ${wait} with no progress this poll${by} — ${scope} (${at}).`;
+    return `restore: waiting on ${wait} with no progress this poll${by} — ${scope}${client} (${at}).`;
+  }
+  if (liveness === 'finishing') {
+    return `restore: ${scope}${client} — FINISHING: everything has been sent, the server is winding the load down (${wait}), ${at}.`;
   }
   if (liveness === 'client-stall') {
-    return `restore: nothing landed since the last poll and the server is waiting on the client (${wait}) — ${scope} (${at}).`;
+    return `restore: nothing landed since the last poll and the server is waiting on the client (${wait}) — ${scope}${client} (${at}).`;
   }
   if (liveness === 'progressing' && prev) {
-    const dRows = (cur.rows ?? 0) - (prev.rows ?? 0);
-    const secs = Math.max(1, (cur.elapsedMs - prev.elapsedMs) / 1000);
-    const moved = dRows > 0
-      ? `+${groupNum(dRows)} row(s) (${groupNum(Math.round(dRows / secs))} rows/s)`
-      : 'COPY advancing';
-    return `restore: ${scope}, ${moved} — ${wait} (${at}).`;
+    return `restore: ${scope}, ${progressLabel(prev, cur)} — ${wait}${client} (${at}).`;
   }
-  return `restore: ${scope}, nothing new this poll — ${wait} (${at}).`;
+  return `restore: ${scope}, nothing new this poll — ${wait}${client} (${at}).`;
 }
 
 /**
@@ -335,6 +416,12 @@ export interface HeartbeatOptions {
    * backend for. Must not throw; a sink that does is ignored so it cannot kill the heartbeat.
    */
   onSample?: (sample: HeartbeatSample) => void;
+  /**
+   * The client's current write position, read fresh each poll. Supplying it is what lets the
+   * heartbeat tell a stalled load from a finishing one (see ClientFeed) — without it the detector
+   * falls back to server-side-only reasoning and will call a quiet tail a stall.
+   */
+  clientFeed?: () => ClientFeed;
 }
 
 /**
@@ -363,8 +450,10 @@ export function startHeartbeat(runner: QueryRunner, opts: HeartbeatOptions): ()
     // Raw sample first: a caller acting on the probe (the dead-backend watchdog) must see every
     // sample, and must never be able to break the heartbeat by throwing.
     try { opts.onSample?.(cur); } catch { /* a sink's failure is not the probe's problem */ }
-    const liveness = classify(prev, cur);
-    const line = formatSample(prev, cur, liveness);
+    let feed: ClientFeed | undefined;
+    try { feed = opts.clientFeed?.(); } catch { /* same: a bad reader must not kill the probe */ }
+    const liveness = classify(prev, cur, feed);
+    const line = formatSample(prev, cur, liveness, feed);
 
     if (liveness === 'blocked') {
       warn(line);

package/src/cli/swap-snapshot.ts

@@ -0,0 +1,260 @@
+/**
+ * db:swap's pre-swap rollback point — which snapshot to take, and proof that it exists.
+ *
+ * The swap's step 2 is "SNAPSHOT — the rollback point, before anything destructive". It was taking a
+ * LOGICAL backup (`db:backup` via the ops Lambda) and that turned out to be wrong twice over:
+ *
+ *  - It never waited. The ops action DISPATCHES a Fargate task and returns the run id; the swap
+ *    printed "snapshot on record" and started restoring immediately. So the pg_dump ran alongside
+ *    the restore — a consumer measured the restore blocked ~5 minutes on
+ *    `Lock/relation HELD BY pid [pg_dump] COPY <schema>.<table>`, the swap contending with its own
+ *    backup. On a small instance that makes the swap's duration a coin flip unrelated to data volume.
+ *  - Worse than slow: unproven. A task that failed to start, or died on its credential, left the
+ *    swap proceeding into a destructive rename believing it had a rollback point it did not have.
+ *
+ * A PHYSICAL RDS snapshot is the better rollback point on every axis that matters here: it is a
+ * control-plane call, so it holds no relation locks, reads nothing through the buffer cache, and
+ * needs no client connection. It is also RDS-only, hence the mode selection below rather than a
+ * straight replacement.
+ *
+ * The decision is pure and the confirmation takes its IO injected, so both are provable without an
+ * AWS account. See docs/schema-swap.md.
+ */
+
+/** The values `--snapshot` accepts. `none` is the explicit opt-out; there is no implicit one. */
+export const SNAPSHOT_MODES = ['physical', 'logical', 'none'] as const;
+export type SnapshotModeRequest = (typeof SNAPSHOT_MODES)[number];
+
+/** What db:swap should do about a rollback point before it touches anything. */
+export type SnapshotDecision =
+  /** Take an RDS snapshot of this instance (no locks, no pg_dump, no connection). */
+  | { mode: 'physical'; instanceId: string }
+  /** Dispatch db:backup and WAIT for the task to finish before restoring. */
+  | { mode: 'logical' }
+  /** The operator already took one and named it — nothing to do but record it. */
+  | { mode: 'attested'; ref: string }
+  /** The operator explicitly accepted no rollback point. */
+  | { mode: 'none' }
+  /** Nothing safe is available and no consent was given — refuse before anything moves. */
+  | { mode: 'refuse'; reason: string };
+
+export interface SnapshotDecisionInput {
+  /**
+   * `stage` = `--stage <name> --direct`, which has an ops Lambda and deployed outputs behind it.
+   * `url` = a bare `--database-url`, which has neither: no stage config to read an instance id from
+   * and no ops function to dispatch a backup to.
+   */
+  venue: 'stage' | 'url';
+  /** `config.databaseInstanceId`. `placeholder` is what a non-RDS or dev-mode stage carries. */
+  instanceId?: string;
+  /** `--snapshot-ref <id>` — a snapshot the operator took themselves (db:apply's precedent). */
+  snapshotRef?: string;
+  /** `--snapshot <physical|logical|none>`. */
+  requested?: SnapshotModeRequest;
+}
+
+/** Is this instance id something we can actually snapshot, or an absent/placeholder output? */
+function usableInstanceId(id?: string): id is string {
+  return !!id && id !== 'placeholder';
+}
+
+/**
+ * Choose the rollback point.
+ *
+ * Precedence, and the reasoning for it:
+ *   1. `--snapshot-ref` — the operator attests one exists. Taking a second is waste, and this is the
+ *      shape db:apply's direct lane already uses, so the two destructive verbs read the same.
+ *   2. `--snapshot none` — informed consent to have no rollback point. Explicit only.
+ *   3. `--snapshot physical|logical` — an explicit choice, honoured or refused with the reason.
+ *   4. The default: physical on an RDS stage, logical on any other stage, refuse on a bare URL.
+ *
+ * The bare-URL default is a REFUSAL rather than a warning. It used to warn and carry on, which is
+ * the same class of hole as the `--stage` bypass 0.4.36 closed: a destructive verb whose safety step
+ * is optional in practice.
+ */
+export function decideSnapshotMode(input: SnapshotDecisionInput): SnapshotDecision {
+  const { venue, instanceId, snapshotRef, requested } = input;
+
+  if (requested !== undefined && !SNAPSHOT_MODES.includes(requested)) {
+    return {
+      mode: 'refuse',
+      reason: `--snapshot ${requested} is not a snapshot mode. Pass one of: ${SNAPSHOT_MODES.join(', ')} `
+        + `(physical = an RDS snapshot, logical = a db:backup pg_dump, none = explicitly no rollback point).`,
+    };
+  }
+
+  if (snapshotRef) return { mode: 'attested', ref: snapshotRef };
+  if (requested === 'none') return { mode: 'none' };
+
+  if (requested === 'physical') {
+    // A physical snapshot is an RDS control-plane call, which needs a REGION as well as an instance
+    // id — and the only source of a region here is the stage's deployed config. A bare
+    // --database-url has none, so honouring the request would mean calling AWS with an undefined
+    // region and reporting a confusing SDK error instead of the real problem.
+    if (venue === 'url') {
+      return {
+        mode: 'refuse',
+        reason: 'A physical snapshot needs the stage\'s region and instance id, and a bare --database-url carries neither. '
+          + 'Take one against the stage (everystack db:snapshot --stage <name>) and attest it here: --snapshot-ref <id>. '
+          + 'Or accept the risk explicitly with --snapshot none.',
+      };
+    }
+    if (!usableInstanceId(instanceId)) {
+      return {
+        mode: 'refuse',
+        reason: 'A physical snapshot needs the RDS instance id, and this stage does not expose one. '
+          + 'Add `databaseInstanceId: database.id` to the outputs return block in sst.config.ts and redeploy '
+          + '(run db:swap from the app directory so .sst/outputs.json is readable), or pass --snapshot logical '
+          + 'for a pg_dump rollback point instead.',
+      };
+    }
+    return { mode: 'physical', instanceId };
+  }
+
+  if (requested === 'logical') {
+    if (venue === 'url') {
+      return {
+        mode: 'refuse',
+        reason: 'A logical snapshot runs in the stage\'s Task lane via the ops Lambda, and a bare --database-url has no stage behind it. '
+          + 'Take one yourself and name it: --snapshot-ref <id> (everystack db:backup --database-url … or db:snapshot), '
+          + 'or accept the risk explicitly with --snapshot none.',
+      };
+    }
+    return { mode: 'logical' };
+  }
+
+  // No explicit request — the default per venue.
+  if (venue === 'stage') {
+    return usableInstanceId(instanceId) ? { mode: 'physical', instanceId } : { mode: 'logical' };
+  }
+  return {
+    mode: 'refuse',
+    reason: 'db:swap over a bare --database-url does not take a snapshot for you, and it will not run destructively without one. '
+      + 'Take a rollback point and name it: --snapshot-ref <id> (everystack db:snapshot, or db:backup --database-url …). '
+      + 'If you genuinely want no rollback point, say so: --snapshot none.',
+  };
+}
+
+/**
+ * A finished `pollTaskUntilStopped` result, read as "is there a rollback point or not".
+ *
+ * Kept pure and separate because this is the judgement the old code never made: it treated the
+ * DISPATCH as the answer. Every non-success outcome here has to abort the swap, and each one needs
+ * different advice, so the wording is worth pinning in a test.
+ */
+export function interpretBackupPoll(
+  poll:
+    | { outcome: 'timeout'; lastStatus: string }
+    | { outcome: 'error'; status: { error?: string } }
+    | { outcome: 'stopped'; status: { exitCode?: number | null; stoppedReason?: string | null } },
+  ids: { runId: string; id: string },
+): { ok: true } | { ok: false; reason: string } {
+  const untouched = 'so the swap was NOT applied and live is untouched.';
+  if (poll.outcome === 'timeout') {
+    return {
+      ok: false,
+      reason: `the pre-swap backup did not finish in time (last status: ${poll.lastStatus}), ${untouched} `
+        + `Run id ${ids.runId} — check everystack.task_log / ECS, then re-run with --snapshot-ref ${ids.id} once the backup is on record.`,
+    };
+  }
+  if (poll.outcome === 'error') {
+    return {
+      ok: false,
+      reason: `the pre-swap backup's status could not be read (${poll.status.error}), so the swap was NOT applied. `
+        + `It may still be running — reconcile run id ${ids.runId} before retrying.`,
+    };
+  }
+  if (poll.status.exitCode !== 0) {
+    return {
+      ok: false,
+      reason: `the pre-swap backup FAILED (exit ${poll.status.exitCode ?? 'unknown'})`
+        + `${poll.status.stoppedReason ? ` — ${poll.status.stoppedReason}` : ''}, ${untouched} `
+        + `Read the task logs (CloudWatch) before retrying.`,
+    };
+  }
+  return { ok: true };
+}
+
+/** The RDS control-plane calls confirmPhysicalSnapshot needs, injected so the wait is testable. */
+export interface PhysicalSnapshotIO {
+  /** CreateDBSnapshot — returns the new snapshot's identifier and initial status. */
+  create: (snapshotId: string) => Promise<{ identifier: string; status: string }>;
+  /** DescribeDBSnapshots for the instance (manual snapshots). */
+  describe: () => Promise<Array<{ identifier: string; status: string }>>;
+  log: (msg: string) => void;
+  sleep: (ms: number) => Promise<void>;
+  now: () => number;
+}
+
+export interface ConfirmPhysicalOptions {
+  instanceId: string;
+  snapshotId: string;
+  /** How long to wait for `available` before refusing. */
+  deadlineMs?: number;
+  /** How often to re-read the status. */
+  pollIntervalMs?: number;
+}
+
+/**
+ * A snapshot's status is polled to a TERMINAL state before the swap is allowed to continue. 15
+ * minutes is generous for a dev-sized instance and short enough that a stuck snapshot surfaces as a
+ * refusal rather than an hour of silence.
+ */
+const DEFAULT_SNAPSHOT_DEADLINE_MS = 15 * 60_000;
+const DEFAULT_SNAPSHOT_POLL_MS = 10_000;
+
+/**
+ * Take an RDS snapshot and do not return until RDS says it is `available`.
+ *
+ * Why wait at all, when the snapshot's consistency point is fixed the moment CreateDBSnapshot is
+ * accepted: because "accepted" is not "exists". A snapshot can go to `failed` (instance state,
+ * storage), and the entire value of this step is that the operator can get back. Proceeding into a
+ * destructive rename on an unconfirmed rollback point is the defect this module was written to
+ * remove — waiting is the only thing that turns the printed id into a fact.
+ *
+ * On the deadline it THROWS naming the snapshot id, because the re-run is one flag: the snapshot is
+ * still coming, so `--snapshot-ref <id>` reuses it instead of starting another.
+ */
+export async function confirmPhysicalSnapshot(
+  io: PhysicalSnapshotIO,
+  opts: ConfirmPhysicalOptions,
+): Promise<{ id: string }> {
+  const deadlineMs = opts.deadlineMs ?? DEFAULT_SNAPSHOT_DEADLINE_MS;
+  const pollMs = opts.pollIntervalMs ?? DEFAULT_SNAPSHOT_POLL_MS;
+  const created = await io.create(opts.snapshotId);
+  const id = created.identifier;
+  io.log(`physical snapshot ${id} of ${opts.instanceId} — status ${created.status}.`);
+  if (created.status === 'available') return { id };
+  if (created.status === 'failed') {
+    throw new Error(`the pre-swap RDS snapshot ${id} failed immediately, so the swap was NOT applied and live is untouched.`);
+  }
+
+  const start = io.now();
+  let lastStatus = created.status;
+  for (;;) {
+    if (io.now() - start >= deadlineMs) {
+      throw new Error(
+        `the pre-swap RDS snapshot ${id} is still "${lastStatus}" after ${Math.round(deadlineMs / 60_000)} minutes, so the swap was NOT applied and live is untouched. `
+        + `The snapshot is still being taken — it is a real rollback point once it reaches available. `
+        + `Watch it with \`everystack db:snapshots\`, then re-run this swap with --snapshot-ref ${id} to reuse it instead of taking another.`,
+      );
+    }
+    await io.sleep(pollMs);
+    const snaps = await io.describe();
+    const mine = snaps.find((s) => s.identifier === id);
+    if (!mine) {
+      throw new Error(
+        `the pre-swap RDS snapshot ${id} is no longer listed on ${opts.instanceId} — it was deleted or never registered, so there is no rollback point. `
+        + `The swap was NOT applied and live is untouched.`,
+      );
+    }
+    if (mine.status !== lastStatus) {
+      lastStatus = mine.status;
+      io.log(`physical snapshot ${id}: ${lastStatus}`);
+    }
+    if (mine.status === 'available') return { id };
+    if (mine.status === 'failed') {
+      throw new Error(`the pre-swap RDS snapshot ${id} FAILED, so the swap was NOT applied and live is untouched. Check the instance's state and storage, then retry.`);
+    }
+  }
+}