@everystack/cli 0.4.33 → 0.4.35

package/src/cli/swap-heartbeat.ts

@@ -0,0 +1,407 @@
+/**
+ * swap-heartbeat — liveness for db:swap's restore leg, read from the OTHER side of the connection.
+ *
+ * The problem: `restoreIntoIncoming` spawns `psql -f <file>` and prints nothing until it exits.
+ * Both stderr streams are accumulated into strings and surfaced only on a non-zero exit, so for the
+ * whole multi-GB push a wedged restore and a healthy one produce the identical observation: silence.
+ *
+ * The probe: db:swap already holds an operator connection (`createUrlRunner`, max: 1) that sits
+ * COMPLETELY IDLE for the duration of the restore — `applyIncoming` ignores the runner it is handed
+ * and spawns psql on its own connection. So liveness costs no new connection and no new credential.
+ *
+ * ============================================================================================
+ * THE PROBE MUST NEVER TOUCH A RELATION. Learned the hard way, 2026-07-27.
+ * ============================================================================================
+ * The first version summed `pg_total_relation_size()` over the incoming schema. That function calls
+ * `relation_open(relid, AccessShareLock)` — it LOCKS every relation it measures, and holds those
+ * locks until the whole `sum()` completes. Against a restore doing DDL, two things follow:
+ *
+ *   1. The probe BLOCKS behind the restore's AccessExclusiveLock. Measured: it hung until killed by
+ *      `statement_timeout`; unbounded without one. A liveness probe that hangs is worse than none —
+ *      it reports "no backend connected" about a restore that is fine.
+ *   2. Worse, it blocks the RESTORE. PostgreSQL queues lock requests fairly, so once the probe's
+ *      pending AccessShare is in line, the restore's next AccessExclusive queues behind IT. Polling
+ *      every 10s injected a lock barrier across every table in the schema.
+ *
+ * Everything below reads ONLY statistics and catalog views — `pg_stat_all_tables`,
+ * `pg_stat_progress_copy`, `pg_stat_activity`, `pg_namespace`. All verified lock-free against a held
+ * AccessExclusiveLock (0.03s vs. blocking-until-killed). Do not reintroduce a relation-size call,
+ * `relpages` arithmetic aside — if you need bytes, `pg_stat_progress_copy.bytes_processed` is the
+ * lock-free source.
+ *
+ * Two signals, and the pair is what discriminates:
+ *   - rows landed (committed inserts + the in-flight COPY's tuples) — is data actually arriving
+ *   - the loader's wait state — what is it waiting ON
+ *
+ * Neither alone is enough. Over a high-latency link a HEALTHY COPY sits in `Client/ClientRead`
+ * constantly, between chunks — treating that as a stall cries wolf on every normal restore. The
+ * deadlock signature is `ClientRead` with FLAT progress across consecutive polls: the server is
+ * waiting for a client that has stopped sending. `Lock` is never healthy here and reports at once.
+ */
+
+import type { QueryRunner } from './authz-contract.js';
+
+/** One poll: what has landed, and what the loading backend is doing. */
+export interface HeartbeatSample {
+  /** ms since the restore phase started. */
+  elapsedMs: number;
+  /**
+   * Rows landed, from `n_tup_ins` ALONE. Null before the schema exists.
+   *
+   * Do NOT add the in-flight COPY's tuples to this. The first version did, on the assumption that
+   * a COPY's tuples only enter `n_tup_ins` at commit. They do not: the backend flushes stats while
+   * the COPY is still running, so the two overlap. Measured on a real run — it reported 767,883
+   * rows landed for an artifact whose entire SQL file contains at most 345,663. In-flight tuples
+   * are carried separately as `copyRows` and reported, never summed.
+   */
+  rows: number | null;
+  /** Tuples processed by in-flight COPYs. Separate from `rows` — see above. */
+  copyRows: number;
+  /** Bytes processed by in-flight COPYs; 0 between them. The finer-grained signal inside one COPY. */
+  copyBytes: number;
+  /** How many COPY commands are running right now. */
+  copies: number;
+  /** Tables present in the incoming schema. */
+  tables: number;
+  /**
+   * ALL relations in the schema — tables, indexes, sequences, matviews. Its growth is the progress
+   * signal for the DDL and index phases, when no row has landed yet and nothing is COPYing.
+   */
+  relations: number;
+  /** Whether the incoming schema exists yet — keeps "empty" and "absent" distinguishable. */
+  schemaExists: boolean;
+  /** The loading backend's state, or null when no psql backend is connected. */
+  state: string | null;
+  waitEventType: string | null;
+  waitEvent: string | null;
+  /** Who holds the lock we are waiting on: `pid N [app] query...`. Null unless blocked. */
+  blockedBy: string | null;
+}
+
+export type Liveness =
+  /** Rows or COPY bytes grew since the last poll. */
+  | 'progressing'
+  /** Waiting on a lock — never healthy on this path. */
+  | 'blocked'
+  /** Flat progress AND the server is waiting on the client: the deadlock signature. */
+  | 'client-stall'
+  /** Connected and working, but nothing measurable moved this poll (DDL, index build). */
+  | 'busy'
+  /** No psql backend connected — not started yet, or already gone. */
+  | 'absent';
+
+/** Consecutive stalled polls before the heartbeat escalates from info to warn. */
+export const STALL_POLLS = 3;
+
+/**
+ * Minimum quiet time since the last observed progress before a stall is called out.
+ *
+ * STALL_POLLS alone fires at the TAIL of a healthy run: a real restore ended with three flat polls
+ * and the warning printed one second before it completed successfully. Requiring a grace window
+ * since progress was last SEEN — not merely N polls in a row — keeps the alarm for genuine stalls.
+ */
+export const STALL_GRACE_MS = 60_000;
+
+/** Default ms between polls. Long enough to be quiet, short enough to localize a stall. */
+const DEFAULT_INTERVAL_MS = 10_000;
+
+/**
+ * Client-side cap on one poll. The query is provably lock-free, but a hung CONNECTION (network,
+ * server-side termination) could still park it forever, and a probe that never returns is exactly
+ * the failure this module exists to report. Racing in JS also covers hangs no `statement_timeout`
+ * would catch, because it does not depend on the server being responsive at all.
+ */
+export const PROBE_TIMEOUT_MS = 5_000;
+
+/** Single-quote a string literal for inlining into the probe SQL. */
+function quoteLiteral(v: string): string {
+  return v.replace(/'/g, "''");
+}
+
+/**
+ * The probe, as ONE round trip. Statistics and catalog views ONLY — see the module header. Every
+ * CTE here was verified to return in ~0.03s while another session held an AccessExclusiveLock on
+ * the schema's tables.
+ */
+export function heartbeatQuery(incoming: string): string {
+  const ns = quoteLiteral(incoming);
+  return `WITH ns AS (
+  SELECT oid FROM pg_namespace WHERE nspname = '${ns}'
+), tbl AS (
+  SELECT coalesce(sum(n_tup_ins), 0)::bigint AS rows_in,
+         count(*)::int AS tables
+  FROM pg_stat_all_tables WHERE schemaname = '${ns}'
+), rel AS (
+  -- EVERY relation, not just tables: indexes, sequences and matviews are relations, so this keeps
+  -- producing a progress signal through the CREATE INDEX phase after the data has landed.
+  SELECT count(*)::int AS relations
+  FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = '${ns}'
+), cp AS (
+  -- Scoped to OUR load: COPY FROM only (never a concurrent pg_dump's COPY TO), and only into
+  -- relations in the incoming schema. Unscoped, this counted every COPY in the database — on a
+  -- real run it reported 2,269,567 rows in flight for an artifact containing 341,160, because a
+  -- concurrent dump was streaming out. Worse than noise: "COPY advancing" reads as progress, so
+  -- another session's work could mask our restore being genuinely stuck.
+  SELECT count(*)::int AS copies,
+         coalesce(sum(p.bytes_processed), 0)::bigint AS copy_bytes,
+         coalesce(sum(p.tuples_processed), 0)::bigint AS copy_tuples
+  FROM pg_stat_progress_copy p
+  JOIN pg_class c     ON c.oid = p.relid
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+  WHERE p.datname = current_database()
+    AND p.command = 'COPY FROM'
+    AND n.nspname = '${ns}'
+), be AS (
+  -- pg_blocking_pids answers the question a bare wait_event cannot: WHO is holding the lock.
+  -- Without it "Lock/relation" is unactionable and invites guessing at the culprit.
+  SELECT a.state, a.wait_event_type, a.wait_event,
+         (SELECT string_agg(
+                   format('pid %s [%s] %s', b.pid,
+                          coalesce(nullif(b.application_name, ''), '?'),
+                          left(regexp_replace(coalesce(b.query, ''), '\\s+', ' ', 'g'), 80)),
+                   '; ')
+            FROM pg_stat_activity b
+           WHERE b.pid = ANY (pg_blocking_pids(a.pid))) AS blocked_by
+  FROM pg_stat_activity a
+  WHERE a.datname = current_database()
+    AND a.pid <> pg_backend_pid()
+    AND a.application_name = 'psql'
+  ORDER BY a.backend_start DESC
+  LIMIT 1
+)
+SELECT (SELECT count(*) FROM ns) > 0 AS schema_exists,
+       tbl.rows_in, tbl.tables, rel.relations,
+       cp.copies, cp.copy_bytes, cp.copy_tuples,
+       be.state, be.wait_event_type, be.wait_event, be.blocked_by
+FROM tbl CROSS JOIN rel CROSS JOIN cp LEFT JOIN be ON true`;
+}
+
+/** Map a probe row to a sample. A missing row, or a schema that does not exist, means null rows. */
+export function toSample(row: any, elapsedMs: number): HeartbeatSample {
+  const schemaExists = row?.schema_exists === true;
+  const committed = row?.rows_in == null ? null : Number(row.rows_in);
+  return {
+    elapsedMs,
+    rows: schemaExists && committed !== null ? committed : null,
+    copyRows: Number(row?.copy_tuples ?? 0),
+    copyBytes: Number(row?.copy_bytes ?? 0),
+    copies: Number(row?.copies ?? 0),
+    tables: Number(row?.tables ?? 0),
+    relations: Number(row?.relations ?? 0),
+    schemaExists,
+    state: row?.state ?? null,
+    waitEventType: row?.wait_event_type ?? null,
+    waitEvent: row?.wait_event ?? null,
+    blockedBy: row?.blocked_by ?? null,
+  };
+}
+
+/**
+ * Did anything measurable move between two samples? Any of the three independent signals counts.
+ * `rows` is monotonic but coarse; the COPY counters are live but reset to 0 when a COPY finishes,
+ * so a drop in them is never evidence of a stall — only a rise is evidence of progress.
+ */
+function movedForward(prev: HeartbeatSample, cur: HeartbeatSample): boolean {
+  if (prev.rows !== null && cur.rows !== null && cur.rows > prev.rows) return true;
+  if (cur.copyBytes > prev.copyBytes) return true;
+  if (cur.copyRows > prev.copyRows) return true;
+  // Schema growth. Without this a restore visibly creating sixteen tables every poll read as a
+  // stall and tripped the deadlock alarm, because no ROW had landed yet — the data phase had not
+  // started. Every phase of a restore must have a signal, or the quiet ones look like hangs.
+  return cur.relations > prev.relations;
+}
+
+/**
+ * The discrimination. Order is the whole point:
+ *   1. no backend            → absent (nothing else can be said)
+ *   2. Lock                  → blocked, even mid-progress, and even with no baseline
+ *   3. no predecessor        → busy (cannot claim progress OR a stall on the first poll)
+ *   4. rows/bytes grew       → progressing, EVEN in ClientRead (the healthy high-latency case)
+ *   5. schema not created    → busy (DDL phase, nothing to measure)
+ *   6. flat + ClientRead     → client-stall (the deadlock signature)
+ */
+export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample): Liveness {
+  if (cur.state === null) return 'absent';
+  if (!prev) return 'busy';
+  // PROGRESS OUTRANKS EVERY SCARY WAIT STATE, including Lock. A restore takes relation locks
+  // constantly while building 84 indexes and 37 constraints, so a 10s sample catches it mid-wait
+  // routinely. The first version returned 'blocked' on sight of a Lock and printed "nothing
+  // landing" over a run whose row count was climbing 25k every poll. Same rule as ClientRead: the
+  // wait state only means something when nothing is moving.
+  if (movedForward(prev, cur)) return 'progressing';
+  if (cur.waitEventType === 'Lock') return 'blocked';
+  if (!cur.schemaExists) return 'busy';
+  if (cur.waitEvent === 'ClientRead') return 'client-stall';
+  return 'busy';
+}
+
+/** `12s`, `1m30s`, `2h05m01s`. */
+export function humanElapsed(ms: number): string {
+  const total = Math.max(0, Math.floor(ms / 1000));
+  const h = Math.floor(total / 3600);
+  const m = Math.floor((total % 3600) / 60);
+  const s = total % 60;
+  if (h) return `${h}h${String(m).padStart(2, '0')}m${String(s).padStart(2, '0')}s`;
+  if (m) return `${m}m${String(s).padStart(2, '0')}s`;
+  return `${s}s`;
+}
+
+/** `1,234,567` — rows read better grouped. */
+function groupNum(n: number): string {
+  return n.toLocaleString('en-US');
+}
+
+/** One operator-readable line per poll. Says what moved, what it is waiting on, and how long in. */
+export function formatSample(prev: HeartbeatSample | null, cur: HeartbeatSample, liveness: Liveness): string {
+  const at = `t+${humanElapsed(cur.elapsedMs)}`;
+  const wait = cur.waitEventType ? `${cur.waitEventType}/${cur.waitEvent ?? '?'}` : (cur.state ?? 'idle');
+
+  if (liveness === 'absent') {
+    return `restore: no psql backend connected to the target (${at}) — the loader has not started yet, or has already exited.`;
+  }
+  if (!cur.schemaExists) {
+    return `restore: schema not created yet, no tables yet — DDL phase, ${wait} (${at}).`;
+  }
+
+  const rows = cur.rows === null ? 'unknown' : groupNum(cur.rows);
+  // In-flight COPY tuples are reported alongside the landed count, never folded into it.
+  const copying = cur.copies > 0
+    ? `, ${cur.copies} COPY running (${groupNum(cur.copyRows)} rows in flight)`
+    : '';
+  const scope = `${rows} row(s) landed across ${cur.tables} table(s)${copying}`;
+
+  // Every branch below states only what it actually read. The first version hardcoded "nothing
+  // landing" into the blocked branch, which then printed over a run whose row count was climbing.
+  if (liveness === 'blocked') {
+    // Naming the holder is the difference between an actionable report and a shrug.
+    const by = cur.blockedBy ? ` HELD BY ${cur.blockedBy}` : ' (holder not visible — it may belong to another role)';
+    return `restore: waiting on ${wait} with no progress this poll${by} — ${scope} (${at}).`;
+  }
+  if (liveness === 'client-stall') {
+    return `restore: nothing landed since the last poll and the server is waiting on the client (${wait}) — ${scope} (${at}).`;
+  }
+  if (liveness === 'progressing' && prev) {
+    const dRows = (cur.rows ?? 0) - (prev.rows ?? 0);
+    const secs = Math.max(1, (cur.elapsedMs - prev.elapsedMs) / 1000);
+    const moved = dRows > 0
+      ? `+${groupNum(dRows)} row(s) (${groupNum(Math.round(dRows / secs))} rows/s)`
+      : 'COPY advancing';
+    return `restore: ${scope}, ${moved} — ${wait} (${at}).`;
+  }
+  return `restore: ${scope}, nothing new this poll — ${wait} (${at}).`;
+}
+
+/**
+ * Run one probe. Never throws and never hangs: a probe that fails must not kill the restore it is
+ * watching, and a probe that parks forever is the failure it exists to report.
+ */
+export async function pollOnce(
+  runner: QueryRunner,
+  incoming: string,
+  elapsedMs: number,
+  timeoutMs: number = PROBE_TIMEOUT_MS,
+): Promise<HeartbeatSample> {
+  let timer: any;
+  const timeout = new Promise<undefined>((resolve) => {
+    timer = setTimeout(() => resolve(undefined), timeoutMs);
+    timer.unref?.();
+  });
+  try {
+    const rows = await Promise.race([
+      runner(heartbeatQuery(incoming)).catch(() => undefined),
+      timeout,
+    ]);
+    return toSample(Array.isArray(rows) ? rows[0] : undefined, elapsedMs);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+export interface HeartbeatOptions {
+  /** The schema the restore is landing into. */
+  incoming: string;
+  /** Per-poll line sink. */
+  log: (msg: string) => void;
+  /** Escalation sink for a sustained stall or a lock wait. Defaults to `log`. */
+  warn?: (msg: string) => void;
+  intervalMs?: number;
+  /** Injectable clock, for tests. */
+  now?: () => number;
+  /** Per-poll cap; exposed for tests. */
+  timeoutMs?: number;
+  /**
+   * Every sample, raw, before any classification. This is how a caller acts on what the probe sees
+   * rather than only reading about it — db:swap uses it to kill a psql that the server has no
+   * backend for. Must not throw; a sink that does is ignored so it cannot kill the heartbeat.
+   */
+  onSample?: (sample: HeartbeatSample) => void;
+}
+
+/**
+ * Poll the idle operator connection until stopped. Returns the stopper, which clears the timer and
+ * awaits any in-flight poll so nothing logs after the caller has moved on.
+ *
+ * Ticks are SKIPPED while one is in flight, never queued. Chaining them (`inFlight.then(tick)`)
+ * serializes but does not drop: one slow poll lets `setInterval` stack dozens behind it, and they
+ * all drain at once the moment it resolves — a wall of identical lines that buries the real signal.
+ * Observed for real: a 7-minute poll produced ~40 stacked lines in a 5-second burst.
+ */
+export function startHeartbeat(runner: QueryRunner, opts: HeartbeatOptions): () => Promise<void> {
+  const now = opts.now ?? (() => Date.now());
+  const warn = opts.warn ?? opts.log;
+  const startedAt = now();
+  let prev: HeartbeatSample | null = null;
+  let consecutiveStalls = 0;
+  let lastProgressMs = 0;
+  let inFlight: Promise<void> = Promise.resolve();
+  let running = false;
+  let stopped = false;
+
+  const tick = async () => {
+    const cur = await pollOnce(runner, opts.incoming, now() - startedAt, opts.timeoutMs);
+    if (stopped) return;
+    // Raw sample first: a caller acting on the probe (the dead-backend watchdog) must see every
+    // sample, and must never be able to break the heartbeat by throwing.
+    try { opts.onSample?.(cur); } catch { /* a sink's failure is not the probe's problem */ }
+    const liveness = classify(prev, cur);
+    const line = formatSample(prev, cur, liveness);
+
+    if (liveness === 'blocked') {
+      warn(line);
+    } else if (liveness === 'client-stall') {
+      consecutiveStalls += 1;
+      // Two conditions, not one. A stall must SUSTAIN (one flat poll is normal between COPY
+      // chunks) AND enough quiet must have passed since progress was last seen — otherwise the
+      // warning fires on the tail of a healthy run that is about to finish.
+      const quietMs = cur.elapsedMs - lastProgressMs;
+      if (consecutiveStalls >= STALL_POLLS && quietMs >= STALL_GRACE_MS) {
+        warn(
+          `${line} STALLED for ${consecutiveStalls} consecutive polls and ${humanElapsed(quietMs)} with no progress — this is the deadlock signature (the server is waiting for data psql is not sending).`,
+        );
+      } else {
+        opts.log(line);
+      }
+    } else {
+      consecutiveStalls = 0;
+      if (liveness === 'progressing') lastProgressMs = cur.elapsedMs;
+      opts.log(line);
+    }
+    prev = cur;
+  };
+
+  const timer = setInterval(() => {
+    if (running || stopped) return; // SKIP, never queue — see the doc comment.
+    running = true;
+    inFlight = tick()
+      .catch(() => {})
+      .finally(() => { running = false; });
+  }, opts.intervalMs ?? DEFAULT_INTERVAL_MS);
+  // Never hold the process open on the heartbeat alone.
+  (timer as any).unref?.();
+
+  return async () => {
+    stopped = true;
+    clearInterval(timer);
+    await inFlight.catch(() => {});
+  };
+}