@everystack/cli 0.4.35 → 0.4.38

package/src/cli/swap-execute.ts

@@ -71,6 +71,41 @@ export interface ExecuteSwapOptions {
    * are still refused: consent cannot cover an object nothing knows how to rebuild.
    */
   rebuildDerived?: boolean;
+  /**
+   * The PAIRED swap: derived schemas that swap alongside the base schema, with their incoming
+   * twins already built over `<schema>_incoming` (see swap-pair). This is the zero-downtime path
+   * — the derived layer is recreated in parallel and renamed in the same transaction, so it is
+   * never absent and never welded to the retiring schema.
+   *
+   * Supplying this makes the CASCADE gate treat the whole set as one unit: an object depending on
+   * a paired schema is INSIDE the swap and rides along, exactly as an object inside the base
+   * schema always has.
+   */
+  paired?: string[];
+  /**
+   * Build the incoming derived layer. Runs after the artifact lands in `<schema>_incoming` and
+   * before the swap transaction, while every live schema keeps serving. Paired swaps only.
+   */
+  buildPairedDerived?: (runner: QueryRunner) => Promise<void>;
+  /**
+   * Schema-level USAGE applied inside the swap transaction (renderSwapSchemaUsage), and the
+   * (schema, role) pairs asserted after it commits. Without these a swap can land correct data
+   * behind schemas no application role can enter — see diffSchemaUsage.
+   */
+  schemaUsage?: string[];
+  schemaUsageRoles?: Array<{ schema: string; role: string }>;
+  /**
+   * The `_incoming`-qualified identities the paired build must produce (expectedIncomingObjects).
+   * Asserted before the rename: a partial derived layer refuses while live is untouched, instead
+   * of committing a swap that reports success with objects missing.
+   */
+  expectedDerived?: string[];
+  /**
+   * Record provenance for the objects the paired build created, once they are live at their final
+   * identities. Without it the next db:reconcile sees the whole layer as drift and rebuilds it —
+   * an expensive, ACCESS EXCLUSIVE no-op that surfaces days later on an unrelated run.
+   */
+  recordProvenance?: (runner: QueryRunner) => Promise<void>;
 }
 
 /** One table's landed-vs-live count, the intrinsic post-swap assertion's unit. */
@@ -158,19 +193,36 @@ export function dropDependentSql(d: CrossSchemaDependent): string {
  * Verified against a live PostgreSQL 16 fixture: a three-deep view chain, a function returning
  * SETOF a table, a view inside the schema, and an unrelated view in another schema.
  */
-export function crossSchemaDependentsQuery(schema: string, incoming: string, retiring: string): string {
+export function crossSchemaDependentsQuery(
+  schema: string,
+  incoming: string,
+  retiring: string,
+  opts: { paired?: string[]; incomingSuffix?: string; retiringSuffix?: string } = {},
+): string {
   const q = (s: string) => s.replace(/'/g, "''");
-  const skip = `ARRAY['${q(incoming)}','${q(retiring)}']`;
-  const notSelf = `n.nspname <> '${q(schema)}' AND n.nspname <> ALL (${skip})`;
+  const arr = (xs: string[]) => `ARRAY[${xs.map((s) => `'${q(s)}'`).join(',')}]`;
+  const paired = opts.paired ?? [];
+  const inSuffix = opts.incomingSuffix ?? '_incoming';
+  const reSuffix = opts.retiringSuffix ?? '_retiring';
+
+  // A PAIRED derived schema is INSIDE the swap: its objects are rebuilt into the incoming twin
+  // and renamed in the same transaction, so they are not at risk and must not trigger a refusal.
+  // They also seed the walk — something outside the set depending on a PAIRED schema is destroyed
+  // by that schema's rename + drop just as surely as a dependent of the base schema is.
+  const swapped = [schema, ...paired];
+  const twins = paired.flatMap((p) => [`${p}${inSuffix}`, `${p}${reSuffix}`]);
+  const skip = arr([incoming, retiring, ...twins]);
+  const notSelf = `n.nspname <> ALL (${arr(swapped)}) AND n.nspname <> ALL (${skip})`;
+  const seedIn = `= ANY (${arr(swapped)})`;
   return `WITH RECURSIVE seed AS (
   SELECT c.oid AS oid, 'pg_class'::regclass AS cls
-    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = '${q(schema)}'
+    FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname ${seedIn}
   UNION
   SELECT t.oid, 'pg_type'::regclass
-    FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE n.nspname = '${q(schema)}'
+    FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE n.nspname ${seedIn}
   UNION
   SELECT p.oid, 'pg_proc'::regclass
-    FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname = '${q(schema)}'
+    FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname ${seedIn}
 ), closure AS (
   SELECT oid, cls FROM seed
   UNION
@@ -199,6 +251,33 @@ SELECT DISTINCT n.nspname, p.proname,
  ORDER BY 1, 2`;
 }
 
+/**
+ * Every view / matview / function physically living in `schemas` — what a paired swap is about to
+ * replace wholesale.
+ *
+ * The paired swap renames a derived schema out and its rebuilt twin in, then drops the retiring
+ * one. The twin is built from the DECLARED descriptors, so anything live-but-undeclared in that
+ * schema has no counterpart in the twin and the drop destroys it. Pairing widens the set of
+ * schemas the gate considers "inside" the swap; this is what keeps that from becoming a licence to
+ * delete. Same reasoning as the undeclared-dependent refusal, applied one schema over.
+ */
+export function schemaObjectsQuery(schemas: string[]): string {
+  const q = (s: string) => s.replace(/'/g, "''");
+  const list = `ARRAY[${schemas.map((s) => `'${q(s)}'`).join(',')}]`;
+  return `SELECT n.nspname AS dep_schema, c.relname AS dep_name,
+       CASE c.relkind WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' ELSE c.relkind::text END AS dep_kind,
+       ''::text AS dep_args
+  FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
+ WHERE n.nspname = ANY (${list}) AND c.relkind IN ('v','m')
+UNION
+SELECT n.nspname, p.proname,
+       CASE p.prokind WHEN 'p' THEN 'procedure' ELSE 'function' END,
+       pg_get_function_identity_arguments(p.oid)
+  FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
+ WHERE n.nspname = ANY (${list})
+ ORDER BY 1, 2`;
+}
+
 /** The refusal message — names what is at risk, and which of the two refusals this is. */
 export function crossSchemaDependentsRefusal(
   schema: string,
@@ -253,6 +332,72 @@ export function isFatalVerdict(verdict: SwapVerdict): boolean {
   return checks.some((c) => !c.ok && (c.severity ?? 'fatal') === 'fatal');
 }
 
+/**
+ * The reachability assertion: every role that holds a declared grant in a swapped schema must
+ * still have USAGE on it after the swap.
+ *
+ * The row-count assertion proves the DATA landed. This proves the data can be REACHED, which is a
+ * different failure and a quieter one: a swap that lands perfect data behind a schema no
+ * application role can enter reports success while every endpoint 500s. PostgreSQL surfaces a
+ * missing schema USAGE as ABSENCE rather than denial, so the error the operator sees is
+ * `relation "..." does not exist` — pointing at the table, not the grant.
+ *
+ * Pure: the impure half reads has_schema_privilege and hands the rows here.
+ */
+export function diffSchemaUsage(rows: Array<{ schema: string; role: string; ok: boolean }>): SwapCheck[] {
+  return rows
+    .filter((r) => !r.ok)
+    .map((r) => ({
+      name: `usage:${r.schema}:${r.role}`,
+      ok: false,
+      severity: 'fatal' as const,
+      detail: `role ${r.role} has no USAGE on schema ${r.schema} after the swap — it holds declared grants there, so every one of them is unreachable (PostgreSQL will report the tables as "does not exist").`,
+    }));
+}
+
+/** `has_schema_privilege` for each (role, schema) pair — catalog-only, locks nothing. */
+export function schemaUsageQuery(pairs: Array<{ schema: string; role: string }>): string {
+  const q = (s: string) => s.replace(/'/g, "''");
+  const values = pairs.map((p) => `('${q(p.schema)}','${q(p.role)}')`).join(',');
+  return `SELECT s AS schema, r AS role,
+       (EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r)
+        AND has_schema_privilege(r, s, 'USAGE')) AS ok
+  FROM (VALUES ${values}) AS t(s, r)`;
+}
+
+/**
+ * The derived-layer completeness assertion: every declared object in the swap set must actually
+ * EXIST in the incoming schemas after the build.
+ *
+ * The row-count assertion proves the base tables landed. This proves the derived layer did. They
+ * are different failures, and this one was silent: a build that throws is caught, but a build that
+ * quietly produces 38 of 79 objects sailed straight through to a successful swap. Seen in the
+ * field — a consumer's run printed the success line with half the layer missing.
+ *
+ * The same silent-success class as the original `--direct` swap reporting success without landing
+ * anything. That got a per-table count; the derived layer never got the equivalent until now.
+ *
+ * Pure: the caller reads the incoming catalog and hands both sides here.
+ */
+export function diffDerivedObjects(expected: string[], present: string[]): SwapCheck[] {
+  const have = new Set(present);
+  const missing = expected.filter((id) => !have.has(id)).sort();
+  if (missing.length === 0) return [];
+  const shown = missing.slice(0, 20).join(', ');
+  return [{
+    name: 'derived:incomplete',
+    ok: false,
+    severity: 'fatal',
+    detail: `the incoming derived layer is INCOMPLETE — ${missing.length} of ${expected.length} declared object(s) were not built: `
+      + `${shown}${missing.length > 20 ? `, and ${missing.length - 20} more` : ''}.`,
+  }];
+}
+
+/** Every view / matview / function physically present in the given schemas, as `schema.name`. */
+export function objectsPresentQuery(schemas: string[]): string {
+  return schemaObjectsQuery(schemas);
+}
+
 /** Double-quote a Postgres identifier read from the catalog (embedded quotes doubled). */
 function quoteIdent(name: string): string {
   return `"${name.replace(/"/g, '""')}"`;
@@ -282,12 +427,43 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
     };
   }
 
-  const plan = renderSchemaSwap(opts.models, { schema: opts.schema, incoming: opts.incoming, retiring: opts.retiring });
+  const paired = opts.paired ?? [];
+  const plan = renderSchemaSwap(opts.models, {
+    schema: opts.schema,
+    incoming: opts.incoming,
+    retiring: opts.retiring,
+    paired,
+    schemaUsage: opts.schemaUsage,
+  });
 
   // 1b. THE CASCADE GATE. Refuse before the snapshot — before anything moves at all — if any object
   //     outside this schema depends on it. The swap's final DROP SCHEMA ... CASCADE would destroy
   //     them silently, and a silent destroyer is the worst thing this command could be.
-  const depRows = await runner(crossSchemaDependentsQuery(opts.schema, plan.incoming, plan.retiring));
+  // 1a. PAIRED PRE-FLIGHT. A paired schema is replaced wholesale by a twin built from the declared
+  //     descriptors, so anything live in it that is NOT declared has no counterpart in the twin
+  //     and the retiring drop would destroy it. Pairing must never become a quieter way to lose an
+  //     object than the unpaired gate already refuses to be.
+  if (paired.length > 0) {
+    const declared = new Set(opts.declaredIdentities ?? []);
+    const liveRows = await runner(schemaObjectsQuery(paired));
+    const orphans: CrossSchemaDependent[] = liveRows
+      .map((r: any) => ({ schema: r.dep_schema, name: r.dep_name, kind: r.dep_kind, args: r.dep_args || undefined }))
+      .filter((d: CrossSchemaDependent) => !declared.has(dependentIdentity(d)));
+    if (orphans.length > 0) {
+      const name = (d: CrossSchemaDependent) => `${d.schema}.${d.name} (${d.kind})`;
+      return {
+        status: 'refused-dependents',
+        reason: `${orphans.length} object(s) live in the paired schema(s) ${paired.join(', ')} but are NOT declared, so the rebuilt schema would not contain them and the swap would DESTROY them: `
+          + `${orphans.slice(0, 20).map(name).join(', ')}${orphans.length > 20 ? `, and ${orphans.length - 20} more` : ''}. `
+          + `A paired swap replaces the whole derived schema with one built from db/models — anything not declared there has nothing to rebuild it. `
+          + `Nothing was changed. Declare them in db/models, or drop them yourself if they are genuinely disposable.`,
+        dependents: orphans,
+        undeclaredDependents: orphans,
+      };
+    }
+  }
+
+  const depRows = await runner(crossSchemaDependentsQuery(opts.schema, plan.incoming, plan.retiring, { paired }));
   const dependents: CrossSchemaDependent[] = depRows.map((r: any) => ({
     schema: r.dep_schema, name: r.dep_name, kind: r.dep_kind, args: r.dep_args || undefined,
   }));
@@ -341,6 +517,51 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
     };
   }
 
+  // 3b. PAIRED: build the derived layer over the incoming base tables, while every live schema
+  //     keeps serving. This is the whole point — the layer exists in full before the rename, so
+  //     there is no window in which it is absent and no refresh that could read stale rows.
+  //     A failure here leaves live untouched: nothing has been renamed yet.
+  if (opts.buildPairedDerived) {
+    const twins = paired.map((p) => `${p}_incoming`);
+    for (const t of twins) await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(t)} CASCADE`);
+    try {
+      await opts.buildPairedDerived(runner);
+    } catch (err: any) {
+      for (const t of twins) {
+        try { await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(t)} CASCADE`); } catch { /* best effort */ }
+      }
+      try { await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(plan.incoming)} CASCADE`); } catch { /* best effort */ }
+      return {
+        status: 'refused-integrity',
+        reason: `building the incoming derived layer failed, so the swap was NOT applied and live is untouched: ${String(err?.message ?? err)}. `
+          + `The declared descriptors must compose against the INCOMING base schema — if the artifact's shape no longer matches what the derived layer selects, that mismatch surfaces here rather than after the rename.`,
+      };
+    }
+    // COMPLETENESS. A build that throws is caught above; a build that quietly produces only some
+    // of its objects is not, and used to reach a successful swap. Assert BEFORE the rename, while
+    // live is still untouched — a refusal here costs nothing, where the same finding after the
+    // rename would cost a snapshot restore.
+    if (opts.expectedDerived?.length) {
+      const rows = await runner(objectsPresentQuery(twins.length ? [plan.incoming, ...twins] : [plan.incoming]));
+      const present = rows.map((r: any) => `${r.dep_schema}.${r.dep_name}`);
+      const checks = diffDerivedObjects(opts.expectedDerived, present);
+      if (checks.length > 0) {
+        for (const t of twins) {
+          try { await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(t)} CASCADE`); } catch { /* best effort */ }
+        }
+        try { await runner(`DROP SCHEMA IF EXISTS ${quoteIdent(plan.incoming)} CASCADE`); } catch { /* best effort */ }
+        return {
+          status: 'refused-integrity',
+          reason: `${checks[0].detail} The swap was NOT applied and live is untouched. `
+            + `A partial derived layer is what a swap that reports success while half the layer is missing looks like from the inside — this refuses instead.`,
+          verdict: { ok: false, checks },
+        };
+      }
+      log(`derived layer complete: ${opts.expectedDerived.length} declared object(s) present in the incoming schemas`);
+    }
+    log(`built the incoming derived layer into ${twins.join(', ')} — live still serving the old one`);
+  }
+
   // 4. The atomic swap. A FK re-validation failure (a bad artifact) rolls the whole thing back.
   await runner('BEGIN');
   try {
@@ -372,6 +593,26 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
     };
   }
 
+  // 5a-bis. The REACHABILITY assertion. Counts prove the data landed; this proves an application
+  //     role can still get to it. A swap that commits perfect data behind a schema nobody can
+  //     enter looks like a success and reads, at the app, as every table having vanished.
+  if (opts.schemaUsageRoles?.length) {
+    const usageRows = await runner(schemaUsageQuery(opts.schemaUsageRoles));
+    const usageChecks = diffSchemaUsage(
+      usageRows.map((r: any) => ({ schema: r.schema, role: r.role, ok: r.ok === true })),
+    );
+    if (usageChecks.length > 0) {
+      if (opts.rollbackToSnapshot) await opts.rollbackToSnapshot();
+      return {
+        status: 'rolled-back-verify',
+        reason: `the swap landed but the schemas are unreachable: ${usageChecks.map((c) => c.detail).join(' ')} `
+          + `${opts.rollbackToSnapshot ? 'Rolled back to the pre-swap snapshot.' : 'NO snapshot was configured to roll back to.'}`,
+        verdict: { ok: false, checks: usageChecks },
+      };
+    }
+    log(`reachability: ${opts.schemaUsageRoles.length} (role, schema) pair(s) verified — the app can still read through the swapped schemas`);
+  }
+
   // 5b. Verify (post-commit). Fatal → roll back to the snapshot; warn → surface only.
   let verdict: SwapVerdict | undefined;
   if (opts.verify) {
@@ -386,8 +627,25 @@ export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions)
     }
   }
 
-  // 6. Drop the retiring schema — the swap is committed and verified.
-  await runner(dropRetiringSql(plan.retiring));
+  // 6. Drop EVERY retiring schema — the swap is committed and verified. On a paired swap that is
+  //    the base plus each derived twin; dropping only the base would strand the old derived
+  //    schemas, which the next swap then collides with on its own rename.
+  for (const r of plan.retiringSchemas) await runner(dropRetiringSql(r));
+
+  // 7. Bookkeeping. The build created the derived layer with raw DDL; the reconciler knows nothing
+  //    about it until this runs. Recorded AFTER the retiring drop so the catalog read behind it
+  //    sees only the live objects. A failure here does not undo a good swap — the data is correct
+  //    and serving; the cost is a redundant rebuild on the next reconcile, which is what this
+  //    prevents rather than something it can break.
+  if (opts.recordProvenance) {
+    try {
+      await opts.recordProvenance(runner);
+    } catch (err: any) {
+      log(`WARNING: the swap succeeded but recording provenance failed: ${String(err?.message ?? err)}. `
+        + `The derived layer is live and correct; the next db:reconcile will rebuild it needlessly. `
+        + `Run db:reconcile --rebaseline to record it without DDL.`);
+    }
+  }
 
   const warnings = (verdict?.checks ?? []).filter((c) => !c.ok && c.severity === 'warn');
   return { status: 'swapped', verdict, ...(warnings.length ? { warnings } : {}) };

package/src/cli/swap-heartbeat.ts

@@ -78,6 +78,24 @@ export interface HeartbeatSample {
   blockedBy: string | null;
 }
 
+/**
+ * What the CLIENT has sent, to sit alongside what the server reports.
+ *
+ * The server side alone cannot tell a stalled load from a finished one. Both look like
+ * `Client/ClientRead` with flat progress: the server is waiting for the next command. The difference
+ * is whether there is anything left to send, and only the client knows that. Without this the
+ * detector called the deadlock signature on a run that had landed all 5,984,956 rows and was one
+ * second from completing.
+ */
+export interface ClientFeed {
+  /** Bytes handed to psql's stdin so far. */
+  fedBytes: number;
+  /** Bytes the restore has to feed in total (the rewritten SQL file's size). */
+  totalBytes: number;
+  /** The local reader has written everything — nothing more is coming from our side. */
+  done: boolean;
+}
+
 export type Liveness =
   /** Rows or COPY bytes grew since the last poll. */
   | 'progressing'
@@ -85,6 +103,12 @@ export type Liveness =
   | 'blocked'
   /** Flat progress AND the server is waiting on the client: the deadlock signature. */
   | 'client-stall'
+  /**
+   * The client has sent everything and the server is waiting on it. That is not a stall — it is the
+   * tail of a successful load (the final commit, the connection winding down). Distinguishing this
+   * from `client-stall` is the whole reason ClientFeed exists.
+   */
+  | 'finishing'
   /** Connected and working, but nothing measurable moved this poll (DDL, index build). */
   | 'busy'
   /** No psql backend connected — not started yet, or already gone. */
@@ -218,9 +242,14 @@ function movedForward(prev: HeartbeatSample, cur: HeartbeatSample): boolean {
  *   3. no predecessor        → busy (cannot claim progress OR a stall on the first poll)
  *   4. rows/bytes grew       → progressing, EVEN in ClientRead (the healthy high-latency case)
  *   5. schema not created    → busy (DDL phase, nothing to measure)
- *   6. flat + ClientRead     → client-stall (the deadlock signature)
+ *   6. flat + ClientRead + the client still has bytes to send → client-stall (the deadlock signature)
+ *   7. flat + ClientRead + the client has sent everything     → finishing (the successful tail)
+ *
+ * Step 7 is the B5 fix. "Nothing is moving" was treated as sufficient evidence of a stall, and it is
+ * not: at the end of a healthy load nothing moves either. A stall claim now requires UNLANDED WORK —
+ * bytes the client still owes the server — which is why `feed` is threaded this far down.
  */
-export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample): Liveness {
+export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample, feed?: ClientFeed): Liveness {
   if (cur.state === null) return 'absent';
   if (!prev) return 'busy';
   // PROGRESS OUTRANKS EVERY SCARY WAIT STATE, including Lock. A restore takes relation locks
@@ -231,7 +260,7 @@ export function classify(prev: HeartbeatSample | null, cur: HeartbeatSample): Li
   if (movedForward(prev, cur)) return 'progressing';
   if (cur.waitEventType === 'Lock') return 'blocked';
   if (!cur.schemaExists) return 'busy';
-  if (cur.waitEvent === 'ClientRead') return 'client-stall';
+  if (cur.waitEvent === 'ClientRead') return feed?.done ? 'finishing' : 'client-stall';
   return 'busy';
 }
 
@@ -251,16 +280,70 @@ function groupNum(n: number): string {
   return n.toLocaleString('en-US');
 }
 
-/** One operator-readable line per poll. Says what moved, what it is waiting on, and how long in. */
-export function formatSample(prev: HeartbeatSample | null, cur: HeartbeatSample, liveness: Liveness): string {
+/** `1.9 GB`, `512 KB` — bytes at poll-line resolution. */
+function fmtBytes(n: number): string {
+  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+  let v = n;
+  let u = 0;
+  while (v >= 1024 && u < units.length - 1) { v /= 1024; u += 1; }
+  return `${u === 0 ? v : v.toFixed(1)} ${units[u]}`;
+}
+
+/**
+ * WHICH signal moved between two samples, named honestly.
+ *
+ * The bug (B6): the progressing line said "COPY advancing" whenever the row delta was zero,
+ * regardless of what had actually moved. So a restore in its index phase — relations growing, no
+ * COPY anywhere — printed "COPY advancing", and so did the poll right after the final COPY finished.
+ * An idle tail read as a live load. Each branch below reports only what it actually observed.
+ */
+export function progressLabel(prev: HeartbeatSample, cur: HeartbeatSample): string {
+  const dRows = (cur.rows ?? 0) - (prev.rows ?? 0);
+  if (dRows > 0) {
+    const secs = Math.max(1, (cur.elapsedMs - prev.elapsedMs) / 1000);
+    return `+${groupNum(dRows)} row(s) (${groupNum(Math.round(dRows / secs))} rows/s)`;
+  }
+  const copyMoved = cur.copyBytes > prev.copyBytes || cur.copyRows > prev.copyRows;
+  // "Advancing" is a claim about NOW, so it needs a COPY running now — not merely one that moved.
+  if (copyMoved && cur.copies > 0) return 'COPY advancing';
+  const dRel = cur.relations - prev.relations;
+  if (dRel > 0) return `+${groupNum(dRel)} relation(s) created`;
+  if (copyMoved) return 'a COPY completed since the last poll';
+  return 'something moved';
+}
+
+/** The client-side position clause — `client fed 1.9 GB/2.0 GB (95%)`. Empty when unknown. */
+function clientClause(feed?: ClientFeed): string {
+  if (!feed) return '';
+  const pct = feed.totalBytes > 0 ? Math.floor((feed.fedBytes / feed.totalBytes) * 100) : 0;
+  const sent = feed.done ? 'client has fed ALL' : 'client fed';
+  return `, ${sent} ${fmtBytes(feed.fedBytes)}/${fmtBytes(feed.totalBytes)} (${pct}%)`;
+}
+
+/**
+ * One operator-readable line per poll. Says what moved, what it is waiting on, how long in — and, when
+ * known, what the CLIENT has sent.
+ *
+ * That last part is B7. A `Client/ClientRead` wait with the client 20% through the file and the same
+ * wait with the file fully fed are completely different situations, and they used to print
+ * identically: two debugging sessions went down the wrong path because "the client is not sending"
+ * and "the instance has no I/O left" were indistinguishable in the log.
+ */
+export function formatSample(
+  prev: HeartbeatSample | null,
+  cur: HeartbeatSample,
+  liveness: Liveness,
+  feed?: ClientFeed,
+): string {
   const at = `t+${humanElapsed(cur.elapsedMs)}`;
   const wait = cur.waitEventType ? `${cur.waitEventType}/${cur.waitEvent ?? '?'}` : (cur.state ?? 'idle');
+  const client = clientClause(feed);
 
   if (liveness === 'absent') {
     return `restore: no psql backend connected to the target (${at}) — the loader has not started yet, or has already exited.`;
   }
   if (!cur.schemaExists) {
-    return `restore: schema not created yet, no tables yet — DDL phase, ${wait} (${at}).`;
+    return `restore: schema not created yet, no tables yet — DDL phase, ${wait}${client} (${at}).`;
   }
 
   const rows = cur.rows === null ? 'unknown' : groupNum(cur.rows);
@@ -275,20 +358,18 @@ export function formatSample(prev: HeartbeatSample | null, cur: HeartbeatSample,
   if (liveness === 'blocked') {
     // Naming the holder is the difference between an actionable report and a shrug.
     const by = cur.blockedBy ? ` HELD BY ${cur.blockedBy}` : ' (holder not visible — it may belong to another role)';
-    return `restore: waiting on ${wait} with no progress this poll${by} — ${scope} (${at}).`;
+    return `restore: waiting on ${wait} with no progress this poll${by} — ${scope}${client} (${at}).`;
+  }
+  if (liveness === 'finishing') {
+    return `restore: ${scope}${client} — FINISHING: everything has been sent, the server is winding the load down (${wait}), ${at}.`;
   }
   if (liveness === 'client-stall') {
-    return `restore: nothing landed since the last poll and the server is waiting on the client (${wait}) — ${scope} (${at}).`;
+    return `restore: nothing landed since the last poll and the server is waiting on the client (${wait}) — ${scope}${client} (${at}).`;
   }
   if (liveness === 'progressing' && prev) {
-    const dRows = (cur.rows ?? 0) - (prev.rows ?? 0);
-    const secs = Math.max(1, (cur.elapsedMs - prev.elapsedMs) / 1000);
-    const moved = dRows > 0
-      ? `+${groupNum(dRows)} row(s) (${groupNum(Math.round(dRows / secs))} rows/s)`
-      : 'COPY advancing';
-    return `restore: ${scope}, ${moved} — ${wait} (${at}).`;
+    return `restore: ${scope}, ${progressLabel(prev, cur)} — ${wait}${client} (${at}).`;
   }
-  return `restore: ${scope}, nothing new this poll — ${wait} (${at}).`;
+  return `restore: ${scope}, nothing new this poll — ${wait}${client} (${at}).`;
 }
 
 /**
@@ -335,6 +416,12 @@ export interface HeartbeatOptions {
    * backend for. Must not throw; a sink that does is ignored so it cannot kill the heartbeat.
    */
   onSample?: (sample: HeartbeatSample) => void;
+  /**
+   * The client's current write position, read fresh each poll. Supplying it is what lets the
+   * heartbeat tell a stalled load from a finishing one (see ClientFeed) — without it the detector
+   * falls back to server-side-only reasoning and will call a quiet tail a stall.
+   */
+  clientFeed?: () => ClientFeed;
 }
 
 /**
@@ -363,8 +450,10 @@ export function startHeartbeat(runner: QueryRunner, opts: HeartbeatOptions): ()
     // Raw sample first: a caller acting on the probe (the dead-backend watchdog) must see every
     // sample, and must never be able to break the heartbeat by throwing.
     try { opts.onSample?.(cur); } catch { /* a sink's failure is not the probe's problem */ }
-    const liveness = classify(prev, cur);
-    const line = formatSample(prev, cur, liveness);
+    let feed: ClientFeed | undefined;
+    try { feed = opts.clientFeed?.(); } catch { /* same: a bad reader must not kill the probe */ }
+    const liveness = classify(prev, cur, feed);
+    const line = formatSample(prev, cur, liveness, feed);
 
     if (liveness === 'blocked') {
       warn(line);