@effectstream/runtime 0.100.20 → 0.100.22

package/test/reproduction/buffering.test.ts

@@ -1,389 +0,0 @@
-/**
- * Verifies the fetch-backpressure feature (Option B′) over the synthetic `test`
- * chain through the real runtime (`start()` + in-process PGLite).
- *
- *  1a — unbounded buffering: with a cap, the fetch loop pauses when the buffer is
- *       full, so it stays bounded (~cap + stepSize) instead of racing to the whole
- *       backlog — while the node keeps applying blocks (liveness).
- *  1b — head-of-line blocking: a stalled slow chain STILL halts block production
- *       (the merge gate is correct), but the fast chain's buffer is now bounded by
- *       the cap instead of ballooning behind the head-of-line block.
- *
- * Before the fix these buffers reached ~49k–50k (see
- * e2e/perf/results/BACKPRESSURE-BASELINE.md); these tests assert they now
- * stay at the cap. Each test also writes a JSON time-series artifact to
- * e2e/perf/results/ whose field names match e2e/perf/metrics.ts `Sample`, so the
- * perf HTML report's Backpressure section can render the same signals.
- */
-import { afterEach, beforeEach, expect, test } from "bun:test";
-import {
-  ConfigBuilder,
-  ConfigNetworkType,
-  ConfigSyncProtocolType,
-} from "@effectstream/config";
-import { TestChainControl } from "@effectstream/sync";
-import fs from "node:fs";
-import path from "node:path";
-import {
-  bufferSize,
-  type Harness,
-  latestFinalizedHeight,
-  type RunningNode,
-  setupHarness,
-  sleep,
-  TEST_PRIMITIVE_TYPE,
-} from "./harness.ts";
-
-const START_TIME = 1_700_000_000_000;
-const BLOCK_TIME_MS = 1000;
-const STEP_SIZE = 1000;
-// A deep backlog the arithmetic fetcher would race through if uncapped (baseline
-// hit ~49k–50k). The slow chain (1b) is pinned low so the merge stalls at its tip.
-const BIG_TIP = 50_000;
-const SLOW_TIP = 50;
-// Explicit small cap so the bound is deterministic and far below the backlog.
-// The fetch pauses at >= CAP; one in-flight chunk can overshoot by ~stepSize, so
-// the buffer settles in [~CAP, CAP + stepSize]. We assert it lands in that band.
-const CAP = 3000;
-const CAP_FLOOR = CAP - STEP_SIZE; // proves backpressure actually engaged
-const BUFFER_BOUND = CAP + STEP_SIZE + 200; // upper bound incl. one overshoot chunk
-
-// e2e/perf/results lives at the repo root: reproduction → test → runtime →
-// node-sdk → packages → <root>/e2e/perf/results.
-const RESULTS_DIR = path.resolve(
-  import.meta.dirname!,
-  "../../../../../e2e/perf/results",
-);
-
-/** One time-series row; field names mirror e2e/perf/metrics.ts `Sample`. */
-type BufSample = {
-  t: number;
-  wall: number;
-  mainBuf: number;
-  evmBuf: number; // the parallel chain's buffer (report labels it "parallel buf")
-  mainOwnBlock: number | null;
-  evmOwnBlock: number | null;
-  rss: number;
-  appliedBlock: number | null;
-};
-
-function ownBlock(node: RunningNode, name: string): number | null {
-  return node.syncProtocols().find((p) => p.name === name)?.lastPage
-    ?.ownBlockNumber ?? null;
-}
-
-/** Backpressure "fired" counter (rising edges) exposed on SyncState / /debug/metrics. */
-function pauses(node: RunningNode, name: string): number {
-  return node.syncProtocols().find((p) => p.name === name)
-    ?.backpressurePauses ?? 0;
-}
-
-/** Side-effect-only artifact write; never throws into the test. */
-function writeArtifact(
-  name: string,
-  samples: BufSample[],
-  meta: Record<string, unknown>,
-): void {
-  try {
-    fs.mkdirSync(RESULTS_DIR, { recursive: true });
-    const stamp = new Date().toISOString().replace(/[:.]/g, "-");
-    const file = path.join(RESULTS_DIR, `${name}-${stamp}.json`);
-    fs.writeFileSync(file, JSON.stringify({ meta, samples }, null, 2));
-    console.log(`buffering artifact: ${file}`);
-  } catch (e) {
-    console.warn(`failed to write buffering artifact: ${String(e)}`);
-  }
-}
-
-let h: Harness;
-beforeEach(async () => {
-  h = await setupHarness();
-});
-afterEach(async () => {
-  await h?.teardown();
-});
-
-// ── 1a — unbounded buffering, now bounded by the cap ────────────────────────────
-
-const MAIN_1A = "mainClock1a";
-const PAR_1A = "parallelP1a";
-
-function buildConfig1a() {
-  return new ConfigBuilder()
-    .setNamespace((b) => b.setSecurityNamespace("sync-buffering-1a"))
-    .buildNetworks((b) =>
-      b
-        .addNetwork({
-          name: "clock",
-          type: ConfigNetworkType.TEST,
-          startTime: START_TIME,
-          blockTimeMS: BLOCK_TIME_MS,
-        })
-        .addNetwork({
-          name: "chainP",
-          type: ConfigNetworkType.TEST,
-          startTime: START_TIME,
-          blockTimeMS: BLOCK_TIME_MS,
-        })
-    )
-    .buildDeployments((b) => b)
-    .buildSyncProtocols((b) =>
-      b
-        .addMain(
-          (n) => n.clock,
-          () => ({
-            name: MAIN_1A,
-            type: ConfigSyncProtocolType.TEST_MAIN,
-            startBlockHeight: 1,
-            pollingInterval: 30,
-            stepSize: STEP_SIZE,
-            maxBufferedPages: CAP,
-          }),
-        )
-        .addParallel(
-          (n) => (n as any).chainP,
-          () => ({
-            name: PAR_1A,
-            type: ConfigSyncProtocolType.TEST_PARALLEL,
-            startBlockHeight: 1,
-            pollingInterval: 30,
-            stepSize: STEP_SIZE,
-            delayMs: 0,
-            events: [],
-            maxBufferedPages: CAP,
-          }),
-        )
-    )
-    .buildPrimitives((b) =>
-      b.addPrimitive(
-        (sp) => (sp as any)[PAR_1A],
-        () => ({ name: "noEvt1a", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
-      )
-    )
-    .build();
-}
-
-test("1a: backpressure caps the buffer while the node keeps applying blocks", async () => {
-  // Tips far ahead of where the one-block-per-txn merge can drain to: without a
-  // cap the fetcher would race to ~50k; with the cap it must plateau near CAP.
-  TestChainControl.setTip(MAIN_1A, BIG_TIP);
-  TestChainControl.setTip(PAR_1A, BIG_TIP);
-
-  const node = await h.runNode({ config: buildConfig1a(), apiPort: 19141 });
-  const samples: BufSample[] = [];
-  let peakMain = 0;
-  let peakEvm = 0;
-  let lastApplied = 0;
-  let mainPauses = 0;
-  try {
-    const startMs = Date.now();
-    const deadline = startMs + 6000;
-    while (Date.now() < deadline) {
-      const wall = Date.now();
-      const mainBuf = bufferSize(node, MAIN_1A);
-      const evmBuf = bufferSize(node, PAR_1A);
-      const appliedBlock = await latestFinalizedHeight(h.pool);
-      samples.push({
-        t: wall - startMs,
-        wall,
-        mainBuf,
-        evmBuf,
-        mainOwnBlock: ownBlock(node, MAIN_1A),
-        evmOwnBlock: ownBlock(node, PAR_1A),
-        rss: process.memoryUsage().rss,
-        appliedBlock,
-      });
-      peakMain = Math.max(peakMain, mainBuf);
-      peakEvm = Math.max(peakEvm, evmBuf);
-      lastApplied = appliedBlock ?? lastApplied;
-      // Backpressure engaged (buffer hit the cap) AND the node is still applying
-      // blocks → we've captured the bounded steady state; stop early.
-      if (peakMain >= CAP && lastApplied >= 20) break;
-      await sleep(50);
-    }
-  } finally {
-    mainPauses = pauses(node, MAIN_1A);
-    writeArtifact("buffering-1a", samples, {
-      mode: "1a-bounded-by-cap",
-      stepSize: STEP_SIZE,
-      bigTip: BIG_TIP,
-      cap: CAP,
-      bufferBound: BUFFER_BOUND,
-      peakMainBuf: peakMain,
-      peakEvmBuf: peakEvm,
-      lastApplied,
-      pauses: mainPauses,
-    });
-    await node.stop();
-  }
-
-  // Backpressure engaged: the buffer reached the cap (not starved)...
-  expect(peakMain).toBeGreaterThanOrEqual(CAP_FLOOR);
-  // ...but stayed bounded — nowhere near the uncapped ~49k backlog.
-  expect(Math.max(peakMain, peakEvm)).toBeLessThanOrEqual(BUFFER_BOUND);
-  // ...the node kept producing/applying blocks (no deadlock)...
-  expect(lastApplied).toBeGreaterThanOrEqual(20);
-  // ...and the cap actually fired (the observability metric populated).
-  expect(mainPauses).toBeGreaterThan(0);
-}, 60_000);
-
-// ── 1b — head-of-line blocking, sibling buffer now bounded ──────────────────────
-
-const MAIN_1B = "mainClock1b";
-const FAST_1B = "fastP1b";
-const SLOW_1B = "slowP1b";
-
-function buildConfig1b() {
-  return new ConfigBuilder()
-    .setNamespace((b) => b.setSecurityNamespace("sync-buffering-1b"))
-    .buildNetworks((b) =>
-      b
-        .addNetwork({
-          name: "clock",
-          type: ConfigNetworkType.TEST,
-          startTime: START_TIME,
-          blockTimeMS: BLOCK_TIME_MS,
-        })
-        .addNetwork({
-          name: "chainFast",
-          type: ConfigNetworkType.TEST,
-          startTime: START_TIME,
-          blockTimeMS: BLOCK_TIME_MS,
-        })
-        .addNetwork({
-          name: "chainSlow",
-          type: ConfigNetworkType.TEST,
-          startTime: START_TIME,
-          blockTimeMS: BLOCK_TIME_MS,
-        })
-    )
-    .buildDeployments((b) => b)
-    .buildSyncProtocols((b) =>
-      b
-        .addMain(
-          (n) => n.clock,
-          () => ({
-            name: MAIN_1B,
-            type: ConfigSyncProtocolType.TEST_MAIN,
-            startBlockHeight: 1,
-            pollingInterval: 30,
-            stepSize: STEP_SIZE,
-            maxBufferedPages: CAP,
-          }),
-        )
-        .addParallel(
-          (n) => (n as any).chainFast,
-          () => ({
-            name: FAST_1B,
-            type: ConfigSyncProtocolType.TEST_PARALLEL,
-            startBlockHeight: 1,
-            pollingInterval: 30,
-            stepSize: STEP_SIZE,
-            delayMs: 0,
-            events: [],
-            maxBufferedPages: CAP,
-          }),
-        )
-        .addParallel(
-          (n) => (n as any).chainSlow,
-          () => ({
-            name: SLOW_1B,
-            type: ConfigSyncProtocolType.TEST_PARALLEL,
-            startBlockHeight: 1,
-            pollingInterval: 30,
-            stepSize: STEP_SIZE,
-            delayMs: 0,
-            events: [],
-            maxBufferedPages: CAP,
-          }),
-        )
-    )
-    .buildPrimitives((b) =>
-      b
-        .addPrimitive(
-          (sp) => (sp as any)[FAST_1B],
-          () => ({ name: "noEvtFast", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
-        )
-        .addPrimitive(
-          (sp) => (sp as any)[SLOW_1B],
-          () => ({ name: "noEvtSlow", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
-        )
-    )
-    .build();
-}
-
-test("1b: a stalled chain still halts production, but the fast chain's buffer stays capped", async () => {
-  // slowP pinned at a low tip → the merge stalls there forever (correct). fastP and
-  // the main clock keep fetching toward their high tips, but the cap now bounds them.
-  TestChainControl.setTip(MAIN_1B, BIG_TIP);
-  TestChainControl.setTip(FAST_1B, BIG_TIP);
-  TestChainControl.setTip(SLOW_1B, SLOW_TIP);
-
-  const node = await h.runNode({ config: buildConfig1b(), apiPort: 19142 });
-  const samples: BufSample[] = [];
-  let peakMain = 0;
-  let peakFast = 0;
-  let stalledHeight = -1;
-  let fastPauses = 0;
-  try {
-    const startMs = Date.now();
-    const deadline = startMs + 15_000;
-    let lastHeight = -1;
-    let lastChange = Date.now();
-    while (Date.now() < deadline) {
-      const wall = Date.now();
-      const mainBuf = bufferSize(node, MAIN_1B);
-      const fastBuf = bufferSize(node, FAST_1B);
-      const appliedBlock = await latestFinalizedHeight(h.pool);
-      samples.push({
-        t: wall - startMs,
-        wall,
-        mainBuf,
-        evmBuf: fastBuf,
-        mainOwnBlock: ownBlock(node, MAIN_1B),
-        evmOwnBlock: ownBlock(node, FAST_1B),
-        rss: process.memoryUsage().rss,
-        appliedBlock,
-      });
-      peakMain = Math.max(peakMain, mainBuf);
-      peakFast = Math.max(peakFast, fastBuf);
-      if (appliedBlock !== lastHeight) {
-        lastHeight = appliedBlock;
-        lastChange = wall;
-      }
-      // Production has stalled (height flat ≥1.5s) AND the fast chain has hit its
-      // cap → the head-of-line block holds while the sibling buffer is bounded.
-      if (wall - lastChange >= 1500 && peakFast >= CAP_FLOOR) {
-        stalledHeight = appliedBlock ?? -1;
-        break;
-      }
-      await sleep(50);
-    }
-  } finally {
-    fastPauses = pauses(node, FAST_1B);
-    writeArtifact("buffering-1b", samples, {
-      mode: "1b-hol-sibling-bounded",
-      stepSize: STEP_SIZE,
-      bigTip: BIG_TIP,
-      slowTip: SLOW_TIP,
-      cap: CAP,
-      bufferBound: BUFFER_BOUND,
-      stalledHeight,
-      peakMainBuf: peakMain,
-      peakFastBuf: peakFast,
-      fastPauses,
-    });
-    await node.stop();
-  }
-
-  // Block production still stalls at ~the slow chain's tip (the merge gate is correct)...
-  expect(stalledHeight).toBeGreaterThanOrEqual(0);
-  expect(stalledHeight).toBeLessThanOrEqual(SLOW_TIP + 10);
-  // ...the fast chain's buffer engaged the cap...
-  expect(peakFast).toBeGreaterThanOrEqual(CAP_FLOOR);
-  // ...but stayed bounded behind the head-of-line block (was ~50k uncapped)...
-  expect(peakFast).toBeLessThanOrEqual(BUFFER_BOUND);
-  expect(peakMain).toBeLessThanOrEqual(BUFFER_BOUND);
-  // ...and the cap actually fired on the fast chain (observability metric populated).
-  expect(fastPauses).toBeGreaterThan(0);
-}, 60_000);

package/test/reproduction/coalesce.test.ts

@@ -1,133 +0,0 @@
-/** E2E: coalesced vs non-coalesced syncs produce the same DB; resume through coalescing matches a clean run. */
-import { expect, test } from "bun:test";
-import { setupHarness } from "./harness.ts";
-import {
-  compareConsistencySnapshots,
-  type ConsistencySnapshot,
-  dumpConsistencySnapshot,
-  type SnapshotDiff,
-  type SnapshotOpts,
-  SYNC_BOOKKEEPING_TABLES,
-} from "./consistency-snapshot.ts";
-
-function summarize(snap: ConsistencySnapshot): string {
-  let rows = 0;
-  for (const f of snap.values()) rows += f.rowCount;
-  return `${snap.size} tables, ${rows} rows`;
-}
-
-function formatDiffs(diffs: SnapshotDiff[]): string {
-  return diffs
-    .map((d) => `${d.table} [${d.reason} A=${d.aRows ?? "—"} B=${d.bRows ?? "—"}]`)
-    .join(", ");
-}
-
-function logComparison(
-  tag: string,
-  aLabel: string,
-  a: ConsistencySnapshot,
-  bLabel: string,
-  b: ConsistencySnapshot,
-  diffs: SnapshotDiff[],
-): void {
-  console.log(tag);
-  console.log(`${tag}   ${aLabel}: ${summarize(a)}`);
-  console.log(`${tag}   ${bLabel}: ${summarize(b)}`);
-  console.log(
-    diffs.length === 0
-      ? `${tag}   → IDENTICAL (0 differences)`
-      : `${tag}   → ${diffs.length} diff(s): ${formatDiffs(diffs)}`,
-  );
-}
-
-test(
-  "coalesced and non-coalesced from-scratch syncs produce identical databases",
-  async () => {
-    const events = [
-      { atBlock: 25, marker: "e25" },
-      { atBlock: 50, marker: "e50" },
-      { atBlock: 75, marker: "e75" },
-    ];
-
-    const runOnce = async (
-      coalesce: boolean,
-      apiPort: number,
-    ): Promise<ConsistencySnapshot> => {
-      const h = await setupHarness();
-      try {
-        await h.runToHeight({
-          events,
-          tips: { mainClock: 100, parallelP: 200 },
-          target: 100,
-          apiPort,
-          coalesce,
-        });
-        return await dumpConsistencySnapshot(h.pool);
-      } finally {
-        await h.teardown();
-      }
-    };
-
-    const off = await runOnce(false, 19160);
-    const on = await runOnce(true, 19161);
-
-    const { diffs } = compareConsistencySnapshots(off, on);
-    logComparison("[coalesce A] from-scratch sync to height 100:", "coalesce OFF", off, "coalesce ON", on, diffs);
-    expect(diffs).toEqual([]);
-  },
-  120_000,
-);
-
-test(
-  "a coalesced sync that resumes matches a clean coalesced sync",
-  async () => {
-    const events = [{ atBlock: 20, marker: "gap20" }];
-    const parallelStepSize = 1000;
-    const dumpOpts: SnapshotOpts = { excludeTables: SYNC_BOOKKEEPING_TABLES };
-
-    const ref = await setupHarness();
-    let refSnap: ConsistencySnapshot;
-    try {
-      await ref.runToHeight({
-        events,
-        parallelStepSize,
-        tips: { mainClock: 30, parallelP: 50 },
-        target: 30,
-        apiPort: 19170,
-        coalesce: true,
-      });
-      refSnap = await dumpConsistencySnapshot(ref.pool, dumpOpts);
-    } finally {
-      await ref.teardown();
-    }
-
-    const resumed = await setupHarness();
-    let resumeSnap: ConsistencySnapshot;
-    try {
-      await resumed.runToHeight({
-        events,
-        parallelStepSize,
-        tips: { mainClock: 10, parallelP: 50 },
-        target: 10,
-        apiPort: 19171,
-        coalesce: true,
-      });
-      await resumed.runToHeight({
-        events,
-        parallelStepSize,
-        tips: { mainClock: 30, parallelP: 50 },
-        target: 30,
-        apiPort: 19172,
-        coalesce: true,
-      });
-      resumeSnap = await dumpConsistencySnapshot(resumed.pool, dumpOpts);
-    } finally {
-      await resumed.teardown();
-    }
-
-    const { diffs } = compareConsistencySnapshots(refSnap, resumeSnap);
-    logComparison("[coalesce B] clean vs resumed coalesced sync to height 30:", "clean ref", refSnap, "resumed", resumeSnap, diffs);
-    expect(diffs).toEqual([]);
-  },
-  120_000,
-);

package/test/reproduction/consistency-snapshot.ts

@@ -1,168 +0,0 @@
-/**
- * In-memory DB *content fingerprints* for test comparison.
- *
- * NOT to be confused with `@effectstream/db`'s snapshot-handler.ts, which does
- * `pg_dump`-style snapshots for retention/recovery. These snapshots never touch
- * disk: they hash every row of every table so two databases can be compared
- * table-by-table, row-by-row (a file hash of the on-disk data dir is useless —
- * the physical layout churns with background-writer / vacuum activity and is
- * not a pure function of the committed rows).
- *
- * Used by consistency.test.ts to assert the engine builds the *same* database
- * regardless of how it got there (one clean run, or stop/restart).
- */
-import { createHash } from "node:crypto";
-import type pg from "pg";
-
-const md5 = (s: string) => createHash("md5").update(s).digest("hex");
-
-const DEFAULT_SCHEMAS = ["effectstream", "public", "primitives"];
-
-/**
- * `rollup_input_result.effectstream_tx_hash` is filled from `Math.random()`
- * (runtime/src/process-blocks.ts), so it differs between two otherwise-identical
- * runs. It is only written for scheduled STF inputs (never on the noop-STF test
- * path), so excluding it is a no-op for the current tests — but it keeps the
- * helper correct if reused against a config that runs real STFs.
- */
-export const VOLATILE_COLUMNS = ["effectstream_tx_hash"];
-
-/**
- * Sync-internal bookkeeping. The page queue legitimately differs after a
- * restart even in a correct engine, and the version/migration/config-snapshot
- * tables are engine plumbing rather than observable app state. Excluded when
- * asserting "the resumed DB reaches the same *state* as a clean sync".
- */
-export const SYNC_BOOKKEEPING_TABLES = [
-  "sync_protocol_pagination",
-  "effectstream_version_history",
-  "effectstream_expected_version",
-  "effectstream_migration_history",
-  "sync_protocol_config_snapshot",
-];
-
-export type TableFingerprint = {
-  rowCount: number;
-  /** md5 over the ascending-sorted per-row hashes (order-independent). */
-  fingerprint: string;
-  /** Ascending-sorted per-row md5s; kept for richer diffs if ever needed. */
-  rowHashes: string[];
-};
-
-/** Keyed by `"<schema>.<table>"`. */
-export type ConsistencySnapshot = Map<string, TableFingerprint>;
-
-export type SnapshotOpts = {
-  /** Schemas to enumerate. Default: effectstream, public, primitives. */
-  schemas?: string[];
-  /** Bare table names (any schema) to skip entirely. */
-  excludeTables?: string[];
-  /** Column names to drop before hashing rows (merged with VOLATILE_COLUMNS). */
-  excludeColumns?: string[];
-};
-
-const quoteIdent = (id: string) => `"${id.replace(/"/g, '""')}"`;
-
-/**
- * Hash the full content of every base table in the given schemas.
- *
- * Each row is hashed as `md5(concat_ws(',', quote_nullable(col::text), …))` over
- * its included columns. Each column is cast to text individually (bytea → `\x…`
- * hex, json/jsonb → canonical text), wrapped in `quote_nullable` so a SQL NULL
- * (rendered as the bare token `NULL`) is distinguishable from the literal string
- * `'NULL'`, and joined with a separator — deterministic given identical inserts.
- * Rows are sorted by their hash before folding into the table fingerprint, so
- * insertion order / serial id ordering does not matter.
- *
- * NB: we hash per-column rather than via a composite `ROW(<cols>)::text` cast so
- * NULL handling is explicit — `quote_nullable` renders a SQL NULL as the bare
- * token `NULL`, distinct from the literal string `'NULL'`, which the composite
- * cast conflates. Per-column casts also keep the rendering of each scalar type
- * (bytea → `\x…` hex, json/jsonb → canonical text) under our direct control.
- */
-export async function dumpConsistencySnapshot(
-  pool: pg.Pool,
-  opts: SnapshotOpts = {},
-): Promise<ConsistencySnapshot> {
-  const schemas = opts.schemas ?? DEFAULT_SCHEMAS;
-  const excludeTables = new Set(opts.excludeTables ?? []);
-  const excludeColumns = new Set([
-    ...VOLATILE_COLUMNS,
-    ...(opts.excludeColumns ?? []),
-  ]);
-
-  const tablesRes = await pool.query<{ table_schema: string; table_name: string }>(
-    `SELECT table_schema, table_name
-       FROM information_schema.tables
-      WHERE table_schema = ANY($1)
-        AND table_type = 'BASE TABLE'
-      ORDER BY table_schema, table_name`,
-    [schemas],
-  );
-
-  const snapshot: ConsistencySnapshot = new Map();
-  for (const { table_schema, table_name } of tablesRes.rows) {
-    if (excludeTables.has(table_name)) continue;
-
-    const colsRes = await pool.query<{ column_name: string }>(
-      `SELECT column_name
-         FROM information_schema.columns
-        WHERE table_schema = $1 AND table_name = $2
-        ORDER BY ordinal_position`,
-      [table_schema, table_name],
-    );
-    const cols = colsRes.rows
-      .map((r) => r.column_name)
-      .filter((c) => !excludeColumns.has(c));
-    if (cols.length === 0) continue;
-
-    const rowExpr = `concat_ws(',', ${
-      cols.map((c) => `quote_nullable(${quoteIdent(c)}::text)`).join(", ")
-    })`;
-    const rowsRes = await pool.query<{ h: string }>(
-      `SELECT md5(${rowExpr}) AS h
-         FROM ${quoteIdent(table_schema)}.${quoteIdent(table_name)}
-        ORDER BY h`,
-    );
-    const rowHashes = rowsRes.rows.map((r) => r.h);
-    snapshot.set(`${table_schema}.${table_name}`, {
-      rowCount: rowHashes.length,
-      fingerprint: md5(rowHashes.join("")),
-      rowHashes,
-    });
-  }
-  return snapshot;
-}
-
-export type SnapshotDiff = {
-  table: string;
-  reason: "only-in-a" | "only-in-b" | "row-count" | "content";
-  aRows: number | null;
-  bRows: number | null;
-};
-
-/** Compare two snapshots table-by-table. `equal` ⇔ `diffs` is empty. */
-export function compareConsistencySnapshots(
-  a: ConsistencySnapshot,
-  b: ConsistencySnapshot,
-): { equal: boolean; diffs: SnapshotDiff[] } {
-  const diffs: SnapshotDiff[] = [];
-  const tables = [...new Set([...a.keys(), ...b.keys()])].sort();
-  for (const table of tables) {
-    const fa = a.get(table);
-    const fb = b.get(table);
-    if (fa && !fb) {
-      diffs.push({ table, reason: "only-in-a", aRows: fa.rowCount, bRows: null });
-    } else if (!fa && fb) {
-      diffs.push({ table, reason: "only-in-b", aRows: null, bRows: fb.rowCount });
-    } else if (fa && fb && fa.fingerprint !== fb.fingerprint) {
-      diffs.push({
-        table,
-        reason: fa.rowCount !== fb.rowCount ? "row-count" : "content",
-        aRows: fa.rowCount,
-        bRows: fb.rowCount,
-      });
-    }
-  }
-  return { equal: diffs.length === 0, diffs };
-}