@effectstream/runtime 0.100.22 → 0.100.23

package/test/reproduction/coalesce-memory.test.ts

@@ -0,0 +1,181 @@
+/**
+ * Memory / queue-depth characterization for empty-block coalescing during a deep
+ * catch-up (sync/CLAUDE.md Finding #1 + Fix E).
+ *
+ * Coalescing is a consumer-side speedup: folding empty catch-up blocks removes the
+ * per-block STF + DB-commit bottleneck that otherwise acts as accidental
+ * backpressure. The worry is that, with that brake gone, the fetchers race to tip
+ * and either (a) balloon the uncapped `finalizedBlockStream` subscriber queue or
+ * (b) keep a parallel chain in the cap-lifted merge-demand exemption long enough to
+ * grow its Deque toward the whole backlog.
+ *
+ * This test drives a deep, all-empty catch-up with coalescing ON and asserts hard
+ * ceilings on the stream backlog (`inFlight`) and per-protocol buffers, plus a soft
+ * memory guard and a liveness check. Green → coalescing keeps everything bounded
+ * (regression guard). Red on the backlog/buffer bound → reproduces the leak, and the
+ * captured peak series localizes it.
+ *
+ * Postgres-only (chosen backend): higher-fidelity apply throughput than PGLite.
+ * Skips when no Postgres binary is on PATH.
+ */
+import { afterEach, beforeEach, expect, test } from "bun:test";
+import {
+  ConfigBuilder,
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+} from "@effectstream/config";
+import { TestChainControl } from "@effectstream/sync";
+import {
+  bufferSize,
+  type Harness,
+  latestFinalizedHeight,
+  postgresAvailable,
+  type RunningNode,
+  setupHarness,
+  sleep,
+  TEST_PRIMITIVE_TYPE,
+} from "./harness.ts";
+
+const RUN = postgresAvailable();
+
+const START_TIME = 1_700_000_000_000; // ~Nov 2023: every catch-up block is "old"
+const MAIN_BLOCK_MS = 1000;
+const PAR_BLOCK_MS = 100; // 10 parallel blocks per root slot
+const STEP_SIZE = 1000;
+const CAP = 3000;
+
+// A deep backlog: ~15k root slots the one-block-per-txn merge could never drain to
+// in time, so without coalescing the node would crawl — and any unbounded structure
+// would grow toward the whole backlog. Coalescing must fold the empties and keep
+// memory flat.
+const MAIN_TIP = 15_000;
+const PAR_TIP = MAIN_TIP * (MAIN_BLOCK_MS / PAR_BLOCK_MS); // cover the same wall range
+
+// Buffer settles in [~CAP, CAP + stepSize]; allow one overshoot chunk + slack.
+const BUFFER_BOUND = CAP + STEP_SIZE + 200;
+// The stream backlog must stay far below the whole-backlog (~MAIN_TIP) failure mode.
+// A healthy fast consumer keeps this in the low hundreds; we allow generous slack.
+const QUEUE_BOUND = 5_000;
+// Liveness: the node must catch up essentially to the tip (no deadlock / no crawl).
+const LIVE_TARGET = Math.floor(MAIN_TIP * 0.9);
+
+const MAIN = "memMainClock";
+const PAR = "memParallelP";
+
+function buildConfig() {
+  return new ConfigBuilder()
+    .setNamespace((b) => b.setSecurityNamespace("sync-coalesce-mem"))
+    .buildNetworks((b) =>
+      b
+        .addNetwork({
+          name: "clock",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: MAIN_BLOCK_MS,
+        })
+        .addNetwork({
+          name: "chainP",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: PAR_BLOCK_MS,
+        })
+    )
+    .buildDeployments((b) => b)
+    .buildSyncProtocols((b) =>
+      b
+        .addMain(
+          (n) => n.clock,
+          () => ({
+            name: MAIN,
+            type: ConfigSyncProtocolType.TEST_MAIN,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: STEP_SIZE,
+            maxBufferedPages: CAP,
+          }),
+        )
+        .addParallel(
+          (n) => (n as any).chainP,
+          () => ({
+            name: PAR,
+            type: ConfigSyncProtocolType.TEST_PARALLEL,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: STEP_SIZE,
+            delayMs: 0,
+            events: [], // no primitives → every block is coalescable
+            maxBufferedPages: CAP,
+          }),
+        )
+    )
+    .buildPrimitives((b) =>
+      b.addPrimitive(
+        (sp) => (sp as any)[PAR],
+        () => ({ name: "noEvtMem", type: TEST_PRIMITIVE_TYPE, startBlockHeight: 1 }),
+      )
+    )
+    .build();
+}
+
+let h: Harness;
+beforeEach(async () => {
+  if (!RUN) return;
+  h = await setupHarness({ backend: "postgres" });
+});
+afterEach(async () => {
+  if (!RUN) return;
+  await h?.teardown();
+});
+
+test("coalescing keeps stream backlog & buffers bounded during a deep empty catch-up", async () => {
+  if (!RUN) {
+    console.warn("coalesce-memory: skipped (no Postgres binary on PATH)");
+    return;
+  }
+
+  TestChainControl.setTip(MAIN, MAIN_TIP);
+  TestChainControl.setTip(PAR, PAR_TIP);
+
+  const node: RunningNode = await h.runNode({
+    config: buildConfig(),
+    apiPort: 19181,
+    coalesce: true,
+  });
+
+  let peakMain = 0;
+  let peakPar = 0;
+  let peakInFlight = 0;
+  let lastCoalesced = 0;
+  let lastApplied = 0;
+
+  try {
+    const startMs = Date.now();
+    const deadline = startMs + 110_000;
+    while (Date.now() < deadline) {
+      peakMain = Math.max(peakMain, bufferSize(node, MAIN));
+      peakPar = Math.max(peakPar, bufferSize(node, PAR));
+      peakInFlight = Math.max(peakInFlight, node.inFlight);
+      lastCoalesced = node.coalescedBlocks;
+      lastApplied = (await latestFinalizedHeight(h.pool)) ?? lastApplied;
+      if (lastApplied >= LIVE_TARGET) break; // caught up — bounded steady state captured
+      await sleep(50);
+    }
+  } finally {
+    console.log(
+      `coalesce-memory: peakInFlight=${peakInFlight} mainBuf=${peakMain} ` +
+        `parBuf=${peakPar} coalesced=${lastCoalesced} applied=${lastApplied}`,
+    );
+    await node.stop();
+  }
+
+  // 1. Coalescing actually fired (else this is a no-op test).
+  expect(lastCoalesced).toBeGreaterThan(1000);
+  // 2. Stream backlog stayed bounded — the merge did NOT outrun the consumer
+  //    toward the whole backlog (the uncapped subscriber queue, Finding #1). This
+  //    bounded queue is what keeps memory flat; no separate RSS check is needed.
+  expect(peakInFlight).toBeLessThanOrEqual(QUEUE_BOUND);
+  // 3. Per-protocol buffers stayed at the cap, even under coalescing's fast consumer.
+  expect(Math.max(peakMain, peakPar)).toBeLessThanOrEqual(BUFFER_BOUND);
+  // 4. Liveness: the node caught up to the tip (no deadlock / no crawl).
+  expect(lastApplied).toBeGreaterThanOrEqual(LIVE_TARGET);
+}, 120_000);

package/test/reproduction/coalesce.test.ts

@@ -0,0 +1,133 @@
+/** E2E: coalesced vs non-coalesced syncs produce the same DB; resume through coalescing matches a clean run. */
+import { expect, test } from "bun:test";
+import { setupHarness } from "./harness.ts";
+import {
+  compareConsistencySnapshots,
+  type ConsistencySnapshot,
+  dumpConsistencySnapshot,
+  type SnapshotDiff,
+  type SnapshotOpts,
+  SYNC_BOOKKEEPING_TABLES,
+} from "./consistency-snapshot.ts";
+
+function summarize(snap: ConsistencySnapshot): string {
+  let rows = 0;
+  for (const f of snap.values()) rows += f.rowCount;
+  return `${snap.size} tables, ${rows} rows`;
+}
+
+function formatDiffs(diffs: SnapshotDiff[]): string {
+  return diffs
+    .map((d) => `${d.table} [${d.reason} A=${d.aRows ?? "—"} B=${d.bRows ?? "—"}]`)
+    .join(", ");
+}
+
+function logComparison(
+  tag: string,
+  aLabel: string,
+  a: ConsistencySnapshot,
+  bLabel: string,
+  b: ConsistencySnapshot,
+  diffs: SnapshotDiff[],
+): void {
+  console.log(tag);
+  console.log(`${tag}   ${aLabel}: ${summarize(a)}`);
+  console.log(`${tag}   ${bLabel}: ${summarize(b)}`);
+  console.log(
+    diffs.length === 0
+      ? `${tag}   → IDENTICAL (0 differences)`
+      : `${tag}   → ${diffs.length} diff(s): ${formatDiffs(diffs)}`,
+  );
+}
+
+test(
+  "coalesced and non-coalesced from-scratch syncs produce identical databases",
+  async () => {
+    const events = [
+      { atBlock: 25, marker: "e25" },
+      { atBlock: 50, marker: "e50" },
+      { atBlock: 75, marker: "e75" },
+    ];
+
+    const runOnce = async (
+      coalesce: boolean,
+      apiPort: number,
+    ): Promise<ConsistencySnapshot> => {
+      const h = await setupHarness();
+      try {
+        await h.runToHeight({
+          events,
+          tips: { mainClock: 100, parallelP: 200 },
+          target: 100,
+          apiPort,
+          coalesce,
+        });
+        return await dumpConsistencySnapshot(h.pool);
+      } finally {
+        await h.teardown();
+      }
+    };
+
+    const off = await runOnce(false, 19160);
+    const on = await runOnce(true, 19161);
+
+    const { diffs } = compareConsistencySnapshots(off, on);
+    logComparison("[coalesce A] from-scratch sync to height 100:", "coalesce OFF", off, "coalesce ON", on, diffs);
+    expect(diffs).toEqual([]);
+  },
+  120_000,
+);
+
+test(
+  "a coalesced sync that resumes matches a clean coalesced sync",
+  async () => {
+    const events = [{ atBlock: 20, marker: "gap20" }];
+    const parallelStepSize = 1000;
+    const dumpOpts: SnapshotOpts = { excludeTables: SYNC_BOOKKEEPING_TABLES };
+
+    const ref = await setupHarness();
+    let refSnap: ConsistencySnapshot;
+    try {
+      await ref.runToHeight({
+        events,
+        parallelStepSize,
+        tips: { mainClock: 30, parallelP: 50 },
+        target: 30,
+        apiPort: 19170,
+        coalesce: true,
+      });
+      refSnap = await dumpConsistencySnapshot(ref.pool, dumpOpts);
+    } finally {
+      await ref.teardown();
+    }
+
+    const resumed = await setupHarness();
+    let resumeSnap: ConsistencySnapshot;
+    try {
+      await resumed.runToHeight({
+        events,
+        parallelStepSize,
+        tips: { mainClock: 10, parallelP: 50 },
+        target: 10,
+        apiPort: 19171,
+        coalesce: true,
+      });
+      await resumed.runToHeight({
+        events,
+        parallelStepSize,
+        tips: { mainClock: 30, parallelP: 50 },
+        target: 30,
+        apiPort: 19172,
+        coalesce: true,
+      });
+      resumeSnap = await dumpConsistencySnapshot(resumed.pool, dumpOpts);
+    } finally {
+      await resumed.teardown();
+    }
+
+    const { diffs } = compareConsistencySnapshots(refSnap, resumeSnap);
+    logComparison("[coalesce B] clean vs resumed coalesced sync to height 30:", "clean ref", refSnap, "resumed", resumeSnap, diffs);
+    expect(diffs).toEqual([]);
+  },
+  120_000,
+);

package/test/reproduction/consistency-snapshot.ts

@@ -0,0 +1,168 @@
+/**
+ * In-memory DB *content fingerprints* for test comparison.
+ *
+ * NOT to be confused with `@effectstream/db`'s snapshot-handler.ts, which does
+ * `pg_dump`-style snapshots for retention/recovery. These snapshots never touch
+ * disk: they hash every row of every table so two databases can be compared
+ * table-by-table, row-by-row (a file hash of the on-disk data dir is useless —
+ * the physical layout churns with background-writer / vacuum activity and is
+ * not a pure function of the committed rows).
+ *
+ * Used by consistency.test.ts to assert the engine builds the *same* database
+ * regardless of how it got there (one clean run, or stop/restart).
+ */
+import { createHash } from "node:crypto";
+import type pg from "pg";
+
+const md5 = (s: string) => createHash("md5").update(s).digest("hex");
+
+const DEFAULT_SCHEMAS = ["effectstream", "public", "primitives"];
+
+/**
+ * `rollup_input_result.effectstream_tx_hash` is filled from `Math.random()`
+ * (runtime/src/process-blocks.ts), so it differs between two otherwise-identical
+ * runs. It is only written for scheduled STF inputs (never on the noop-STF test
+ * path), so excluding it is a no-op for the current tests — but it keeps the
+ * helper correct if reused against a config that runs real STFs.
+ */
+export const VOLATILE_COLUMNS = ["effectstream_tx_hash"];
+
+/**
+ * Sync-internal bookkeeping. The page queue legitimately differs after a
+ * restart even in a correct engine, and the version/migration/config-snapshot
+ * tables are engine plumbing rather than observable app state. Excluded when
+ * asserting "the resumed DB reaches the same *state* as a clean sync".
+ */
+export const SYNC_BOOKKEEPING_TABLES = [
+  "sync_protocol_pagination",
+  "effectstream_version_history",
+  "effectstream_expected_version",
+  "effectstream_migration_history",
+  "sync_protocol_config_snapshot",
+];
+
+export type TableFingerprint = {
+  rowCount: number;
+  /** md5 over the ascending-sorted per-row hashes (order-independent). */
+  fingerprint: string;
+  /** Ascending-sorted per-row md5s; kept for richer diffs if ever needed. */
+  rowHashes: string[];
+};
+
+/** Keyed by `"<schema>.<table>"`. */
+export type ConsistencySnapshot = Map<string, TableFingerprint>;
+
+export type SnapshotOpts = {
+  /** Schemas to enumerate. Default: effectstream, public, primitives. */
+  schemas?: string[];
+  /** Bare table names (any schema) to skip entirely. */
+  excludeTables?: string[];
+  /** Column names to drop before hashing rows (merged with VOLATILE_COLUMNS). */
+  excludeColumns?: string[];
+};
+
+const quoteIdent = (id: string) => `"${id.replace(/"/g, '""')}"`;
+
+/**
+ * Hash the full content of every base table in the given schemas.
+ *
+ * Each row is hashed as `md5(concat_ws(',', quote_nullable(col::text), …))` over
+ * its included columns. Each column is cast to text individually (bytea → `\x…`
+ * hex, json/jsonb → canonical text), wrapped in `quote_nullable` so a SQL NULL
+ * (rendered as the bare token `NULL`) is distinguishable from the literal string
+ * `'NULL'`, and joined with a separator — deterministic given identical inserts.
+ * Rows are sorted by their hash before folding into the table fingerprint, so
+ * insertion order / serial id ordering does not matter.
+ *
+ * NB: we hash per-column rather than via a composite `ROW(<cols>)::text` cast so
+ * NULL handling is explicit — `quote_nullable` renders a SQL NULL as the bare
+ * token `NULL`, distinct from the literal string `'NULL'`, which the composite
+ * cast conflates. Per-column casts also keep the rendering of each scalar type
+ * (bytea → `\x…` hex, json/jsonb → canonical text) under our direct control.
+ */
+export async function dumpConsistencySnapshot(
+  pool: pg.Pool,
+  opts: SnapshotOpts = {},
+): Promise<ConsistencySnapshot> {
+  const schemas = opts.schemas ?? DEFAULT_SCHEMAS;
+  const excludeTables = new Set(opts.excludeTables ?? []);
+  const excludeColumns = new Set([
+    ...VOLATILE_COLUMNS,
+    ...(opts.excludeColumns ?? []),
+  ]);
+
+  const tablesRes = await pool.query<{ table_schema: string; table_name: string }>(
+    `SELECT table_schema, table_name
+       FROM information_schema.tables
+      WHERE table_schema = ANY($1)
+        AND table_type = 'BASE TABLE'
+      ORDER BY table_schema, table_name`,
+    [schemas],
+  );
+
+  const snapshot: ConsistencySnapshot = new Map();
+  for (const { table_schema, table_name } of tablesRes.rows) {
+    if (excludeTables.has(table_name)) continue;
+
+    const colsRes = await pool.query<{ column_name: string }>(
+      `SELECT column_name
+         FROM information_schema.columns
+        WHERE table_schema = $1 AND table_name = $2
+        ORDER BY ordinal_position`,
+      [table_schema, table_name],
+    );
+    const cols = colsRes.rows
+      .map((r) => r.column_name)
+      .filter((c) => !excludeColumns.has(c));
+    if (cols.length === 0) continue;
+
+    const rowExpr = `concat_ws(',', ${
+      cols.map((c) => `quote_nullable(${quoteIdent(c)}::text)`).join(", ")
+    })`;
+    const rowsRes = await pool.query<{ h: string }>(
+      `SELECT md5(${rowExpr}) AS h
+         FROM ${quoteIdent(table_schema)}.${quoteIdent(table_name)}
+        ORDER BY h`,
+    );
+    const rowHashes = rowsRes.rows.map((r) => r.h);
+    snapshot.set(`${table_schema}.${table_name}`, {
+      rowCount: rowHashes.length,
+      fingerprint: md5(rowHashes.join("")),
+      rowHashes,
+    });
+  }
+  return snapshot;
+}
+
+export type SnapshotDiff = {
+  table: string;
+  reason: "only-in-a" | "only-in-b" | "row-count" | "content";
+  aRows: number | null;
+  bRows: number | null;
+};
+
+/** Compare two snapshots table-by-table. `equal` ⇔ `diffs` is empty. */
+export function compareConsistencySnapshots(
+  a: ConsistencySnapshot,
+  b: ConsistencySnapshot,
+): { equal: boolean; diffs: SnapshotDiff[] } {
+  const diffs: SnapshotDiff[] = [];
+  const tables = [...new Set([...a.keys(), ...b.keys()])].sort();
+  for (const table of tables) {
+    const fa = a.get(table);
+    const fb = b.get(table);
+    if (fa && !fb) {
+      diffs.push({ table, reason: "only-in-a", aRows: fa.rowCount, bRows: null });
+    } else if (!fa && fb) {
+      diffs.push({ table, reason: "only-in-b", aRows: null, bRows: fb.rowCount });
+    } else if (fa && fb && fa.fingerprint !== fb.fingerprint) {
+      diffs.push({
+        table,
+        reason: fa.rowCount !== fb.rowCount ? "row-count" : "content",
+        aRows: fa.rowCount,
+        bRows: fb.rowCount,
+      });
+    }
+  }
+  return { equal: diffs.length === 0, diffs };
+}

package/test/reproduction/consistency.test.ts

@@ -0,0 +1,197 @@
+/**
+ * E2E database-consistency checks over the synthetic `test` chain.
+ *
+ * The database the engine builds must be a pure function of the source blocks —
+ * independent of *how* it got there. Two scenarios:
+ *
+ *   A. Determinism baseline — two from-scratch syncs to the same height produce
+ *      identical databases. Expected to PASS (proves reproducibility).
+ *
+ *   B. Resume guarantee — a sync that stops mid-chunk and resumes to N must
+ *      match a clean sync to N. Regression test for block-accurate resume
+ *      (see @effectstream/sync CLAUDE.md, design idea #5): a restart must
+ *      re-fetch (committed, chunk_end] instead of silently skipping it.
+ *
+ * Each node runs in its own subprocess (see harness.ts / node-runner.ts): a
+ * restart is a genuine new OS process whose in-memory Deque is gone while the
+ * database keeps every committed row — exactly the production restart condition.
+ *
+ * These tests assert a property of the **production** database path, so they run
+ * only against real Postgres — i.e. only when `PGLITE=false` and `postgresql@18`
+ * is installed. In the default PGLite mode (or if Postgres isn't installed) they
+ * skip rather than run, since PGLite is not the backend whose fidelity these
+ * guarantees are about.
+ */
+import { expect, test } from "bun:test";
+import { postgresAvailable, setupHarness } from "./harness.ts";
+import {
+  compareConsistencySnapshots,
+  type ConsistencySnapshot,
+  dumpConsistencySnapshot,
+  type SnapshotDiff,
+  type SnapshotOpts,
+  SYNC_BOOKKEEPING_TABLES,
+} from "./consistency-snapshot.ts";
+
+// Postgres-only: run only when PGLITE=false and Postgres is present.
+const canRun = process.env.PGLITE === "false" && postgresAvailable();
+const it = canRun ? test : test.skip;
+
+/** "<n> tables, <m> rows". */
+function summarize(snap: ConsistencySnapshot): string {
+  let rows = 0;
+  for (const f of snap.values()) rows += f.rowCount;
+  return `${snap.size} tables, ${rows} rows`;
+}
+
+/** e.g. "effectstream.primitive_accounting [row-count A=1 B=0]". */
+function formatDiffs(diffs: SnapshotDiff[]): string {
+  return diffs
+    .map((d) => `${d.table} [${d.reason} A=${d.aRows ?? "—"} B=${d.bRows ?? "—"}]`)
+    .join(", ");
+}
+
+/** Log the two snapshots and their comparison result. */
+function logComparison(
+  tag: string,
+  label: string,
+  aLabel: string,
+  a: ConsistencySnapshot,
+  bLabel: string,
+  b: ConsistencySnapshot,
+  diffs: SnapshotDiff[],
+): void {
+  console.log(`${tag} ${label}`);
+  console.log(`${tag}   ${aLabel}: ${summarize(a)}`);
+  console.log(`${tag}   ${bLabel}: ${summarize(b)}`);
+  console.log(
+    diffs.length === 0
+      ? `${tag}   → IDENTICAL (0 differences)`
+      : `${tag}   → ${diffs.length} difference(s): ${formatDiffs(diffs)}`,
+  );
+}
+
+it("two from-scratch syncs to the same height produce identical databases", async () => {
+  const events = [
+    { atBlock: 25, marker: "e25" },
+    { atBlock: 50, marker: "e50" },
+    { atBlock: 75, marker: "e75" },
+  ];
+
+  // A fresh harness == a fresh database == "syncing again from scratch".
+  const runFromScratch = async (
+    apiPort: number,
+  ): Promise<ConsistencySnapshot> => {
+    const h = await setupHarness({ backend: "postgres" });
+    try {
+      await h.runToHeight({
+        events,
+        tips: { mainClock: 100, parallelP: 200 },
+        target: 100,
+        apiPort,
+      });
+      return await dumpConsistencySnapshot(h.pool);
+    } finally {
+      await h.teardown();
+    }
+  };
+
+  const snapA = await runFromScratch(19140);
+  const snapB = await runFromScratch(19141);
+
+  const { diffs } = compareConsistencySnapshots(snapA, snapB);
+  logComparison(
+    "[consistency A]",
+    "two from-scratch syncs to height 100:",
+    "run A",
+    snapA,
+    "run B",
+    snapB,
+    diffs,
+  );
+  expect(diffs).toEqual([]);
+}, 120_000);
+
+it(
+  "a mid-chunk restart resumes to the same database as a clean sync",
+  async () => {
+    // One event inside the gap the restart will skip: it must fall in
+    // (phase-1 committed height 10, parallel page 50] AND be ≤ the phase-2 main
+    // tip 30, so a *correct* engine would re-fetch and commit it after resume.
+    const events = [{ atBlock: 20, marker: "gap20" }];
+    // Large parallel stepSize => the whole tip is fetched as ONE chunk, so its
+    // page row persists at page_number = tip (the chunk-granular watermark that
+    // Fix D wrongly resumes from).
+    const parallelStepSize = 1000;
+    // Sync bookkeeping (page queue, version/migration history) legitimately
+    // differs after a restart even in a correct engine; compare observable
+    // state only.
+    const dumpOpts: SnapshotOpts = { excludeTables: SYNC_BOOKKEEPING_TABLES };
+
+    // Reference: one clean sync straight to height 30. Parallel stays ahead of
+    // the main clock so the final main block can finalize (the merge gates block
+    // T on the parallel page advancing past T). Contains the block-20 event.
+    const ref = await setupHarness({ backend: "postgres" });
+    let refSnap: ConsistencySnapshot;
+    try {
+      await ref.runToHeight({
+        events,
+        parallelStepSize,
+        tips: { mainClock: 30, parallelP: 50 },
+        target: 30,
+        apiPort: 19150,
+      });
+      refSnap = await dumpConsistencySnapshot(ref.pool, dumpOpts);
+    } finally {
+      await ref.teardown();
+    }
+
+    // Resumed: one database, two subprocess runs (= a real restart).
+    const resumed = await setupHarness({ backend: "postgres" });
+    let resumeSnap: ConsistencySnapshot;
+    try {
+      // Phase 1: the parallel fetcher (no backpressure) races to its own tip
+      // (50) into the in-memory Deque, but the main clock only drives commits to
+      // height 10 — so parallel blocks 11..50 (incl. the event at 20) are fetched
+      // but never committed, and the in-memory Deque is lost when the process
+      // exits. The runtime persists a block-accurate resume marker at the last
+      // COMMITTED parallel block (≤ 10), not the fetched-ahead tip.
+      await resumed.runToHeight({
+        events,
+        parallelStepSize,
+        tips: { mainClock: 10, parallelP: 50 },
+        target: 10,
+        apiPort: 19151,
+      });
+
+      // Phase 2 (restart): a new process boots against the same database. Its
+      // Deque is empty; restoreState reads the persisted resume marker (the last
+      // committed parallel block ≤ 10) and resumes parallel from there. The main
+      // clock now advances to 30, so the parallel blocks it needs (11..30,
+      // including the event at 20) are re-fetched and committed.
+      await resumed.runToHeight({
+        events,
+        parallelStepSize,
+        tips: { mainClock: 30, parallelP: 50 },
+        target: 30,
+        apiPort: 19152,
+      });
+      resumeSnap = await dumpConsistencySnapshot(resumed.pool, dumpOpts);
+    } finally {
+      await resumed.teardown();
+    }
+
+    const { diffs } = compareConsistencySnapshots(refSnap, resumeSnap);
+    logComparison(
+      "[consistency B]",
+      "clean sync vs mid-chunk restart, both to height 30:",
+      "clean ref",
+      refSnap,
+      "resumed  ",
+      resumeSnap,
+      diffs,
+    );
+    expect(diffs).toEqual([]);
+  },
+  120_000,
+);