@effectstream/runtime 0.100.20 → 0.100.22

package/test/reproduction/consistency.test.ts

@@ -1,197 +0,0 @@
-/**
- * E2E database-consistency checks over the synthetic `test` chain.
- *
- * The database the engine builds must be a pure function of the source blocks —
- * independent of *how* it got there. Two scenarios:
- *
- *   A. Determinism baseline — two from-scratch syncs to the same height produce
- *      identical databases. Expected to PASS (proves reproducibility).
- *
- *   B. Resume guarantee — a sync that stops mid-chunk and resumes to N must
- *      match a clean sync to N. Regression test for block-accurate resume
- *      (see @effectstream/sync CLAUDE.md, design idea #5): a restart must
- *      re-fetch (committed, chunk_end] instead of silently skipping it.
- *
- * Each node runs in its own subprocess (see harness.ts / node-runner.ts): a
- * restart is a genuine new OS process whose in-memory Deque is gone while the
- * database keeps every committed row — exactly the production restart condition.
- *
- * These tests assert a property of the **production** database path, so they run
- * only against real Postgres — i.e. only when `PGLITE=false` and `postgresql@18`
- * is installed. In the default PGLite mode (or if Postgres isn't installed) they
- * skip rather than run, since PGLite is not the backend whose fidelity these
- * guarantees are about.
- */
-import { expect, test } from "bun:test";
-import { postgresAvailable, setupHarness } from "./harness.ts";
-import {
-  compareConsistencySnapshots,
-  type ConsistencySnapshot,
-  dumpConsistencySnapshot,
-  type SnapshotDiff,
-  type SnapshotOpts,
-  SYNC_BOOKKEEPING_TABLES,
-} from "./consistency-snapshot.ts";
-
-// Postgres-only: run only when PGLITE=false and Postgres is present.
-const canRun = process.env.PGLITE === "false" && postgresAvailable();
-const it = canRun ? test : test.skip;
-
-/** "<n> tables, <m> rows". */
-function summarize(snap: ConsistencySnapshot): string {
-  let rows = 0;
-  for (const f of snap.values()) rows += f.rowCount;
-  return `${snap.size} tables, ${rows} rows`;
-}
-
-/** e.g. "effectstream.primitive_accounting [row-count A=1 B=0]". */
-function formatDiffs(diffs: SnapshotDiff[]): string {
-  return diffs
-    .map((d) => `${d.table} [${d.reason} A=${d.aRows ?? "—"} B=${d.bRows ?? "—"}]`)
-    .join(", ");
-}
-
-/** Log the two snapshots and their comparison result. */
-function logComparison(
-  tag: string,
-  label: string,
-  aLabel: string,
-  a: ConsistencySnapshot,
-  bLabel: string,
-  b: ConsistencySnapshot,
-  diffs: SnapshotDiff[],
-): void {
-  console.log(`${tag} ${label}`);
-  console.log(`${tag}   ${aLabel}: ${summarize(a)}`);
-  console.log(`${tag}   ${bLabel}: ${summarize(b)}`);
-  console.log(
-    diffs.length === 0
-      ? `${tag}   → IDENTICAL (0 differences)`
-      : `${tag}   → ${diffs.length} difference(s): ${formatDiffs(diffs)}`,
-  );
-}
-
-it("two from-scratch syncs to the same height produce identical databases", async () => {
-  const events = [
-    { atBlock: 25, marker: "e25" },
-    { atBlock: 50, marker: "e50" },
-    { atBlock: 75, marker: "e75" },
-  ];
-
-  // A fresh harness == a fresh database == "syncing again from scratch".
-  const runFromScratch = async (
-    apiPort: number,
-  ): Promise<ConsistencySnapshot> => {
-    const h = await setupHarness({ backend: "postgres" });
-    try {
-      await h.runToHeight({
-        events,
-        tips: { mainClock: 100, parallelP: 200 },
-        target: 100,
-        apiPort,
-      });
-      return await dumpConsistencySnapshot(h.pool);
-    } finally {
-      await h.teardown();
-    }
-  };
-
-  const snapA = await runFromScratch(19140);
-  const snapB = await runFromScratch(19141);
-
-  const { diffs } = compareConsistencySnapshots(snapA, snapB);
-  logComparison(
-    "[consistency A]",
-    "two from-scratch syncs to height 100:",
-    "run A",
-    snapA,
-    "run B",
-    snapB,
-    diffs,
-  );
-  expect(diffs).toEqual([]);
-}, 120_000);
-
-it(
-  "a mid-chunk restart resumes to the same database as a clean sync",
-  async () => {
-    // One event inside the gap the restart will skip: it must fall in
-    // (phase-1 committed height 10, parallel page 50] AND be ≤ the phase-2 main
-    // tip 30, so a *correct* engine would re-fetch and commit it after resume.
-    const events = [{ atBlock: 20, marker: "gap20" }];
-    // Large parallel stepSize => the whole tip is fetched as ONE chunk, so its
-    // page row persists at page_number = tip (the chunk-granular watermark that
-    // Fix D wrongly resumes from).
-    const parallelStepSize = 1000;
-    // Sync bookkeeping (page queue, version/migration history) legitimately
-    // differs after a restart even in a correct engine; compare observable
-    // state only.
-    const dumpOpts: SnapshotOpts = { excludeTables: SYNC_BOOKKEEPING_TABLES };
-
-    // Reference: one clean sync straight to height 30. Parallel stays ahead of
-    // the main clock so the final main block can finalize (the merge gates block
-    // T on the parallel page advancing past T). Contains the block-20 event.
-    const ref = await setupHarness({ backend: "postgres" });
-    let refSnap: ConsistencySnapshot;
-    try {
-      await ref.runToHeight({
-        events,
-        parallelStepSize,
-        tips: { mainClock: 30, parallelP: 50 },
-        target: 30,
-        apiPort: 19150,
-      });
-      refSnap = await dumpConsistencySnapshot(ref.pool, dumpOpts);
-    } finally {
-      await ref.teardown();
-    }
-
-    // Resumed: one database, two subprocess runs (= a real restart).
-    const resumed = await setupHarness({ backend: "postgres" });
-    let resumeSnap: ConsistencySnapshot;
-    try {
-      // Phase 1: the parallel fetcher (no backpressure) races to its own tip
-      // (50) into the in-memory Deque, but the main clock only drives commits to
-      // height 10 — so parallel blocks 11..50 (incl. the event at 20) are fetched
-      // but never committed, and the in-memory Deque is lost when the process
-      // exits. The runtime persists a block-accurate resume marker at the last
-      // COMMITTED parallel block (≤ 10), not the fetched-ahead tip.
-      await resumed.runToHeight({
-        events,
-        parallelStepSize,
-        tips: { mainClock: 10, parallelP: 50 },
-        target: 10,
-        apiPort: 19151,
-      });
-
-      // Phase 2 (restart): a new process boots against the same database. Its
-      // Deque is empty; restoreState reads the persisted resume marker (the last
-      // committed parallel block ≤ 10) and resumes parallel from there. The main
-      // clock now advances to 30, so the parallel blocks it needs (11..30,
-      // including the event at 20) are re-fetched and committed.
-      await resumed.runToHeight({
-        events,
-        parallelStepSize,
-        tips: { mainClock: 30, parallelP: 50 },
-        target: 30,
-        apiPort: 19152,
-      });
-      resumeSnap = await dumpConsistencySnapshot(resumed.pool, dumpOpts);
-    } finally {
-      await resumed.teardown();
-    }
-
-    const { diffs } = compareConsistencySnapshots(refSnap, resumeSnap);
-    logComparison(
-      "[consistency B]",
-      "clean sync vs mid-chunk restart, both to height 30:",
-      "clean ref",
-      refSnap,
-      "resumed  ",
-      resumeSnap,
-      diffs,
-    );
-    expect(diffs).toEqual([]);
-  },
-  120_000,
-);

package/test/reproduction/node-runner.ts

@@ -1,154 +0,0 @@
-/**
- * One-node subprocess entry point.
- *
- * A real engine restart is a new OS process whose in-memory state (notably the
- * sync Deque) is gone while PostgreSQL keeps every committed row. We model that
- * literally: each node runs here, in its own `bun` process, against a database
- * the parent harness created. Booting the runtime twice *in one process* is not
- * how production works and proved unreliable (a second in-process `start()`
- * stalls the sync loops under `bun test`), so the harness spawns this instead.
- *
- * Invocation (by the harness):
- *   bun node-runner.ts '<json-RunSpec>'
- * with DB_* / PGLITE env already set (by the harness) to point at the target
- * database — a real Postgres server or a PGLite gateway.
- *
- * Exit 0 once the node has synced to the requested height (and, for the resume
- * scenario, the requested page watermarks); exit 1 on boot failure.
- */
-import pg from "pg";
-import { run, type Task } from "effection";
-import {
-  toSyncProtocolWithNetwork,
-  withEffectstreamStaticConfig,
-} from "@effectstream/config";
-import { TestChainControl } from "@effectstream/sync";
-import { init, start } from "../../src/main.ts";
-import type { StartConfigGameStateTransitions } from "../../src/types.ts";
-import {
-  buildConfig,
-  type EventSpec,
-  TEST_PRIMITIVE_TYPE,
-  TestEventPrimitive,
-} from "./scenario.ts";
-import { waitForHeight, waitForPage, waitUntilStable } from "./poll.ts";
-
-export type RunSpec = {
-  securityNamespace: string;
-  events: EventSpec[];
-  parallelStepSize?: number;
-  /** Manual chain tips, e.g. { mainClock: 100, parallelP: 200 }. */
-  tips: Record<string, number>;
-  /** Finalized height to reach before exiting. */
-  target: number;
-  apiPort: number;
-  /**
-   * Optional per-protocol page watermarks to also wait for before exiting.
-   * Used by the resume scenario to guarantee the parallel fetcher has raced
-   * ahead and persisted its page row before we kill the process.
-   */
-  waitPages?: Record<string, number>;
-  coalesce?: boolean;
-};
-
-const noopStf: StartConfigGameStateTransitions = function* () {};
-
-async function main() {
-  const spec: RunSpec = JSON.parse(process.argv[2] ?? "{}");
-
-  // DB_* and PGLITE are set by the harness (real Postgres or a PGLite gateway);
-  // do NOT override them here. Only force the bits every run shares: no MQTT
-  // broker (it binds fixed ports and isn't freed on halt) and a per-run API port.
-  process.env.MQTT_BROKER = "false";
-  process.env.EFFECTSTREAM_API_PORT = String(spec.apiPort);
-  process.env.EFFECTSTREAM_COALESCE_EMPTY_BLOCKS = spec.coalesce ? "true" : "false";
-  const isPglite = process.env.PGLITE === "true";
-
-  for (const [name, tip] of Object.entries(spec.tips)) {
-    TestChainControl.setTip(name, tip);
-  }
-
-  const pool = new pg.Pool({
-    host: process.env.DB_HOST,
-    port: Number(process.env.DB_PORT),
-    user: process.env.DB_USER,
-    database: process.env.DB_NAME,
-    password: process.env.DB_PW || undefined,
-    // PGLite is a single WASM backend: cap polls at one connection (they're also
-    // mutex-serialized against the engine — see poll.ts).
-    max: isPglite ? 1 : 10,
-  });
-
-  const cfg = (spec as any).config ?? buildConfig({
-    securityNamespace: spec.securityNamespace,
-    events: spec.events,
-    parallelStepSize: spec.parallelStepSize,
-  });
-
-  let bootErr: unknown;
-  let started = false;
-  let live: any[] = [];
-  const task: Task<unknown> = run(function* () {
-    yield* init();
-    yield* withEffectstreamStaticConfig(cfg, function* () {
-      yield* start({
-        appName: "sync-repro",
-        appVersion: "1.0.0",
-        syncInfo: toSyncProtocolWithNetwork(cfg),
-        gameStateTransitions: noopStf,
-        userDefinedPrimitives: { [TEST_PRIMITIVE_TYPE]: TestEventPrimitive },
-        dev: {
-          resetPublicData: false,
-          onStarted: ({ syncProtocols }: any) => {
-            live = syncProtocols;
-            started = true;
-          },
-        },
-      });
-    });
-  });
-  // start() runs forever; capture a genuine boot failure (not the expected
-  // "halted" rejection from our own task.halt() below).
-  Promise.resolve(task).catch((e) => {
-    if (!String(e).includes("halted")) bootErr = e;
-  });
-
-  const guard = async <T>(p: Promise<T>): Promise<T> => {
-    const r = await p;
-    if (bootErr) throw new Error(`node boot failed: ${String(bootErr)}`);
-    return r;
-  };
-
-  // Don't poll until `onStarted` fires: init()'s migrations and startup() run
-  // before it and aren't mutex-guarded, so on PGLite a poll could overlap them
-  // and corrupt the single WASM backend. Steady-state queries after this are.
-  while (!started) {
-    if (bootErr) throw new Error(`node boot failed: ${String(bootErr)}`);
-    await new Promise((r) => setTimeout(r, 20));
-  }
-
-  try {
-    await guard(waitForHeight(pool, spec.target));
-    await guard(waitUntilStable(pool));
-    for (const [name, page] of Object.entries(spec.waitPages ?? {})) {
-      await guard(waitForPage(pool, name, page));
-    }
-  } catch (e) {
-    const probe = live.map((p) =>
-      `${p.name}: lastPage=${JSON.stringify(p.lastPage)} buf=${p.bufferedData.size()} errs=${p.consecutiveErrors}`
-    );
-    console.error("PROBE:\n  " + probe.join("\n  "));
-    throw e;
-  } finally {
-    await task.halt().catch(() => {});
-    await pool.end().catch(() => {});
-  }
-}
-
-main().then(
-  () => process.exit(0),
-  (e) => {
-    console.error(String(e instanceof Error ? e.stack ?? e.message : e));
-    process.exit(1);
-  },
-);

package/test/reproduction/poll.ts

@@ -1,160 +0,0 @@
-/**
- * pg.Pool query helpers shared by the test harness (parent process, used for
- * assertions) and {@link ./node-runner.ts} (child process, used to decide when a
- * node has finished syncing). No cluster / runtime imports.
- *
- * On PGLite every query is serialized through the engine's process-global DB
- * mutex — see {@link q}. On a real Postgres server that mutex is a no-op and
- * these are plain pooled queries.
- */
-import type pg from "pg";
-import { run } from "effection";
-import { ENV } from "@effectstream/utils/node-env";
-import { acquireDBMutex, releaseDBMutex } from "@effectstream/db";
-
-export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
-
-/**
- * Run a query, serialized through the engine's process-global DB mutex on PGLite
- * (a single WASM backend that two overlapping queries corrupt). node-runner
- * polls while the engine syncs in the SAME process, so taking the same mutex the
- * engine uses guarantees no overlap — they share the module-global lock even
- * across separate `run()` roots. On real Postgres the mutex is a no-op.
- */
-async function q<T extends pg.QueryResultRow = pg.QueryResultRow>(
-  pool: pg.Pool,
-  sql: string,
-  params?: unknown[],
-): Promise<T[]> {
-  if (!ENV.PGLITE) {
-    return (await pool.query<T>(sql, params)).rows;
-  }
-  const lock = "repro-poll";
-  await run(() => acquireDBMutex(lock));
-  try {
-    return (await pool.query<T>(sql, params)).rows;
-  } finally {
-    releaseDBMutex(lock);
-  }
-}
-
-/** Latest finalized effectstream block height (incl. empty blocks). 0 if none. */
-export async function latestFinalizedHeight(pool: pg.Pool): Promise<number> {
-  try {
-    const rows = await q<{ h: number | null }>(
-      pool,
-      `SELECT MAX(block_height)::int AS h FROM effectstream.effectstream_blocks
-       WHERE effectstream_block_hash IS NOT NULL`,
-    );
-    return rows[0]?.h ?? 0;
-  } catch (e) {
-    // The system migrations may not have created the table yet during boot.
-    if ((e as { code?: string })?.code === "42P01") return 0;
-    throw e;
-  }
-}
-
-/** Highest persisted page-queue number for a protocol (-1 if none yet). */
-export async function maxPage(
-  pool: pg.Pool,
-  protocolName: string,
-): Promise<number> {
-  try {
-    const rows = await q<{ p: number | null }>(
-      pool,
-      `SELECT MAX(page_number)::int AS p FROM effectstream.sync_protocol_pagination
-       WHERE protocol_name = $1`,
-      [protocolName],
-    );
-    return rows[0]?.p ?? -1;
-  } catch (e) {
-    if ((e as { code?: string })?.code === "42P01") return -1;
-    throw e;
-  }
-}
-
-/** Wait until finalized height reaches `target` (or throw on timeout). */
-export async function waitForHeight(
-  pool: pg.Pool,
-  target: number,
-  timeoutMs = 30_000,
-): Promise<number> {
-  const startedAt = Date.now();
-  while (Date.now() - startedAt < timeoutMs) {
-    const h = await latestFinalizedHeight(pool);
-    if (h >= target) return h;
-    await sleep(50);
-  }
-  throw new Error(
-    `waitForHeight(${target}) timed out at ${await latestFinalizedHeight(pool)}`,
-  );
-}
-
-/**
- * Wait until the finalized height stops advancing for `quietMs` (the system has
- * reached its deterministic stable state given fixed tips).
- */
-export async function waitUntilStable(
-  pool: pg.Pool,
-  { quietMs = 800, timeoutMs = 30_000 } = {},
-): Promise<number> {
-  const startedAt = Date.now();
-  let last = -1;
-  let lastChange = Date.now();
-  while (Date.now() - startedAt < timeoutMs) {
-    const h = await latestFinalizedHeight(pool);
-    if (h !== last) {
-      last = h;
-      lastChange = Date.now();
-    } else if (Date.now() - lastChange >= quietMs) {
-      return h;
-    }
-    await sleep(50);
-  }
-  return last;
-}
-
-/** Wait until a protocol's persisted page reaches `target` (or throw). */
-export async function waitForPage(
-  pool: pg.Pool,
-  protocolName: string,
-  target: number,
-  timeoutMs = 30_000,
-): Promise<number> {
-  const startedAt = Date.now();
-  while (Date.now() - startedAt < timeoutMs) {
-    const p = await maxPage(pool, protocolName);
-    if (p >= target) return p;
-    await sleep(50);
-  }
-  throw new Error(
-    `waitForPage(${protocolName}, ${target}) timed out at ${await maxPage(pool, protocolName)}`,
-  );
-}
-
-/** Count primitive_accounting rows whose payload carries the given marker. */
-export async function countEventRows(
-  pool: pg.Pool,
-  marker: string,
-): Promise<number> {
-  const rows = await q<{ c: number | null }>(
-    pool,
-    `SELECT COUNT(*)::int AS c FROM effectstream.primitive_accounting
-     WHERE payload->>'marker' = $1`,
-    [marker],
-  );
-  return rows[0]?.c ?? 0;
-}
-
-/** Page-queue rows for a protocol (the durable per-chunk queue). */
-export async function pageRows(
-  pool: pg.Pool,
-  protocolName: string,
-): Promise<{ page_number: number }[]> {
-  return await q<{ page_number: number }>(
-    pool,
-    `SELECT page_number FROM effectstream.sync_protocol_pagination
-     WHERE protocol_name = $1 ORDER BY page_number ASC`,
-    [protocolName],
-  );
-}

package/test/reproduction/resume.test.ts

@@ -1,77 +0,0 @@
-/**
- * Resume-marker position over the synthetic `test` chain.
- *
- * Companion to consistency.test.ts (which proves a mid-chunk restart re-fetches
- * uncommitted data instead of skipping it). This file pins down *where* the
- * persisted resume marker lands for a chain whose data is sparse — the property
- * that bounds restart re-scan (see @effectstream/sync CLAUDE.md, design idea #5).
- *
- * The marker is the highest datum merged into the committed block, and because
- * each fetched chunk's boundary page is buffered and consumed even when empty, it
- * tracks the **committed frontier** — NOT the last block that happened to carry
- * data. So a chain that produced one event long ago and has been quiet since
- * still resumes near the frontier, re-fetching at most ~stepSize blocks, never
- * its whole quiet tail. Here: data lives only at block 5, yet the marker must sit
- * at the frontier (height 50 / 60), not stuck back at 5.
- */
-import { afterAll, beforeAll, expect, test } from "bun:test";
-import {
-  countEventRows,
-  type Harness,
-  latestFinalizedHeight,
-  maxPage,
-  setupHarness,
-} from "./harness.ts";
-
-let h: Harness;
-
-beforeAll(async () => {
-  h = await setupHarness();
-});
-
-afterAll(async () => {
-  await h?.teardown();
-});
-
-test(
-  "a sparse chain's resume marker tracks the frontier, not its last data block",
-  async () => {
-    // Data lives ONLY at parallel block 5; everything after is quiet. Block 5 is
-    // the marker a naive "resume from last data" scheme would persist.
-    const events = [{ atBlock: 5, marker: "evt5" }];
-
-    // Phase 1 — sync the main clock to 50 (parallel tip stays ahead so the merge
-    // can finalize). The block-5 event is merged early; the chain is quiet after.
-    await h.runToHeight({
-      events,
-      tips: { mainClock: 50, parallelP: 200 },
-      target: 50,
-      apiPort: 19161,
-    });
-
-    expect(await latestFinalizedHeight(h.pool)).toBe(50);
-    expect(await countEventRows(h.pool, "evt5")).toBe(1);
-
-    // The crux: the parallel marker tracks the committed frontier (50), NOT the
-    // last data block (5). A restart therefore resumes from ~51 and re-scans only
-    // the bounded gap up to the frontier — never blocks 6..50 from scratch.
-    expect(await maxPage(h.pool, "parallelP")).toBe(50);
-    expect(await maxPage(h.pool, "parallelP")).not.toBe(5);
-
-    // Phase 2 (restart) — boot a fresh process against the same DB and sync on to
-    // 60. The parallel chain resumes from the frontier (51), not from block 5.
-    await h.runToHeight({
-      events,
-      tips: { mainClock: 60, parallelP: 200 },
-      target: 60,
-      apiPort: 19162,
-    });
-
-    expect(await latestFinalizedHeight(h.pool)).toBe(60);
-    // Marker advanced with the new frontier...
-    expect(await maxPage(h.pool, "parallelP")).toBe(60);
-    // ...and block 5 was neither re-fetched (resume started past it) nor lost.
-    expect(await countEventRows(h.pool, "evt5")).toBe(1);
-  },
-  120_000,
-);