@effectstream/runtime 0.100.18 → 0.100.19

package/test/reproduction/node-runner.ts

@@ -0,0 +1,152 @@
+/**
+ * One-node subprocess entry point.
+ *
+ * A real engine restart is a new OS process whose in-memory state (notably the
+ * sync Deque) is gone while PostgreSQL keeps every committed row. We model that
+ * literally: each node runs here, in its own `bun` process, against a database
+ * the parent harness created. Booting the runtime twice *in one process* is not
+ * how production works and proved unreliable (a second in-process `start()`
+ * stalls the sync loops under `bun test`), so the harness spawns this instead.
+ *
+ * Invocation (by the harness):
+ *   bun node-runner.ts '<json-RunSpec>'
+ * with DB_* / PGLITE env already set (by the harness) to point at the target
+ * database — a real Postgres server or a PGLite gateway.
+ *
+ * Exit 0 once the node has synced to the requested height (and, for the resume
+ * scenario, the requested page watermarks); exit 1 on boot failure.
+ */
+import pg from "pg";
+import { run, type Task } from "effection";
+import {
+  toSyncProtocolWithNetwork,
+  withEffectstreamStaticConfig,
+} from "@effectstream/config";
+import { TestChainControl } from "@effectstream/sync";
+import { init, start } from "../../src/main.ts";
+import type { StartConfigGameStateTransitions } from "../../src/types.ts";
+import {
+  buildConfig,
+  type EventSpec,
+  TEST_PRIMITIVE_TYPE,
+  TestEventPrimitive,
+} from "./scenario.ts";
+import { waitForHeight, waitForPage, waitUntilStable } from "./poll.ts";
+
+export type RunSpec = {
+  securityNamespace: string;
+  events: EventSpec[];
+  parallelStepSize?: number;
+  /** Manual chain tips, e.g. { mainClock: 100, parallelP: 200 }. */
+  tips: Record<string, number>;
+  /** Finalized height to reach before exiting. */
+  target: number;
+  apiPort: number;
+  /**
+   * Optional per-protocol page watermarks to also wait for before exiting.
+   * Used by the resume scenario to guarantee the parallel fetcher has raced
+   * ahead and persisted its page row before we kill the process.
+   */
+  waitPages?: Record<string, number>;
+};
+
+const noopStf: StartConfigGameStateTransitions = function* () {};
+
+async function main() {
+  const spec: RunSpec = JSON.parse(process.argv[2] ?? "{}");
+
+  // DB_* and PGLITE are set by the harness (real Postgres or a PGLite gateway);
+  // do NOT override them here. Only force the bits every run shares: no MQTT
+  // broker (it binds fixed ports and isn't freed on halt) and a per-run API port.
+  process.env.MQTT_BROKER = "false";
+  process.env.EFFECTSTREAM_API_PORT = String(spec.apiPort);
+  const isPglite = process.env.PGLITE === "true";
+
+  for (const [name, tip] of Object.entries(spec.tips)) {
+    TestChainControl.setTip(name, tip);
+  }
+
+  const pool = new pg.Pool({
+    host: process.env.DB_HOST,
+    port: Number(process.env.DB_PORT),
+    user: process.env.DB_USER,
+    database: process.env.DB_NAME,
+    password: process.env.DB_PW || undefined,
+    // PGLite is a single WASM backend: cap polls at one connection (they're also
+    // mutex-serialized against the engine — see poll.ts).
+    max: isPglite ? 1 : 10,
+  });
+
+  const cfg = (spec as any).config ?? buildConfig({
+    securityNamespace: spec.securityNamespace,
+    events: spec.events,
+    parallelStepSize: spec.parallelStepSize,
+  });
+
+  let bootErr: unknown;
+  let started = false;
+  let live: any[] = [];
+  const task: Task<unknown> = run(function* () {
+    yield* init();
+    yield* withEffectstreamStaticConfig(cfg, function* () {
+      yield* start({
+        appName: "sync-repro",
+        appVersion: "1.0.0",
+        syncInfo: toSyncProtocolWithNetwork(cfg),
+        gameStateTransitions: noopStf,
+        userDefinedPrimitives: { [TEST_PRIMITIVE_TYPE]: TestEventPrimitive },
+        dev: {
+          resetPublicData: false,
+          onStarted: ({ syncProtocols }: any) => {
+            live = syncProtocols;
+            started = true;
+          },
+        },
+      });
+    });
+  });
+  // start() runs forever; capture a genuine boot failure (not the expected
+  // "halted" rejection from our own task.halt() below).
+  Promise.resolve(task).catch((e) => {
+    if (!String(e).includes("halted")) bootErr = e;
+  });
+
+  const guard = async <T>(p: Promise<T>): Promise<T> => {
+    const r = await p;
+    if (bootErr) throw new Error(`node boot failed: ${String(bootErr)}`);
+    return r;
+  };
+
+  // Don't poll until `onStarted` fires: init()'s migrations and startup() run
+  // before it and aren't mutex-guarded, so on PGLite a poll could overlap them
+  // and corrupt the single WASM backend. Steady-state queries after this are.
+  while (!started) {
+    if (bootErr) throw new Error(`node boot failed: ${String(bootErr)}`);
+    await new Promise((r) => setTimeout(r, 20));
+  }
+
+  try {
+    await guard(waitForHeight(pool, spec.target));
+    await guard(waitUntilStable(pool));
+    for (const [name, page] of Object.entries(spec.waitPages ?? {})) {
+      await guard(waitForPage(pool, name, page));
+    }
+  } catch (e) {
+    const probe = live.map((p) =>
+      `${p.name}: lastPage=${JSON.stringify(p.lastPage)} buf=${p.bufferedData.size()} errs=${p.consecutiveErrors}`
+    );
+    console.error("PROBE:\n  " + probe.join("\n  "));
+    throw e;
+  } finally {
+    await task.halt().catch(() => {});
+    await pool.end().catch(() => {});
+  }
+}
+
+main().then(
+  () => process.exit(0),
+  (e) => {
+    console.error(String(e instanceof Error ? e.stack ?? e.message : e));
+    process.exit(1);
+  },
+);

package/test/reproduction/poll.ts

@@ -0,0 +1,160 @@
+/**
+ * pg.Pool query helpers shared by the test harness (parent process, used for
+ * assertions) and {@link ./node-runner.ts} (child process, used to decide when a
+ * node has finished syncing). No cluster / runtime imports.
+ *
+ * On PGLite every query is serialized through the engine's process-global DB
+ * mutex — see {@link q}. On a real Postgres server that mutex is a no-op and
+ * these are plain pooled queries.
+ */
+import type pg from "pg";
+import { run } from "effection";
+import { ENV } from "@effectstream/utils/node-env";
+import { acquireDBMutex, releaseDBMutex } from "@effectstream/db";
+
+export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Run a query, serialized through the engine's process-global DB mutex on PGLite
+ * (a single WASM backend that two overlapping queries corrupt). node-runner
+ * polls while the engine syncs in the SAME process, so taking the same mutex the
+ * engine uses guarantees no overlap — they share the module-global lock even
+ * across separate `run()` roots. On real Postgres the mutex is a no-op.
+ */
+async function q<T extends pg.QueryResultRow = pg.QueryResultRow>(
+  pool: pg.Pool,
+  sql: string,
+  params?: unknown[],
+): Promise<T[]> {
+  if (!ENV.PGLITE) {
+    return (await pool.query<T>(sql, params)).rows;
+  }
+  const lock = "repro-poll";
+  await run(() => acquireDBMutex(lock));
+  try {
+    return (await pool.query<T>(sql, params)).rows;
+  } finally {
+    releaseDBMutex(lock);
+  }
+}
+
+/** Latest finalized effectstream block height (incl. empty blocks). 0 if none. */
+export async function latestFinalizedHeight(pool: pg.Pool): Promise<number> {
+  try {
+    const rows = await q<{ h: number | null }>(
+      pool,
+      `SELECT MAX(block_height)::int AS h FROM effectstream.effectstream_blocks
+       WHERE effectstream_block_hash IS NOT NULL`,
+    );
+    return rows[0]?.h ?? 0;
+  } catch (e) {
+    // The system migrations may not have created the table yet during boot.
+    if ((e as { code?: string })?.code === "42P01") return 0;
+    throw e;
+  }
+}
+
+/** Highest persisted page-queue number for a protocol (-1 if none yet). */
+export async function maxPage(
+  pool: pg.Pool,
+  protocolName: string,
+): Promise<number> {
+  try {
+    const rows = await q<{ p: number | null }>(
+      pool,
+      `SELECT MAX(page_number)::int AS p FROM effectstream.sync_protocol_pagination
+       WHERE protocol_name = $1`,
+      [protocolName],
+    );
+    return rows[0]?.p ?? -1;
+  } catch (e) {
+    if ((e as { code?: string })?.code === "42P01") return -1;
+    throw e;
+  }
+}
+
+/** Wait until finalized height reaches `target` (or throw on timeout). */
+export async function waitForHeight(
+  pool: pg.Pool,
+  target: number,
+  timeoutMs = 30_000,
+): Promise<number> {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    const h = await latestFinalizedHeight(pool);
+    if (h >= target) return h;
+    await sleep(50);
+  }
+  throw new Error(
+    `waitForHeight(${target}) timed out at ${await latestFinalizedHeight(pool)}`,
+  );
+}
+
+/**
+ * Wait until the finalized height stops advancing for `quietMs` (the system has
+ * reached its deterministic stable state given fixed tips).
+ */
+export async function waitUntilStable(
+  pool: pg.Pool,
+  { quietMs = 800, timeoutMs = 30_000 } = {},
+): Promise<number> {
+  const startedAt = Date.now();
+  let last = -1;
+  let lastChange = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    const h = await latestFinalizedHeight(pool);
+    if (h !== last) {
+      last = h;
+      lastChange = Date.now();
+    } else if (Date.now() - lastChange >= quietMs) {
+      return h;
+    }
+    await sleep(50);
+  }
+  return last;
+}
+
+/** Wait until a protocol's persisted page reaches `target` (or throw). */
+export async function waitForPage(
+  pool: pg.Pool,
+  protocolName: string,
+  target: number,
+  timeoutMs = 30_000,
+): Promise<number> {
+  const startedAt = Date.now();
+  while (Date.now() - startedAt < timeoutMs) {
+    const p = await maxPage(pool, protocolName);
+    if (p >= target) return p;
+    await sleep(50);
+  }
+  throw new Error(
+    `waitForPage(${protocolName}, ${target}) timed out at ${await maxPage(pool, protocolName)}`,
+  );
+}
+
+/** Count primitive_accounting rows whose payload carries the given marker. */
+export async function countEventRows(
+  pool: pg.Pool,
+  marker: string,
+): Promise<number> {
+  const rows = await q<{ c: number | null }>(
+    pool,
+    `SELECT COUNT(*)::int AS c FROM effectstream.primitive_accounting
+     WHERE payload->>'marker' = $1`,
+    [marker],
+  );
+  return rows[0]?.c ?? 0;
+}
+
+/** Page-queue rows for a protocol (the durable per-chunk queue). */
+export async function pageRows(
+  pool: pg.Pool,
+  protocolName: string,
+): Promise<{ page_number: number }[]> {
+  return await q<{ page_number: number }>(
+    pool,
+    `SELECT page_number FROM effectstream.sync_protocol_pagination
+     WHERE protocol_name = $1 ORDER BY page_number ASC`,
+    [protocolName],
+  );
+}

package/test/reproduction/resume.test.ts

@@ -0,0 +1,77 @@
+/**
+ * Resume-marker position over the synthetic `test` chain.
+ *
+ * Companion to consistency.test.ts (which proves a mid-chunk restart re-fetches
+ * uncommitted data instead of skipping it). This file pins down *where* the
+ * persisted resume marker lands for a chain whose data is sparse — the property
+ * that bounds restart re-scan (see @effectstream/sync CLAUDE.md, design idea #5).
+ *
+ * The marker is the highest datum merged into the committed block, and because
+ * each fetched chunk's boundary page is buffered and consumed even when empty, it
+ * tracks the **committed frontier** — NOT the last block that happened to carry
+ * data. So a chain that produced one event long ago and has been quiet since
+ * still resumes near the frontier, re-fetching at most ~stepSize blocks, never
+ * its whole quiet tail. Here: data lives only at block 5, yet the marker must sit
+ * at the frontier (height 50 / 60), not stuck back at 5.
+ */
+import { afterAll, beforeAll, expect, test } from "bun:test";
+import {
+  countEventRows,
+  type Harness,
+  latestFinalizedHeight,
+  maxPage,
+  setupHarness,
+} from "./harness.ts";
+
+let h: Harness;
+
+beforeAll(async () => {
+  h = await setupHarness();
+});
+
+afterAll(async () => {
+  await h?.teardown();
+});
+
+test(
+  "a sparse chain's resume marker tracks the frontier, not its last data block",
+  async () => {
+    // Data lives ONLY at parallel block 5; everything after is quiet. Block 5 is
+    // the marker a naive "resume from last data" scheme would persist.
+    const events = [{ atBlock: 5, marker: "evt5" }];
+
+    // Phase 1 — sync the main clock to 50 (parallel tip stays ahead so the merge
+    // can finalize). The block-5 event is merged early; the chain is quiet after.
+    await h.runToHeight({
+      events,
+      tips: { mainClock: 50, parallelP: 200 },
+      target: 50,
+      apiPort: 19161,
+    });
+
+    expect(await latestFinalizedHeight(h.pool)).toBe(50);
+    expect(await countEventRows(h.pool, "evt5")).toBe(1);
+
+    // The crux: the parallel marker tracks the committed frontier (50), NOT the
+    // last data block (5). A restart therefore resumes from ~51 and re-scans only
+    // the bounded gap up to the frontier — never blocks 6..50 from scratch.
+    expect(await maxPage(h.pool, "parallelP")).toBe(50);
+    expect(await maxPage(h.pool, "parallelP")).not.toBe(5);
+
+    // Phase 2 (restart) — boot a fresh process against the same DB and sync on to
+    // 60. The parallel chain resumes from the frontier (51), not from block 5.
+    await h.runToHeight({
+      events,
+      tips: { mainClock: 60, parallelP: 200 },
+      target: 60,
+      apiPort: 19162,
+    });
+
+    expect(await latestFinalizedHeight(h.pool)).toBe(60);
+    // Marker advanced with the new frontier...
+    expect(await maxPage(h.pool, "parallelP")).toBe(60);
+    // ...and block 5 was neither re-fetched (resume started past it) nor lost.
+    expect(await countEventRows(h.pool, "evt5")).toBe(1);
+  },
+  120_000,
+);

package/test/reproduction/sanity.test.ts

@@ -0,0 +1,74 @@
+/**
+ * Sanity end-to-end test for the synthetic `test` chain (no fixes required).
+ *
+ * Proves the foundation this PR adds works: a TEST_MAIN clock + a TEST_PARALLEL
+ * chain sync through the real runtime (`start()`), a config-declared event is
+ * processed into `effectstream.primitive_accounting`, and committed state
+ * survives a restart.
+ *
+ * Runs on whichever backend `setupHarness` picks: the default in-memory PGLite
+ * (no external dependency) or real Postgres when `PGLITE=false`. Either way
+ * each node runs in its own subprocess (see harness.ts / node-runner.ts), so a
+ * restart is a genuine new OS process booting against the same database.
+ *
+ * (The buffering / resume / fast-catchup reproduction tests land alongside their
+ * fixes in follow-up PRs — they assert behaviour that only those fixes provide.)
+ */
+import { afterAll, beforeAll, expect, test } from "bun:test";
+import {
+  countEventRows,
+  type Harness,
+  latestFinalizedHeight,
+  setupHarness,
+} from "./harness.ts";
+
+let h: Harness;
+
+beforeAll(async () => {
+  h = await setupHarness();
+});
+
+afterAll(async () => {
+  await h?.teardown();
+});
+
+test("test chain syncs, processes a configured event, and survives restart", async () => {
+  const events = [{ atBlock: 50, marker: "evt50" }];
+
+  // Parallel tip (200) stays ahead of the main clock (100) so the merge is not
+  // gated at the chunk boundary.
+  await h.runToHeight({
+    events,
+    tips: { mainClock: 100, parallelP: 200 },
+    target: 100,
+    apiPort: 19131,
+  });
+
+  const height1 = await latestFinalizedHeight(h.pool);
+  const events1 = await countEventRows(h.pool, "evt50");
+  console.log(
+    `[sanity] after sync:    height ${height1} (expect 100), ` +
+      `evt50 rows ${events1} (expect 1)`,
+  );
+  expect(height1).toBe(100);
+  // The event declared at block 50 was processed into primitive_accounting.
+  expect(events1).toBe(1);
+
+  // Restart: a new process boots against the same database. Committed state must
+  // persist; the chain is already fully caught up, so nothing new is produced.
+  await h.runToHeight({
+    events,
+    tips: { mainClock: 100, parallelP: 200 },
+    target: 100,
+    apiPort: 19132,
+  });
+
+  const height2 = await latestFinalizedHeight(h.pool);
+  const events2 = await countEventRows(h.pool, "evt50");
+  console.log(
+    `[sanity] after restart: height ${height2} (expect 100, was ${height1}), ` +
+      `evt50 rows ${events2} (expect 1, was ${events1})`,
+  );
+  expect(height2).toBe(100);
+  expect(events2).toBe(1);
+}, 60_000);

package/test/reproduction/scenario.ts

@@ -0,0 +1,126 @@
+/**
+ * Shared scenario building blocks for the reproduction tests.
+ *
+ * Both the in-process test files and the out-of-process {@link ./node-runner.ts}
+ * subprocess build the *same* synthetic `test`-chain config from these helpers,
+ * so a node booted in a child process is identical to one a test would describe.
+ * Keeping the config + primitive here (no DB, no cluster imports) lets the
+ * subprocess import it cheaply.
+ */
+import {
+  ConfigBuilder,
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+} from "@effectstream/config";
+import { Primitive } from "@effectstream/sm";
+
+/** Fixed wall-clock origin so block timestamps are pure arithmetic. */
+export const START_TIME = 1_700_000_000_000;
+export const BLOCK_TIME_MS = 1000;
+
+export type EventSpec = { atBlock: number; marker: string };
+
+export type ScenarioOpts = {
+  events: EventSpec[];
+  /** Parallel-chain page size. Large (default 1000) => whole tip is one chunk. */
+  parallelStepSize?: number;
+  /** Security namespace; distinct per test keeps databases independent. */
+  securityNamespace?: string;
+};
+
+export function buildConfig(opts: ScenarioOpts) {
+  const stepSize = opts.parallelStepSize ?? 1000;
+  const namespace = opts.securityNamespace ?? "sync-repro";
+  return new ConfigBuilder()
+    .setNamespace((b) => b.setSecurityNamespace(namespace))
+    .buildNetworks((b) =>
+      b
+        .addNetwork({
+          name: "clock",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+        .addNetwork({
+          name: "chainP",
+          type: ConfigNetworkType.TEST,
+          startTime: START_TIME,
+          blockTimeMS: BLOCK_TIME_MS,
+        })
+    )
+    .buildDeployments((b) => b)
+    .buildSyncProtocols((b) =>
+      b
+        .addMain(
+          (n) => n.clock,
+          () => ({
+            name: "mainClock",
+            type: ConfigSyncProtocolType.TEST_MAIN,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize: 1000,
+          }),
+        )
+        .addParallel(
+          (n) => (n as any).chainP,
+          () => ({
+            name: "parallelP",
+            type: ConfigSyncProtocolType.TEST_PARALLEL,
+            startBlockHeight: 1,
+            pollingInterval: 30,
+            stepSize,
+            delayMs: 0,
+            events: opts.events.map((e) => ({
+              atBlock: e.atBlock,
+              payload: { marker: e.marker },
+            })),
+          }),
+        )
+    )
+    .buildPrimitives((b) =>
+      b.addPrimitive(
+        (sp) => (sp as any).parallelP,
+        () => ({
+          name: "testEvt",
+          type: TEST_PRIMITIVE_TYPE,
+          startBlockHeight: 1,
+        }),
+      )
+    )
+    .build();
+}
+
+// ── A minimal user-defined primitive ─────────────────────────────────────────
+// Writes one row to effectstream.primitive_accounting per event (no STF needed:
+// stateMachinePayload is null). Tests assert on that table.
+export const TEST_PRIMITIVE_TYPE = "TEST:EVENT";
+
+export class TestEventPrimitive extends Primitive<any, any> {
+  readonly internalTypeName = TEST_PRIMITIVE_TYPE as any;
+  override grammar = [] as any;
+  constructor(config: any) {
+    super(config);
+  }
+  // deno-lint-ignore require-yield
+  override *getPayload(_height: any, primitiveTransactionData: any): any {
+    const payload = primitiveTransactionData.output.payload;
+    return {
+      isBatched: false,
+      data: [
+        {
+          fromAddressAndType: { type: "none", address: "0x0" },
+          accountingPayload: payload,
+          stateMachinePayload: null,
+        },
+      ],
+    };
+  }
+  override getConfig(): any {
+    return {
+      name: this.instanceName,
+      type: this.internalTypeName,
+      startBlockHeight: this.startBlockHeight,
+      scheduledPrefix: this.stateMachinePrefix,
+    };
+  }
+}