@effectstream/sync 0.101.1 → 0.103.0

package/test/spin-probe.ts

@@ -0,0 +1,102 @@
+/**
+ * Child process for `poll-loop-spin.test.ts` (sync/CLAUDE.md Finding #3).
+ *
+ * Drives the REAL `startSync` fetch loop with a fake `SyncState` that is always
+ * "caught up" (`stateToInput` → `undefined`) — the steady state of any protocol
+ * whose tip advances slowly. The only variable is whether the protocol config
+ * carries a `pollingInterval`, because `orchestration/sync.ts` guards all three
+ * of its `sleep`s with `if ("pollingInterval" in config.syncProtocol)`.
+ *
+ * Runs in its own process because a loop with no sleep can starve the event
+ * loop outright, which would hang the test runner rather than fail it. The two
+ * exit paths below discriminate the outcomes:
+ *
+ *   - `reason: "limit"`  — the loop reached `limit` iterations. Reported via
+ *                          `writeSync` + `process.exit`, which work even when
+ *                          the event loop is starved.
+ *   - `reason: "window"` — a `setTimeout` fired first, so the loop is paced and
+ *                          the event loop is healthy.
+ *
+ * Usage: bun spin-probe.ts '{"polling":20,"limit":50000,"windowMs":500,"mode":"call"}'
+ */
+import { writeSync } from "node:fs";
+import { call, type Operation, run } from "effection";
+import { startSync } from "../src/sync-protocols/orchestration/sync.ts";
+
+type ProbeSpec = {
+  /** `pollingInterval` for the fake protocol config; `null` omits the key entirely. */
+  polling: number | null;
+  /** Iteration count at which we declare the loop unpaced and bail out. */
+  limit: number;
+  /** Wall-clock budget; if this timer fires, the loop is paced. */
+  windowMs: number;
+  /**
+   * How `stateToInput` returns:
+   *  - "sync" — returns `undefined` with no suspension at all (worst case).
+   *  - "call" — `yield* call(() => <sync value>)` first, mirroring utxorpc's
+   *             `stateToInput`, which calls the in-memory `fetcher.lastHeight()`
+   *             before deciding it is caught up. This is the production shape.
+   */
+  mode: "sync" | "call";
+};
+
+const spec: ProbeSpec = JSON.parse(process.argv[2]);
+
+const startedAt = Date.now();
+let iterations = 0;
+
+function report(reason: "limit" | "window"): never {
+  writeSync(
+    1,
+    JSON.stringify({
+      reason,
+      iterations,
+      elapsedMs: Date.now() - startedAt,
+    }) + "\n",
+  );
+  process.exit(0);
+}
+
+// Fires only if the loop leaves room for macrotasks.
+setTimeout(() => report("window"), spec.windowMs);
+
+const syncProtocol: Record<string, unknown> = { name: "probe" };
+if (spec.polling != null) syncProtocol.pollingInterval = spec.polling;
+
+/**
+ * Minimal stand-in for a `SyncState`. Only the members `startSync` actually
+ * touches are implemented; it is cast at the call site the same way the real
+ * states are (`startSync` takes `AllSyncProtocols` and narrows to `ISyncProtocol`).
+ */
+const fakeState = {
+  name: "probe",
+  consecutiveErrors: 0,
+  lastErrorTimestamp: 0,
+  lastSuccessfulFetchMs: 0,
+  getNamespace: () => ["probe", "probe"],
+  *startAsync(): Operation<void> {},
+  *stateToInput(): Operation<undefined> {
+    iterations++;
+    if (iterations >= spec.limit) report("limit");
+    if (spec.mode === "call") {
+      // utxorpc reads an in-memory tip here — a `call` over a synchronous value.
+      yield* call(() => 0);
+    }
+    // "caught up": nothing to fetch this pass.
+    return undefined;
+  },
+  fetcher: {
+    config: { syncProtocol },
+    producerChannel: { *send(): Operation<void> {} },
+    *readData(): Operation<never> {
+      throw new Error("unreachable: stateToInput always returns undefined");
+    },
+  },
+  *updateState(): Operation<void> {},
+};
+
+await run(function* () {
+  yield* startSync(fakeState as never);
+  // Park forever; one of the two exit paths above ends the process.
+  yield* call(() => new Promise<void>(() => {}));
+});

package/test/start-async-supervision.test.ts

@@ -0,0 +1,139 @@
+/**
+ * Regression guard for sync/CLAUDE.md Finding #5 (unsupervised `startAsync`).
+ *
+ * `orchestration/sync.ts` used to start the streaming producer with a bare
+ * `yield* spawn(function* () { yield* iState.startAsync(); })` — no try/catch,
+ * no restart, no liveness check. Streaming chains (utxorpc today, midnight if
+ * it moves to subscriptions) put ALL of their data on that path, so both of its
+ * exit modes were unhandled:
+ *
+ *   5a — the stream ENDS CLEANLY. `WatchMultiplexer.runWatcher` logs
+ *        "stream ended unexpectedly" and returns; the spawned task completed
+ *        normally. The chain then received nothing forever while every health
+ *        counter stayed clean, and the merge blocked on its page — a total stall
+ *        with nothing reporting an error.
+ *
+ *   5b — the stream THROWS. The error escaped the spawned task and tore down the
+ *        enclosing scope, which in production is `start()`: one flaky gRPC
+ *        connection took down every unrelated chain with it.
+ *
+ * Now supervised — both modes are treated as failure and restarted with capped
+ * exponential backoff, counted in `producerRestarts`. A returning producer
+ * counts as failure because the observable effect is identical: no more data.
+ *
+ * Supervision is gated on `hasAsyncProducer` so polled chains, whose
+ * `startAsync` is the base-class no-op, are still run exactly once (5c).
+ */
+import { expect, test } from "bun:test";
+import { type Operation, run, sleep } from "effection";
+import { startSync } from "../src/sync-protocols/orchestration/sync.ts";
+
+const POLL_MS = 20;
+
+/**
+ * Minimal stand-in for a streaming `SyncState`. Only the members `startSync`
+ * touches are implemented. `stateToInput` always reports "caught up", which is
+ * what a streaming chain does while it waits for its producer to deliver.
+ */
+function makeStreamingState(
+  startAsyncBody: () => Operation<void>,
+  hasAsyncProducer = true,
+) {
+  const counters = { startAsyncCalls: 0, pollPasses: 0 };
+  const state = {
+    name: "streamer",
+    consecutiveErrors: 0,
+    lastErrorTimestamp: 0,
+    lastSuccessfulFetchMs: 0,
+    hasAsyncProducer,
+    producerRestarts: 0,
+    producerErrors: 0,
+    lastProducerErrorMs: 0,
+    pollingIntervalMs: POLL_MS,
+    getNamespace: () => ["streamer", "streamer"],
+    *startAsync(): Operation<void> {
+      counters.startAsyncCalls++;
+      yield* startAsyncBody();
+    },
+    *stateToInput(): Operation<undefined> {
+      counters.pollPasses++;
+      return undefined;
+    },
+    fetcher: {
+      config: { syncProtocol: { name: "streamer", pollingInterval: POLL_MS } },
+      producerChannel: { *send(): Operation<void> {} },
+      *readData(): Operation<never> {
+        throw new Error("unreachable: stateToInput always returns undefined");
+      },
+    },
+    *updateState(): Operation<void> {},
+  };
+  return { state, counters };
+}
+
+test("5a: a producer that ends cleanly is restarted and counted", async () => {
+  // The multiplexer's `for await (const event of stream)` falling through —
+  // the stream closed and `runWatcher` returned.
+  const { state, counters } = makeStreamingState(function* () {
+    // returns immediately
+  });
+
+  await run(function* () {
+    yield* startSync(state as never);
+    // Long enough for the first restart (1s backoff) plus room to spare.
+    yield* sleep(1_500);
+  });
+
+  // Restarted rather than abandoned...
+  expect(counters.startAsyncCalls).toBeGreaterThan(1);
+  // ...and the restart is visible to health instead of being silent.
+  expect(state.producerRestarts).toBeGreaterThan(0);
+  // A clean return is not an *error*, so only the restart counter moves.
+  expect(state.producerErrors).toBe(0);
+  // The sibling polling loop is unaffected.
+  expect(counters.pollPasses).toBeGreaterThan(1);
+}, 30_000);
+
+test("5b: a producer that throws is restarted and does not kill the scope", async () => {
+  const { state, counters } = makeStreamingState(function* () {
+    yield* sleep(50);
+    throw new Error("gRPC stream died");
+  });
+
+  let rejection: unknown;
+  await run(function* () {
+    yield* startSync(state as never);
+    yield* sleep(1_500);
+  }).catch((err) => {
+    rejection = err;
+  });
+
+  // The scope survived: no error escaped to tear down `start()`.
+  expect(rejection).toBeUndefined();
+  // The throw was caught, counted, and the producer restarted.
+  expect(counters.startAsyncCalls).toBeGreaterThan(1);
+  expect(state.producerRestarts).toBeGreaterThan(0);
+  expect(state.producerErrors).toBeGreaterThan(0);
+  expect(state.lastProducerErrorMs).toBeGreaterThan(0);
+  // Counted on the PRODUCER's own tally, not the fetch loop's. Sharing
+  // `consecutiveErrors` conflated two signals: only a successful `readData`
+  // clears it, so an idle streaming chain stayed `erroring` long after its
+  // producer recovered, and a successful poll erased the record of a flap.
+  expect(state.consecutiveErrors).toBe(0);
+  // And the sibling polling loop kept running throughout.
+  expect(counters.pollPasses).toBeGreaterThan(1);
+}, 30_000);
+
+test("5c: a polled chain's no-op producer is run once, not supervised", async () => {
+  const { state, counters } = makeStreamingState(function* () {}, false);
+
+  await run(function* () {
+    yield* startSync(state as never);
+    yield* sleep(1_500);
+  });
+
+  // Without this gate, every polled chain would "restart" its no-op forever.
+  expect(counters.startAsyncCalls).toBe(1);
+  expect(state.producerRestarts).toBe(0);
+  expect(counters.pollPasses).toBeGreaterThan(1);
+}, 30_000);

package/test/zswap-decoder.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, test } from "bun:test";
+import { Buffer } from "node:buffer";
+import { decodeZswapEvent } from "../src/sync-protocols/midnight/zswap-decoder.ts";
+
+// Synthetic events built per the real midnight-ledger 8.x wire format:
+// "midnight:event[v9]:" tag + TopoSortedNodes envelope wrapping the event
+// struct. Both blobs round-trip through the native ledger-v8 deserializer.
+
+function compactU32(n: number): Buffer {
+  if (n < 64) return Buffer.from([n << 2]);
+  if (n < 16384) {
+    const v = (n << 2) | 0b01;
+    return Buffer.from([v & 0xff, (v >> 8) & 0xff]);
+  }
+  throw new Error("value too large for this helper");
+}
+
+const TX_HASH = Buffer.alloc(32, 0xaa);
+const BODY32 = Buffer.alloc(32, 0xbb);
+// logical=3, physical=1
+const SEGMENTS = Buffer.from([0x03, 0x00, 0x01, 0x00]);
+const CONTRACT_NONE = Buffer.from([0x01]);
+
+function wrapEnvelope(data: Buffer): string {
+  return Buffer.concat([
+    Buffer.from("midnight:event[v9]:", "ascii"),
+    compactU32(1), // node count
+    compactU32(0), // child count of the single node
+    compactU32(data.length),
+    data,
+  ]).toString("hex");
+}
+
+describe("decodeZswapEvent", () => {
+  test("decodes a ZswapInput (nullifier) event", () => {
+    const raw = wrapEnvelope(Buffer.concat([
+      TX_HASH,
+      SEGMENTS,
+      Buffer.from([0x00]), // variant: ZswapInput
+      BODY32, // nullifier
+      CONTRACT_NONE,
+    ]));
+    expect(decodeZswapEvent(raw)).toEqual({
+      kind: "nullifier",
+      nullifier: "bb".repeat(32),
+      txHash: "aa".repeat(32),
+      logicalSegment: 3,
+    });
+  });
+
+  test("decodes a ZswapOutput (commitment) event", () => {
+    const raw = wrapEnvelope(Buffer.concat([
+      TX_HASH,
+      SEGMENTS,
+      Buffer.from([0x01]), // variant: ZswapOutput
+      BODY32, // commitment
+      Buffer.from([0x02]), // ZswapPreimageEvidence::None
+      CONTRACT_NONE,
+      compactU32(7), // mt_index
+    ]));
+    expect(decodeZswapEvent(raw)).toEqual({
+      kind: "commitment",
+      commitment: "bb".repeat(32),
+      mtIndex: "7",
+      txHash: "aa".repeat(32),
+      logicalSegment: 3,
+    });
+  });
+
+  test("returns null for undecodable input", () => {
+    expect(decodeZswapEvent("deadbeef")).toBeNull();
+    expect(decodeZswapEvent("")).toBeNull();
+  });
+});