@effectstream/sync 0.101.1 → 0.102.0

package/src/sync-protocols/utxorpc/WatchMultiplexer.ts

@@ -31,6 +31,21 @@ export class WatchMultiplexer {
   private highestConfirmedHeight: number | undefined = undefined;
   private lowestIncompleteHeight: number = 0;
   private slotOriginTimestamp: bigint | undefined = undefined;
+  /**
+   * Incremented by every {@link start} call so events from a previous
+   * generation's streams can be identified and dropped.
+   *
+   * `start` can now be called more than once: `startSync` supervises the
+   * producer and restarts it when the stream dies. All multiplexer state lives
+   * on this instance, and the gRPC streams of the previous generation are not
+   * cancellable from here — a `Promise.all` rejection from ONE watcher aborts
+   * `start` while its siblings keep streaming. Without this guard a restart
+   * would run two live streams per predicate into the same `pendingBlocks`,
+   * pushing each matched tx twice. Duplicate primitives in a block mean
+   * duplicate scheduled STF inputs, i.e. non-deterministic state — the exact
+   * failure the whole pipeline is built to avoid.
+   */
+  private generation = 0;
 
   constructor(
     private readonly clientOptions: ClientBuilderOptions,
@@ -88,14 +103,39 @@ export class WatchMultiplexer {
       }
     }
 
+    const generation = ++this.generation;
+
+    // Drop half-assembled work from the previous generation. A block is only
+    // completed once EVERY watcher has passed it, so at the moment a stream
+    // died some heights had been seen by some watchers and not others. The new
+    // stream re-delivers those heights from the resume point, and merging the
+    // two views would double their txs. Heights below `lowestIncompleteHeight`
+    // are already completed and are never revisited, so this discards exactly
+    // the in-flight window.
+    if (generation > 1) {
+      for (const [, watcher] of this.watchers) {
+        for (const [height] of watcher.pendingBlocks) {
+          if (height >= this.lowestIncompleteHeight) {
+            watcher.pendingBlocks.delete(height);
+          }
+        }
+        watcher.highestSeenHeight = Math.max(
+          this.lowestIncompleteHeight - 1,
+          0,
+        );
+      }
+    }
+
     const intersect = point ? [point] : [];
     const promises: Promise<void>[] = [];
     for (const [key, watcher] of this.watchers) {
       const watchClient = new CardanoWatchClient(this.clientOptions);
-      promises.push(this.runWatcher(key, watcher, watchClient, intersect));
+      promises.push(this.runWatcher(key, watcher, watchClient, intersect, generation));
     }
 
-    console.log(`[WatchMultiplexer] Started ${this.watchers.size} watcher(s)`);
+    console.log(
+      `[WatchMultiplexer] Started ${this.watchers.size} watcher(s) (generation ${generation})`,
+    );
     await Promise.all(promises);
   }
 
@@ -104,9 +144,19 @@ export class WatchMultiplexer {
     watcher: WatcherState,
     watchClient: CardanoWatchClient,
     intersect: ChainPoint[],
+    generation: number,
   ): Promise<void> {
     const stream = watchClient.watchTxByPredicate(watcher.sdkPredicate, intersect);
     for await (const event of stream) {
+      // A superseded generation must contribute nothing. Breaking here also
+      // closes the underlying stream via the async iterator's return path,
+      // which is the only way to stop it from this side.
+      if (generation !== this.generation) {
+        console.warn(
+          `[WatchMultiplexer] Watcher "${key}" from generation ${generation} superseded by ${this.generation}; closing`,
+        );
+        return;
+      }
       this.handleEvent(key, watcher, event);
     }
     console.warn(`[WatchMultiplexer] Watcher "${key}" stream ended unexpectedly`);

package/src/sync-protocols/utxorpc/state.ts

@@ -35,6 +35,9 @@ export class UtxoRpcSyncState extends SyncState<
     );
   }
 
+  /** All utxorpc data arrives via the watch stream in {@link startAsync}. */
+  public override readonly hasAsyncProducer = true;
+
   @bound
   override *startAsync(): Operation<void> {
     if (this.lastPage == null) {

package/src/syncProtocolFactory.ts

@@ -31,8 +31,11 @@ import { CelestiaFetcher } from "./sync-protocols/celestia/fetcher.ts";
 import { CelestiaSyncState } from "./sync-protocols/celestia/state.ts";
 import { NearFetcher } from "./sync-protocols/near/fetcher.ts";
 import { NearSyncState } from "./sync-protocols/near/state.ts";
+import { SolanaFetcher } from "./sync-protocols/solana/fetcher.ts";
+import { SolanaSyncState } from "./sync-protocols/solana/state.ts";
 import { TestFetcher } from "./sync-protocols/test/fetcher.ts";
 import { TestSyncState } from "./sync-protocols/test/state.ts";
+import { requestTimeoutOf } from "./sync-protocols/common/http.ts";
 
 export function* genSyncProtocols(
   dbConn: PoolClient,
@@ -56,6 +59,7 @@ export function* genSyncProtocols(
         entry,
         createViemPublicClient(viemNetwork, {
           cacheTime: entry.syncProtocol.pollingInterval,
+          timeout: requestTimeoutOf(entry.syncProtocol),
         }),
       );
       const state = yield* EvmSyncState.restoreState(
@@ -123,6 +127,7 @@ export function* genSyncProtocols(
         url: entry.network.rpcUrl,
         username: entry.network.rpcAuth?.username ?? null,
         password: entry.network.rpcAuth?.password ?? null,
+        requestTimeoutMs: requestTimeoutOf(entry.syncProtocol),
       });
       const fetcher = new BitcoinFetcher(entry, rpcClient);
       const state = yield* BitcoinSyncState.restoreState(
@@ -151,6 +156,16 @@ export function* genSyncProtocols(
         fetcher,
       );
       result.push(state);
+    } else if (
+      entry.networkType === ConfigNetworkType.SOLANA
+    ) {
+      const fetcher = new SolanaFetcher(entry);
+      const state = yield* SolanaSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
     } else if (
       entry.networkType === ConfigNetworkType.TEST
     ) {

package/test/poll-loop-spin.test.ts

@@ -0,0 +1,112 @@
+/**
+ * Regression guard for sync/CLAUDE.md Finding #3 (unpaced fetch loop).
+ *
+ * `orchestration/sync.ts` used to guard all three of its `sleep`s with
+ * `if ("pollingInterval" in config.syncProtocol)`, so a protocol whose config
+ * lacked that key never slept — not on error, and not in the ordinary "caught
+ * up" pass where `stateToInput` returns `undefined`.
+ *
+ * The Cardano/utxorpc schema was exactly that case: the only sync protocol that
+ * did not merge `PollingSyncProtocol`. Every config in this repo happened to
+ * pass `pollingInterval` anyway as an undeclared extra field, which is the only
+ * reason Cardano sync worked at all; a config built strictly from the schema
+ * produced a node that froze.
+ *
+ * "Froze", not "span": the probe showed the loop starving the event loop
+ * outright — a `setTimeout` scheduled for 500ms had still not fired after 4s of
+ * looping. Nothing else in the process runs in that state: not the HTTP server,
+ * not the other chains' fetch loops, and not the stream callbacks that would
+ * advance utxorpc's own tip and let the loop escape.
+ *
+ * Fixed on both sides: the utxorpc schema now merges `PollingSyncProtocol`, and
+ * `startSync` sleeps unconditionally with a fallback interval so no protocol
+ * can hot-loop regardless of what its schema declares. Both halves are asserted
+ * below — the second is what makes the first non-load-bearing.
+ */
+import { expect, test } from "bun:test";
+import { join } from "node:path";
+import {
+  ConfigSyncProtocolSchemaCardanoUtxoRpcParallel,
+  ConfigSyncProtocolSchemaEvmParallel,
+} from "@effectstream/config";
+
+const PROBE = join(import.meta.dir, "spin-probe.ts");
+
+/** Every property a protocol config may carry, required + optional. */
+function schemaKeys(schema: unknown): string[] {
+  const config = (schema as { config: { required: any; optional: any } }).config;
+  return [
+    ...Object.keys(config.required.properties ?? {}),
+    ...Object.keys(config.optional.properties ?? {}),
+  ];
+}
+
+type ProbeResult = {
+  reason: "limit" | "window";
+  iterations: number;
+  elapsedMs: number;
+};
+
+async function runProbe(spec: {
+  polling: number | null;
+  limit: number;
+  windowMs: number;
+  mode: "sync" | "call";
+}): Promise<ProbeResult> {
+  const proc = Bun.spawn(["bun", PROBE, JSON.stringify(spec)], {
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const stdout = await new Response(proc.stdout).text();
+  await proc.exited;
+  const line = stdout.trim().split("\n").filter(Boolean).at(-1);
+  if (!line) {
+    const stderr = await new Response(proc.stderr).text();
+    throw new Error(`probe produced no result\n${stderr}`);
+  }
+  return JSON.parse(line) as ProbeResult;
+}
+
+test("every polling protocol declares pollingInterval, utxorpc included", () => {
+  const utxorpc = schemaKeys(ConfigSyncProtocolSchemaCardanoUtxoRpcParallel);
+  const evm = schemaKeys(ConfigSyncProtocolSchemaEvmParallel);
+
+  // The runtime fetch loop keys its sleeps off this exact property name.
+  expect(utxorpc).toContain("pollingInterval");
+  expect(evm).toContain("pollingInterval");
+  // Merged in from PollingSyncProtocol along with it.
+  expect(utxorpc).toContain("requestTimeoutMs");
+});
+
+test("with a pollingInterval the fetch loop is paced", async () => {
+  const result = await runProbe({
+    polling: 20,
+    limit: 200_000,
+    windowMs: 500,
+    mode: "call",
+  });
+
+  // The window timer fired, so the event loop stayed responsive.
+  expect(result.reason).toBe("window");
+  // ~500ms / 20ms ≈ 25 passes. Generous bound; the point is it's tens, not tens
+  // of thousands.
+  expect(result.iterations).toBeLessThan(200);
+}, 30_000);
+
+test("without a pollingInterval the fetch loop still paces itself", async () => {
+  const WINDOW_MS = 1_500;
+  const result = await runProbe({
+    polling: null,
+    limit: 200_000,
+    // Comfortably longer than one FALLBACK_POLLING_INTERVAL_MS (1s) so the
+    // paced loop gets at least one pass in before the window closes.
+    windowMs: WINDOW_MS,
+    mode: "call",
+  });
+
+  // Before the fix this reported `reason: "limit"` with 200k iterations and a
+  // 500ms timer that never fired at all.
+  expect(result.reason).toBe("window");
+  // One pass per fallback interval — single digits over this window.
+  expect(result.iterations).toBeLessThan(10);
+}, 30_000);

package/test/rpc-timeout.test.ts

@@ -0,0 +1,150 @@
+/**
+ * Repro for sync/CLAUDE.md Finding #4 (missing RPC timeouts).
+ *
+ * Every per-chain client is pointed at a **blackhole**: a TCP server that
+ * completes the handshake and then never writes a byte. This is the realistic
+ * production failure (a load balancer that drops the backend, a half-open
+ * socket after a NAT rebind) — and the one plain `fetch` cannot see, because
+ * `fetch` has no default timeout in Bun/Node. The connection is *up*, so there
+ * is no ECONNREFUSED, no DNS error, nothing to catch.
+ *
+ * Consequence in production: `readData` never returns, so the fetch loop in
+ * `orchestration/sync.ts` never reaches its `catch` → `consecutiveErrors` stays
+ * 0, `lastSuccessfulFetchMs` freezes, and the merge blocks on that chain's page
+ * forever. The node stops producing blocks and reports nothing.
+ *
+ * Every client now bounds a single request with `AbortSignal.timeout` via
+ * `sync-protocols/common/http.ts:fetchWithTimeout`, configured per protocol by
+ * the optional `requestTimeoutMs` schema field. Retry is deliberately left to
+ * the fetch loop, which already re-runs the same page range and counts the
+ * failure in `consecutiveErrors` — retrying inside the client would hide the
+ * error from the health endpoint.
+ *
+ * These tests are the permanent regression guard: each client must REJECT
+ * against a blackhole, never hang.
+ */
+import { afterAll, beforeAll, expect, test } from "bun:test";
+import net from "node:net";
+import { BitcoinRpcClient } from "../src/sync-protocols/bitcoin/fetcher.ts";
+import { NearClient } from "../src/sync-protocols/near/NearClient.ts";
+import { SolanaClient } from "../src/sync-protocols/solana/SolanaClient.ts";
+import { AvailClient } from "../src/sync-protocols/avail/AvailClient.ts";
+import { MidnightClient } from "../src/sync-protocols/midnight/MidnightClient.ts";
+
+/** How long we watch a call before declaring it hung. */
+const OBSERVE_MS = 1_500;
+/** Timeout handed to clients that support one, kept well under OBSERVE_MS. */
+const CLIENT_TIMEOUT_MS = 300;
+
+let server: net.Server;
+let port: number;
+/** Held so the accepted sockets aren't garbage-collected mid-test. */
+const openSockets: net.Socket[] = [];
+
+beforeAll(async () => {
+  server = net.createServer((socket) => {
+    // Accept and go silent: the client sees a healthy connection and waits
+    // for a response that never comes.
+    openSockets.push(socket);
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  port = (server.address() as net.AddressInfo).port;
+});
+
+afterAll(async () => {
+  for (const socket of openSockets) socket.destroy();
+  await new Promise<void>((resolve) => server.close(() => resolve()));
+});
+
+type Settlement = "pending" | "resolved" | "rejected";
+
+/**
+ * Watch `p` for `ms` and report whether it settled. The promise is left with a
+ * no-op catch attached so a late rejection (after the test ends) can't surface
+ * as an unhandled rejection and fail an unrelated test.
+ */
+async function settlementWithin(
+  p: Promise<unknown>,
+  ms: number,
+): Promise<Settlement> {
+  const pending = Symbol("pending");
+  let outcome: Settlement = "pending";
+  const tracked = p.then(
+    () => {
+      outcome = "resolved";
+    },
+    () => {
+      outcome = "rejected";
+    },
+  );
+  const winner = await Promise.race([
+    tracked.then(() => "settled" as const),
+    new Promise<typeof pending>((resolve) => setTimeout(() => resolve(pending), ms)),
+  ]);
+  return winner === pending ? "pending" : outcome;
+}
+
+test("Bitcoin RPC rejects on a blackholed endpoint", async () => {
+  const client = new BitcoinRpcClient({
+    url: `http://127.0.0.1:${port}`,
+    username: null,
+    password: null,
+    requestTimeoutMs: CLIENT_TIMEOUT_MS,
+  });
+
+  const settlement = await settlementWithin(client.getBlockCount(), OBSERVE_MS);
+
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);
+
+test("NEAR RPC rejects on a blackholed endpoint", async () => {
+  const client = new NearClient(`http://127.0.0.1:${port}`, CLIENT_TIMEOUT_MS);
+
+  const settlement = await settlementWithin(
+    client.getBlock({ finality: "final" }),
+    OBSERVE_MS,
+  );
+
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);
+
+test("Solana RPC rejects on a blackholed endpoint", async () => {
+  const client = new SolanaClient(`http://127.0.0.1:${port}`, CLIENT_TIMEOUT_MS);
+
+  const settlement = await settlementWithin(client.getSlot(), OBSERVE_MS);
+
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);
+
+test("Avail light-client HTTP rejects on a blackholed endpoint", async () => {
+  // Built without the constructor on purpose: `new AvailClient(...)` eagerly
+  // calls `SDK.New(nodeUrl)`, whose websocket provider keeps reconnect timers
+  // alive and would outlive the test process. `getStatus()` only touches the
+  // light-client HTTP url, which is the code path under test.
+  const client: AvailClient = Object.create(AvailClient.prototype);
+  Object.assign(client, {
+    url: `http://127.0.0.1:${port}`,
+    requestTimeoutMs: CLIENT_TIMEOUT_MS,
+  });
+
+  const settlement = await settlementWithin(client.getStatus(), OBSERVE_MS);
+
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);
+
+test("CONTROL: Midnight rejects on a blackholed endpoint (has a request timeout)", async () => {
+  const client = new MidnightClient(
+    `http://127.0.0.1:${port}`,
+    undefined,
+    CLIENT_TIMEOUT_MS,
+  );
+
+  const settlement = await settlementWithin(
+    client.gqlQuery("query { __typename }"),
+    OBSERVE_MS,
+  );
+
+  // MidnightClient.gqlQuery() passes AbortSignal.timeout(requestTimeoutMs).
+  // This is the shape the three clients above are missing.
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);

package/test/spin-probe.ts

@@ -0,0 +1,102 @@
+/**
+ * Child process for `poll-loop-spin.test.ts` (sync/CLAUDE.md Finding #3).
+ *
+ * Drives the REAL `startSync` fetch loop with a fake `SyncState` that is always
+ * "caught up" (`stateToInput` → `undefined`) — the steady state of any protocol
+ * whose tip advances slowly. The only variable is whether the protocol config
+ * carries a `pollingInterval`, because `orchestration/sync.ts` guards all three
+ * of its `sleep`s with `if ("pollingInterval" in config.syncProtocol)`.
+ *
+ * Runs in its own process because a loop with no sleep can starve the event
+ * loop outright, which would hang the test runner rather than fail it. The two
+ * exit paths below discriminate the outcomes:
+ *
+ *   - `reason: "limit"`  — the loop reached `limit` iterations. Reported via
+ *                          `writeSync` + `process.exit`, which work even when
+ *                          the event loop is starved.
+ *   - `reason: "window"` — a `setTimeout` fired first, so the loop is paced and
+ *                          the event loop is healthy.
+ *
+ * Usage: bun spin-probe.ts '{"polling":20,"limit":50000,"windowMs":500,"mode":"call"}'
+ */
+import { writeSync } from "node:fs";
+import { call, type Operation, run } from "effection";
+import { startSync } from "../src/sync-protocols/orchestration/sync.ts";
+
+type ProbeSpec = {
+  /** `pollingInterval` for the fake protocol config; `null` omits the key entirely. */
+  polling: number | null;
+  /** Iteration count at which we declare the loop unpaced and bail out. */
+  limit: number;
+  /** Wall-clock budget; if this timer fires, the loop is paced. */
+  windowMs: number;
+  /**
+   * How `stateToInput` returns:
+   *  - "sync" — returns `undefined` with no suspension at all (worst case).
+   *  - "call" — `yield* call(() => <sync value>)` first, mirroring utxorpc's
+   *             `stateToInput`, which calls the in-memory `fetcher.lastHeight()`
+   *             before deciding it is caught up. This is the production shape.
+   */
+  mode: "sync" | "call";
+};
+
+const spec: ProbeSpec = JSON.parse(process.argv[2]);
+
+const startedAt = Date.now();
+let iterations = 0;
+
+function report(reason: "limit" | "window"): never {
+  writeSync(
+    1,
+    JSON.stringify({
+      reason,
+      iterations,
+      elapsedMs: Date.now() - startedAt,
+    }) + "\n",
+  );
+  process.exit(0);
+}
+
+// Fires only if the loop leaves room for macrotasks.
+setTimeout(() => report("window"), spec.windowMs);
+
+const syncProtocol: Record<string, unknown> = { name: "probe" };
+if (spec.polling != null) syncProtocol.pollingInterval = spec.polling;
+
+/**
+ * Minimal stand-in for a `SyncState`. Only the members `startSync` actually
+ * touches are implemented; it is cast at the call site the same way the real
+ * states are (`startSync` takes `AllSyncProtocols` and narrows to `ISyncProtocol`).
+ */
+const fakeState = {
+  name: "probe",
+  consecutiveErrors: 0,
+  lastErrorTimestamp: 0,
+  lastSuccessfulFetchMs: 0,
+  getNamespace: () => ["probe", "probe"],
+  *startAsync(): Operation<void> {},
+  *stateToInput(): Operation<undefined> {
+    iterations++;
+    if (iterations >= spec.limit) report("limit");
+    if (spec.mode === "call") {
+      // utxorpc reads an in-memory tip here — a `call` over a synchronous value.
+      yield* call(() => 0);
+    }
+    // "caught up": nothing to fetch this pass.
+    return undefined;
+  },
+  fetcher: {
+    config: { syncProtocol },
+    producerChannel: { *send(): Operation<void> {} },
+    *readData(): Operation<never> {
+      throw new Error("unreachable: stateToInput always returns undefined");
+    },
+  },
+  *updateState(): Operation<void> {},
+};
+
+await run(function* () {
+  yield* startSync(fakeState as never);
+  // Park forever; one of the two exit paths above ends the process.
+  yield* call(() => new Promise<void>(() => {}));
+});

package/test/start-async-supervision.test.ts

@@ -0,0 +1,139 @@
+/**
+ * Regression guard for sync/CLAUDE.md Finding #5 (unsupervised `startAsync`).
+ *
+ * `orchestration/sync.ts` used to start the streaming producer with a bare
+ * `yield* spawn(function* () { yield* iState.startAsync(); })` — no try/catch,
+ * no restart, no liveness check. Streaming chains (utxorpc today, midnight if
+ * it moves to subscriptions) put ALL of their data on that path, so both of its
+ * exit modes were unhandled:
+ *
+ *   5a — the stream ENDS CLEANLY. `WatchMultiplexer.runWatcher` logs
+ *        "stream ended unexpectedly" and returns; the spawned task completed
+ *        normally. The chain then received nothing forever while every health
+ *        counter stayed clean, and the merge blocked on its page — a total stall
+ *        with nothing reporting an error.
+ *
+ *   5b — the stream THROWS. The error escaped the spawned task and tore down the
+ *        enclosing scope, which in production is `start()`: one flaky gRPC
+ *        connection took down every unrelated chain with it.
+ *
+ * Now supervised — both modes are treated as failure and restarted with capped
+ * exponential backoff, counted in `producerRestarts`. A returning producer
+ * counts as failure because the observable effect is identical: no more data.
+ *
+ * Supervision is gated on `hasAsyncProducer` so polled chains, whose
+ * `startAsync` is the base-class no-op, are still run exactly once (5c).
+ */
+import { expect, test } from "bun:test";
+import { type Operation, run, sleep } from "effection";
+import { startSync } from "../src/sync-protocols/orchestration/sync.ts";
+
+const POLL_MS = 20;
+
+/**
+ * Minimal stand-in for a streaming `SyncState`. Only the members `startSync`
+ * touches are implemented. `stateToInput` always reports "caught up", which is
+ * what a streaming chain does while it waits for its producer to deliver.
+ */
+function makeStreamingState(
+  startAsyncBody: () => Operation<void>,
+  hasAsyncProducer = true,
+) {
+  const counters = { startAsyncCalls: 0, pollPasses: 0 };
+  const state = {
+    name: "streamer",
+    consecutiveErrors: 0,
+    lastErrorTimestamp: 0,
+    lastSuccessfulFetchMs: 0,
+    hasAsyncProducer,
+    producerRestarts: 0,
+    producerErrors: 0,
+    lastProducerErrorMs: 0,
+    pollingIntervalMs: POLL_MS,
+    getNamespace: () => ["streamer", "streamer"],
+    *startAsync(): Operation<void> {
+      counters.startAsyncCalls++;
+      yield* startAsyncBody();
+    },
+    *stateToInput(): Operation<undefined> {
+      counters.pollPasses++;
+      return undefined;
+    },
+    fetcher: {
+      config: { syncProtocol: { name: "streamer", pollingInterval: POLL_MS } },
+      producerChannel: { *send(): Operation<void> {} },
+      *readData(): Operation<never> {
+        throw new Error("unreachable: stateToInput always returns undefined");
+      },
+    },
+    *updateState(): Operation<void> {},
+  };
+  return { state, counters };
+}
+
+test("5a: a producer that ends cleanly is restarted and counted", async () => {
+  // The multiplexer's `for await (const event of stream)` falling through —
+  // the stream closed and `runWatcher` returned.
+  const { state, counters } = makeStreamingState(function* () {
+    // returns immediately
+  });
+
+  await run(function* () {
+    yield* startSync(state as never);
+    // Long enough for the first restart (1s backoff) plus room to spare.
+    yield* sleep(1_500);
+  });
+
+  // Restarted rather than abandoned...
+  expect(counters.startAsyncCalls).toBeGreaterThan(1);
+  // ...and the restart is visible to health instead of being silent.
+  expect(state.producerRestarts).toBeGreaterThan(0);
+  // A clean return is not an *error*, so only the restart counter moves.
+  expect(state.producerErrors).toBe(0);
+  // The sibling polling loop is unaffected.
+  expect(counters.pollPasses).toBeGreaterThan(1);
+}, 30_000);
+
+test("5b: a producer that throws is restarted and does not kill the scope", async () => {
+  const { state, counters } = makeStreamingState(function* () {
+    yield* sleep(50);
+    throw new Error("gRPC stream died");
+  });
+
+  let rejection: unknown;
+  await run(function* () {
+    yield* startSync(state as never);
+    yield* sleep(1_500);
+  }).catch((err) => {
+    rejection = err;
+  });
+
+  // The scope survived: no error escaped to tear down `start()`.
+  expect(rejection).toBeUndefined();
+  // The throw was caught, counted, and the producer restarted.
+  expect(counters.startAsyncCalls).toBeGreaterThan(1);
+  expect(state.producerRestarts).toBeGreaterThan(0);
+  expect(state.producerErrors).toBeGreaterThan(0);
+  expect(state.lastProducerErrorMs).toBeGreaterThan(0);
+  // Counted on the PRODUCER's own tally, not the fetch loop's. Sharing
+  // `consecutiveErrors` conflated two signals: only a successful `readData`
+  // clears it, so an idle streaming chain stayed `erroring` long after its
+  // producer recovered, and a successful poll erased the record of a flap.
+  expect(state.consecutiveErrors).toBe(0);
+  // And the sibling polling loop kept running throughout.
+  expect(counters.pollPasses).toBeGreaterThan(1);
+}, 30_000);
+
+test("5c: a polled chain's no-op producer is run once, not supervised", async () => {
+  const { state, counters } = makeStreamingState(function* () {}, false);
+
+  await run(function* () {
+    yield* startSync(state as never);
+    yield* sleep(1_500);
+  });
+
+  // Without this gate, every polled chain would "restart" its no-op forever.
+  expect(counters.startAsyncCalls).toBe(1);
+  expect(state.producerRestarts).toBe(0);
+  expect(counters.pollPasses).toBeGreaterThan(1);
+}, 30_000);