@effectstream/sync 0.101.1 → 0.103.0

package/src/sync-protocols/solana/state.ts

@@ -0,0 +1,167 @@
+import { call, type Operation } from "effection";
+import { bound } from "@effectstream/utils";
+import type { PoolClient } from "pg";
+import { type LastPage, SyncState } from "../base/state.ts";
+import type { RootOutput, RootPage } from "../types.ts";
+import type { Input, Output, Page } from "./types.ts";
+import { toMsTimestamp } from "./types.ts";
+import { blockNumberRelation } from "../common/utils.ts";
+import type { SolanaFetcher } from "./fetcher.ts";
+import type {
+  ConfigNetworkType,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+import { getPage } from "@effectstream/db";
+import { SolanaClient } from "./SolanaClient.ts";
+import { applyDelay } from "../common/utils.ts";
+import { bufferAtCap } from "../common/page-helpers.ts";
+import { requestTimeoutOf } from "../common/http.ts";
+
+export class SolanaSyncState extends SyncState<
+  Input,
+  Output,
+  Page,
+  RootOutput,
+  RootPage,
+  SolanaFetcher
+> {
+  constructor(
+    lastPage: LastPage<Page, RootPage> | undefined,
+    readonly config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.SOLANA }
+    >,
+    fetcher: SolanaFetcher,
+    public readonly client: SolanaClient,
+    dbConn: PoolClient,
+  ) {
+    super(
+      config.syncProtocol.name,
+      lastPage,
+      fetcher,
+      blockNumberRelation,
+      dbConn,
+    );
+  }
+
+  @bound
+  override toPage(_input: Input, data: Output[]): Page {
+    const lastBlock = data[data.length - 1];
+    return lastBlock.slot as Page;
+  }
+
+  @bound
+  override toRootPage(data: Output): RootPage {
+    // Solana blockTime is monotonically non-decreasing (enforced by the
+    // cluster) — exactly what the merge gate needs: it pulls buffered parallel
+    // outputs in slot order while their root timestamp is <= the main chain's,
+    // so it disambiguates duplicate blockTimes by buffer/slot order, NOT by
+    // this value. We therefore must NOT add a slot-derived offset: the previous
+    // `(slot % 1000) * 0.001` was unnecessary AND broke monotonicity (it wrapped
+    // every 1000 slots, so slot 1000's root could sort before slot 999's).
+    return applyDelay(
+      toMsTimestamp(data.blockTime),
+      this.config.syncProtocol.delayMs,
+    );
+  }
+
+  @bound
+  override toRootOutput(_data: Output): RootOutput {
+    throw new Error("Only main chains create root outputs");
+  }
+
+  @bound
+  override *stateToInput(): Operation<Input | undefined> {
+    // Pause fetching while the merge drains our buffer (CLAUDE.md finding #1).
+    // Every other protocol gates on this first; skipping it lets the Deque grow
+    // toward the whole backlog during catch-up.
+    if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
+    // Query the chain tip directly (like NearSyncState) rather than via
+    // genInputRange, which requires the fetcher to implement PaginatedFetcher
+    // (getLatestPage/nextInterval/...) — SolanaFetcher does not.
+    const latestSlot = yield* call(() => this.client.getSlot());
+    const finalizedSlot = latestSlot - this.config.syncProtocol.confirmationDepth;
+
+    const lastSlot = this.lastPage?.own
+      ?? ((this.config.syncProtocol.startBlockHeight as Page) - 1);
+
+    if (lastSlot >= finalizedSlot) {
+      return undefined;
+    }
+
+    const from = (lastSlot + 1) as Page;
+    const to = Math.min(
+      Number(lastSlot) + this.config.syncProtocol.stepSize,
+      finalizedSlot,
+    ) as Page;
+
+    return {
+      from,
+      to,
+      isPresync: false,
+    };
+  }
+
+  @bound
+  override mergeDatum(ourOutput: Output, rootOutput: RootOutput): void {
+    const primitives = ourOutput.primitives.map((p) => ({
+      ...p,
+      source: this.config.syncProtocol.name,
+    }));
+    const blockInfo = [{
+      protocol_name: this.config.syncProtocol.name,
+      block_number: ourOutput.slot,
+      blockHash: ourOutput.blockhash,
+    }];
+    rootOutput.blockInfo.push(...blockInfo);
+    rootOutput.primitives.push(...primitives);
+  }
+
+  /**
+   * Resume marker for a single slot. `Page` is a flat slot number, so `own` and
+   * `ownBlockNumber` are the same value (cf. EvmSyncState). Declared `abstract`
+   * on SyncState and called unconditionally by the merge for every protocol that
+   * contributes data to a block — see CLAUDE.md design idea #5.
+   */
+  @bound
+  override outputToLastPage(data: Output): LastPage<Page, RootPage> {
+    return {
+      own: data.slot as Page,
+      ownBlockNumber: data.slot as Page,
+      root: this.toRootPage(data),
+    };
+  }
+
+  @bound
+  override getNamespace(): string[] {
+    return [this.config.network.name, this.config.syncProtocol.name];
+  }
+
+  static *restoreState(
+    dbConn: PoolClient,
+    config: Extract<
+      SyncProtocolWithNetwork,
+      { networkType: ConfigNetworkType.SOLANA }
+    >,
+    fetcher: SolanaFetcher,
+  ): Operation<SolanaSyncState> {
+    const [result] = yield* call(async () =>
+      await getPage.run({
+        protocol_name: config.syncProtocol.name,
+      }, dbConn)
+    );
+    const page = result
+      ? result.page as unknown as LastPage<Page, RootPage>
+      : undefined;
+    return new SolanaSyncState(
+      page,
+      config,
+      fetcher,
+      new SolanaClient(
+        config.network.rpcUrl,
+        requestTimeoutOf(config.syncProtocol),
+      ),
+      dbConn,
+    );
+  }
+}

package/src/sync-protocols/solana/types.ts

@@ -0,0 +1,58 @@
+import type {
+  BlockNumber,
+  TimestampMs,
+} from "@effectstream/utils";
+import type { PageSyncRange } from "../common/page-helpers.ts";
+import type {
+  ConfigNetworkType,
+  ConfigSyncProtocolType,
+  FlattenSyncProtocolIOFor,
+  PrimitiveEntry,
+  SyncProtocolWithNetwork,
+} from "@effectstream/config";
+
+/** Solana page type: a slot number (block height) */
+export type Page = BlockNumber;
+
+export type ConfigType = Extract<
+  SyncProtocolWithNetwork,
+  { networkType: ConfigNetworkType.SOLANA }
+>;
+
+export type PrimitiveEntryType = Extract<
+  PrimitiveEntry,
+  { syncProtocol: ConfigSyncProtocolType.SOLANA_RPC_PARALLEL }
+>;
+
+export type PrimitiveType = FlattenSyncProtocolIOFor<
+  ConfigSyncProtocolType.SOLANA_RPC_PARALLEL
+>;
+
+export type Input = PageSyncRange<Page>;
+
+export type SolanaTransactionMeta = {
+  err: unknown | null;
+  logMessages: string[] | null;
+  preBalances: number[];
+  postBalances: number[];
+};
+
+export type Output = {
+  slot: number;
+  blockhash: string;
+  /**
+   * Unix seconds. Non-null by construction: the fetcher resolves the RPC's
+   * nullable `blockTime` before building an Output, because a block with no
+   * timestamp cannot be placed in the time-ordered merge.
+   */
+  blockTime: number;
+  blockHeight: number | null;
+  parentSlot: number;
+  transactions: SolanaTransactionMeta[];
+  primitives: PrimitiveType[];
+};
+
+/** Convert a Solana blockTime (unix seconds) to milliseconds. */
+export function toMsTimestamp(blockTime: number): TimestampMs {
+  return blockTime * 1000 as TimestampMs;
+}

package/src/sync-protocols/types.ts

@@ -10,6 +10,7 @@ import type { AvailSyncState } from "./avail/state.ts";
 import type { BitcoinSyncState } from "./bitcoin/state.ts";
 import type { CelestiaSyncState } from "./celestia/state.ts";
 import type { NearSyncState } from "./near/state.ts";
+import type { SolanaSyncState } from "./solana/state.ts";
 import type { TestSyncState } from "./test/state.ts";
 
 // TODO: move folders
@@ -26,6 +27,7 @@ export type AllSyncProtocols =
   | BitcoinSyncState
   | CelestiaSyncState
   | NearSyncState
+  | SolanaSyncState
   | TestSyncState;
 export type ISyncProtocol = UnionToIntersection<AllSyncProtocols>;
 

package/src/sync-protocols/utxorpc/WatchMultiplexer.ts

@@ -31,6 +31,21 @@ export class WatchMultiplexer {
   private highestConfirmedHeight: number | undefined = undefined;
   private lowestIncompleteHeight: number = 0;
   private slotOriginTimestamp: bigint | undefined = undefined;
+  /**
+   * Incremented by every {@link start} call so events from a previous
+   * generation's streams can be identified and dropped.
+   *
+   * `start` can now be called more than once: `startSync` supervises the
+   * producer and restarts it when the stream dies. All multiplexer state lives
+   * on this instance, and the gRPC streams of the previous generation are not
+   * cancellable from here — a `Promise.all` rejection from ONE watcher aborts
+   * `start` while its siblings keep streaming. Without this guard a restart
+   * would run two live streams per predicate into the same `pendingBlocks`,
+   * pushing each matched tx twice. Duplicate primitives in a block mean
+   * duplicate scheduled STF inputs, i.e. non-deterministic state — the exact
+   * failure the whole pipeline is built to avoid.
+   */
+  private generation = 0;
 
   constructor(
     private readonly clientOptions: ClientBuilderOptions,
@@ -88,14 +103,39 @@ export class WatchMultiplexer {
       }
     }
 
+    const generation = ++this.generation;
+
+    // Drop half-assembled work from the previous generation. A block is only
+    // completed once EVERY watcher has passed it, so at the moment a stream
+    // died some heights had been seen by some watchers and not others. The new
+    // stream re-delivers those heights from the resume point, and merging the
+    // two views would double their txs. Heights below `lowestIncompleteHeight`
+    // are already completed and are never revisited, so this discards exactly
+    // the in-flight window.
+    if (generation > 1) {
+      for (const [, watcher] of this.watchers) {
+        for (const [height] of watcher.pendingBlocks) {
+          if (height >= this.lowestIncompleteHeight) {
+            watcher.pendingBlocks.delete(height);
+          }
+        }
+        watcher.highestSeenHeight = Math.max(
+          this.lowestIncompleteHeight - 1,
+          0,
+        );
+      }
+    }
+
     const intersect = point ? [point] : [];
     const promises: Promise<void>[] = [];
     for (const [key, watcher] of this.watchers) {
       const watchClient = new CardanoWatchClient(this.clientOptions);
-      promises.push(this.runWatcher(key, watcher, watchClient, intersect));
+      promises.push(this.runWatcher(key, watcher, watchClient, intersect, generation));
     }
 
-    console.log(`[WatchMultiplexer] Started ${this.watchers.size} watcher(s)`);
+    console.log(
+      `[WatchMultiplexer] Started ${this.watchers.size} watcher(s) (generation ${generation})`,
+    );
     await Promise.all(promises);
   }
 
@@ -104,9 +144,19 @@ export class WatchMultiplexer {
     watcher: WatcherState,
     watchClient: CardanoWatchClient,
     intersect: ChainPoint[],
+    generation: number,
   ): Promise<void> {
     const stream = watchClient.watchTxByPredicate(watcher.sdkPredicate, intersect);
     for await (const event of stream) {
+      // A superseded generation must contribute nothing. Breaking here also
+      // closes the underlying stream via the async iterator's return path,
+      // which is the only way to stop it from this side.
+      if (generation !== this.generation) {
+        console.warn(
+          `[WatchMultiplexer] Watcher "${key}" from generation ${generation} superseded by ${this.generation}; closing`,
+        );
+        return;
+      }
       this.handleEvent(key, watcher, event);
     }
     console.warn(`[WatchMultiplexer] Watcher "${key}" stream ended unexpectedly`);

package/src/sync-protocols/utxorpc/state.ts

@@ -35,6 +35,9 @@ export class UtxoRpcSyncState extends SyncState<
     );
   }
 
+  /** All utxorpc data arrives via the watch stream in {@link startAsync}. */
+  public override readonly hasAsyncProducer = true;
+
   @bound
   override *startAsync(): Operation<void> {
     if (this.lastPage == null) {

package/src/syncProtocolFactory.ts

@@ -31,8 +31,11 @@ import { CelestiaFetcher } from "./sync-protocols/celestia/fetcher.ts";
 import { CelestiaSyncState } from "./sync-protocols/celestia/state.ts";
 import { NearFetcher } from "./sync-protocols/near/fetcher.ts";
 import { NearSyncState } from "./sync-protocols/near/state.ts";
+import { SolanaFetcher } from "./sync-protocols/solana/fetcher.ts";
+import { SolanaSyncState } from "./sync-protocols/solana/state.ts";
 import { TestFetcher } from "./sync-protocols/test/fetcher.ts";
 import { TestSyncState } from "./sync-protocols/test/state.ts";
+import { requestTimeoutOf } from "./sync-protocols/common/http.ts";
 
 export function* genSyncProtocols(
   dbConn: PoolClient,
@@ -56,6 +59,7 @@ export function* genSyncProtocols(
         entry,
         createViemPublicClient(viemNetwork, {
           cacheTime: entry.syncProtocol.pollingInterval,
+          timeout: requestTimeoutOf(entry.syncProtocol),
         }),
       );
       const state = yield* EvmSyncState.restoreState(
@@ -123,6 +127,7 @@ export function* genSyncProtocols(
         url: entry.network.rpcUrl,
         username: entry.network.rpcAuth?.username ?? null,
         password: entry.network.rpcAuth?.password ?? null,
+        requestTimeoutMs: requestTimeoutOf(entry.syncProtocol),
       });
       const fetcher = new BitcoinFetcher(entry, rpcClient);
       const state = yield* BitcoinSyncState.restoreState(
@@ -151,6 +156,16 @@ export function* genSyncProtocols(
         fetcher,
       );
       result.push(state);
+    } else if (
+      entry.networkType === ConfigNetworkType.SOLANA
+    ) {
+      const fetcher = new SolanaFetcher(entry);
+      const state = yield* SolanaSyncState.restoreState(
+        dbConn,
+        entry,
+        fetcher,
+      );
+      result.push(state);
     } else if (
       entry.networkType === ConfigNetworkType.TEST
     ) {

package/test/poll-loop-spin.test.ts

@@ -0,0 +1,112 @@
+/**
+ * Regression guard for sync/CLAUDE.md Finding #3 (unpaced fetch loop).
+ *
+ * `orchestration/sync.ts` used to guard all three of its `sleep`s with
+ * `if ("pollingInterval" in config.syncProtocol)`, so a protocol whose config
+ * lacked that key never slept — not on error, and not in the ordinary "caught
+ * up" pass where `stateToInput` returns `undefined`.
+ *
+ * The Cardano/utxorpc schema was exactly that case: the only sync protocol that
+ * did not merge `PollingSyncProtocol`. Every config in this repo happened to
+ * pass `pollingInterval` anyway as an undeclared extra field, which is the only
+ * reason Cardano sync worked at all; a config built strictly from the schema
+ * produced a node that froze.
+ *
+ * "Froze", not "span": the probe showed the loop starving the event loop
+ * outright — a `setTimeout` scheduled for 500ms had still not fired after 4s of
+ * looping. Nothing else in the process runs in that state: not the HTTP server,
+ * not the other chains' fetch loops, and not the stream callbacks that would
+ * advance utxorpc's own tip and let the loop escape.
+ *
+ * Fixed on both sides: the utxorpc schema now merges `PollingSyncProtocol`, and
+ * `startSync` sleeps unconditionally with a fallback interval so no protocol
+ * can hot-loop regardless of what its schema declares. Both halves are asserted
+ * below — the second is what makes the first non-load-bearing.
+ */
+import { expect, test } from "bun:test";
+import { join } from "node:path";
+import {
+  ConfigSyncProtocolSchemaCardanoUtxoRpcParallel,
+  ConfigSyncProtocolSchemaEvmParallel,
+} from "@effectstream/config";
+
+const PROBE = join(import.meta.dir, "spin-probe.ts");
+
+/** Every property a protocol config may carry, required + optional. */
+function schemaKeys(schema: unknown): string[] {
+  const config = (schema as { config: { required: any; optional: any } }).config;
+  return [
+    ...Object.keys(config.required.properties ?? {}),
+    ...Object.keys(config.optional.properties ?? {}),
+  ];
+}
+
+type ProbeResult = {
+  reason: "limit" | "window";
+  iterations: number;
+  elapsedMs: number;
+};
+
+async function runProbe(spec: {
+  polling: number | null;
+  limit: number;
+  windowMs: number;
+  mode: "sync" | "call";
+}): Promise<ProbeResult> {
+  const proc = Bun.spawn(["bun", PROBE, JSON.stringify(spec)], {
+    stdout: "pipe",
+    stderr: "pipe",
+  });
+  const stdout = await new Response(proc.stdout).text();
+  await proc.exited;
+  const line = stdout.trim().split("\n").filter(Boolean).at(-1);
+  if (!line) {
+    const stderr = await new Response(proc.stderr).text();
+    throw new Error(`probe produced no result\n${stderr}`);
+  }
+  return JSON.parse(line) as ProbeResult;
+}
+
+test("every polling protocol declares pollingInterval, utxorpc included", () => {
+  const utxorpc = schemaKeys(ConfigSyncProtocolSchemaCardanoUtxoRpcParallel);
+  const evm = schemaKeys(ConfigSyncProtocolSchemaEvmParallel);
+
+  // The runtime fetch loop keys its sleeps off this exact property name.
+  expect(utxorpc).toContain("pollingInterval");
+  expect(evm).toContain("pollingInterval");
+  // Merged in from PollingSyncProtocol along with it.
+  expect(utxorpc).toContain("requestTimeoutMs");
+});
+
+test("with a pollingInterval the fetch loop is paced", async () => {
+  const result = await runProbe({
+    polling: 20,
+    limit: 200_000,
+    windowMs: 500,
+    mode: "call",
+  });
+
+  // The window timer fired, so the event loop stayed responsive.
+  expect(result.reason).toBe("window");
+  // ~500ms / 20ms ≈ 25 passes. Generous bound; the point is it's tens, not tens
+  // of thousands.
+  expect(result.iterations).toBeLessThan(200);
+}, 30_000);
+
+test("without a pollingInterval the fetch loop still paces itself", async () => {
+  const WINDOW_MS = 1_500;
+  const result = await runProbe({
+    polling: null,
+    limit: 200_000,
+    // Comfortably longer than one FALLBACK_POLLING_INTERVAL_MS (1s) so the
+    // paced loop gets at least one pass in before the window closes.
+    windowMs: WINDOW_MS,
+    mode: "call",
+  });
+
+  // Before the fix this reported `reason: "limit"` with 200k iterations and a
+  // 500ms timer that never fired at all.
+  expect(result.reason).toBe("window");
+  // One pass per fallback interval — single digits over this window.
+  expect(result.iterations).toBeLessThan(10);
+}, 30_000);

package/test/rpc-timeout.test.ts

@@ -0,0 +1,150 @@
+/**
+ * Repro for sync/CLAUDE.md Finding #4 (missing RPC timeouts).
+ *
+ * Every per-chain client is pointed at a **blackhole**: a TCP server that
+ * completes the handshake and then never writes a byte. This is the realistic
+ * production failure (a load balancer that drops the backend, a half-open
+ * socket after a NAT rebind) — and the one plain `fetch` cannot see, because
+ * `fetch` has no default timeout in Bun/Node. The connection is *up*, so there
+ * is no ECONNREFUSED, no DNS error, nothing to catch.
+ *
+ * Consequence in production: `readData` never returns, so the fetch loop in
+ * `orchestration/sync.ts` never reaches its `catch` → `consecutiveErrors` stays
+ * 0, `lastSuccessfulFetchMs` freezes, and the merge blocks on that chain's page
+ * forever. The node stops producing blocks and reports nothing.
+ *
+ * Every client now bounds a single request with `AbortSignal.timeout` via
+ * `sync-protocols/common/http.ts:fetchWithTimeout`, configured per protocol by
+ * the optional `requestTimeoutMs` schema field. Retry is deliberately left to
+ * the fetch loop, which already re-runs the same page range and counts the
+ * failure in `consecutiveErrors` — retrying inside the client would hide the
+ * error from the health endpoint.
+ *
+ * These tests are the permanent regression guard: each client must REJECT
+ * against a blackhole, never hang.
+ */
+import { afterAll, beforeAll, expect, test } from "bun:test";
+import net from "node:net";
+import { BitcoinRpcClient } from "../src/sync-protocols/bitcoin/fetcher.ts";
+import { NearClient } from "../src/sync-protocols/near/NearClient.ts";
+import { SolanaClient } from "../src/sync-protocols/solana/SolanaClient.ts";
+import { AvailClient } from "../src/sync-protocols/avail/AvailClient.ts";
+import { MidnightClient } from "../src/sync-protocols/midnight/MidnightClient.ts";
+
+/** How long we watch a call before declaring it hung. */
+const OBSERVE_MS = 1_500;
+/** Timeout handed to clients that support one, kept well under OBSERVE_MS. */
+const CLIENT_TIMEOUT_MS = 300;
+
+let server: net.Server;
+let port: number;
+/** Held so the accepted sockets aren't garbage-collected mid-test. */
+const openSockets: net.Socket[] = [];
+
+beforeAll(async () => {
+  server = net.createServer((socket) => {
+    // Accept and go silent: the client sees a healthy connection and waits
+    // for a response that never comes.
+    openSockets.push(socket);
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  port = (server.address() as net.AddressInfo).port;
+});
+
+afterAll(async () => {
+  for (const socket of openSockets) socket.destroy();
+  await new Promise<void>((resolve) => server.close(() => resolve()));
+});
+
+type Settlement = "pending" | "resolved" | "rejected";
+
+/**
+ * Watch `p` for `ms` and report whether it settled. The promise is left with a
+ * no-op catch attached so a late rejection (after the test ends) can't surface
+ * as an unhandled rejection and fail an unrelated test.
+ */
+async function settlementWithin(
+  p: Promise<unknown>,
+  ms: number,
+): Promise<Settlement> {
+  const pending = Symbol("pending");
+  let outcome: Settlement = "pending";
+  const tracked = p.then(
+    () => {
+      outcome = "resolved";
+    },
+    () => {
+      outcome = "rejected";
+    },
+  );
+  const winner = await Promise.race([
+    tracked.then(() => "settled" as const),
+    new Promise<typeof pending>((resolve) => setTimeout(() => resolve(pending), ms)),
+  ]);
+  return winner === pending ? "pending" : outcome;
+}
+
+test("Bitcoin RPC rejects on a blackholed endpoint", async () => {
+  const client = new BitcoinRpcClient({
+    url: `http://127.0.0.1:${port}`,
+    username: null,
+    password: null,
+    requestTimeoutMs: CLIENT_TIMEOUT_MS,
+  });
+
+  const settlement = await settlementWithin(client.getBlockCount(), OBSERVE_MS);
+
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);
+
+test("NEAR RPC rejects on a blackholed endpoint", async () => {
+  const client = new NearClient(`http://127.0.0.1:${port}`, CLIENT_TIMEOUT_MS);
+
+  const settlement = await settlementWithin(
+    client.getBlock({ finality: "final" }),
+    OBSERVE_MS,
+  );
+
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);
+
+test("Solana RPC rejects on a blackholed endpoint", async () => {
+  const client = new SolanaClient(`http://127.0.0.1:${port}`, CLIENT_TIMEOUT_MS);
+
+  const settlement = await settlementWithin(client.getSlot(), OBSERVE_MS);
+
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);
+
+test("Avail light-client HTTP rejects on a blackholed endpoint", async () => {
+  // Built without the constructor on purpose: `new AvailClient(...)` eagerly
+  // calls `SDK.New(nodeUrl)`, whose websocket provider keeps reconnect timers
+  // alive and would outlive the test process. `getStatus()` only touches the
+  // light-client HTTP url, which is the code path under test.
+  const client: AvailClient = Object.create(AvailClient.prototype);
+  Object.assign(client, {
+    url: `http://127.0.0.1:${port}`,
+    requestTimeoutMs: CLIENT_TIMEOUT_MS,
+  });
+
+  const settlement = await settlementWithin(client.getStatus(), OBSERVE_MS);
+
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);
+
+test("CONTROL: Midnight rejects on a blackholed endpoint (has a request timeout)", async () => {
+  const client = new MidnightClient(
+    `http://127.0.0.1:${port}`,
+    undefined,
+    CLIENT_TIMEOUT_MS,
+  );
+
+  const settlement = await settlementWithin(
+    client.gqlQuery("query { __typename }"),
+    OBSERVE_MS,
+  );
+
+  // MidnightClient.gqlQuery() passes AbortSignal.timeout(requestTimeoutMs).
+  // This is the shape the three clients above are missing.
+  expect(settlement).toBe("rejected");
+}, OBSERVE_MS + 5_000);