@effectstream/sync 0.101.1 → 0.102.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/CLAUDE.md +82 -18
  2. package/LICENSE-APACHE +201 -0
  3. package/LICENSE-MIT +21 -0
  4. package/README.md +63 -4
  5. package/docs/ADDING-A-SYNC-PROTOCOL.md +16 -9
  6. package/package.json +8 -8
  7. package/src/sync-protocols/avail/AvailClient.ts +34 -6
  8. package/src/sync-protocols/avail/fetcher.ts +2 -0
  9. package/src/sync-protocols/avail/state.ts +2 -0
  10. package/src/sync-protocols/base/state.ts +51 -0
  11. package/src/sync-protocols/bitcoin/fetcher.ts +18 -10
  12. package/src/sync-protocols/celestia/CelestiaClient.ts +16 -2
  13. package/src/sync-protocols/celestia/fetcher.ts +5 -1
  14. package/src/sync-protocols/celestia/state.ts +5 -1
  15. package/src/sync-protocols/common/http.ts +77 -0
  16. package/src/sync-protocols/midnight/fetcher.ts +2 -0
  17. package/src/sync-protocols/midnight/types.ts +1 -1
  18. package/src/sync-protocols/near/NearClient.ts +16 -6
  19. package/src/sync-protocols/near/fetcher.ts +5 -1
  20. package/src/sync-protocols/near/state.ts +5 -1
  21. package/src/sync-protocols/orchestration/sync.ts +101 -13
  22. package/src/sync-protocols/solana/SolanaClient.ts +158 -0
  23. package/src/sync-protocols/solana/fetcher.ts +358 -0
  24. package/src/sync-protocols/solana/program-logs.test.ts +113 -0
  25. package/src/sync-protocols/solana/program-logs.ts +84 -0
  26. package/src/sync-protocols/solana/read-data-retry.test.ts +153 -0
  27. package/src/sync-protocols/solana/read-primitives.test.ts +188 -0
  28. package/src/sync-protocols/solana/root-page.test.ts +66 -0
  29. package/src/sync-protocols/solana/state.ts +167 -0
  30. package/src/sync-protocols/solana/types.ts +58 -0
  31. package/src/sync-protocols/types.ts +2 -0
  32. package/src/sync-protocols/utxorpc/WatchMultiplexer.ts +52 -2
  33. package/src/sync-protocols/utxorpc/state.ts +3 -0
  34. package/src/syncProtocolFactory.ts +15 -0
  35. package/test/poll-loop-spin.test.ts +112 -0
  36. package/test/rpc-timeout.test.ts +150 -0
  37. package/test/spin-probe.ts +102 -0
  38. package/test/start-async-supervision.test.ts +139 -0
@@ -69,11 +69,62 @@ export abstract class SyncState<
69
69
  public lastPage: undefined | LastPage<Page, RootPage>;
70
70
  /** Timestamp of the last successful fetch (readData completed without error). */
71
71
  public lastSuccessfulFetchMs: number = 0;
72
+ /**
73
+ * Wall-clock time of the last completed pass through the fetch loop, whatever
74
+ * its outcome (fetched, caught up, or errored).
75
+ *
76
+ * This — not `lastSuccessfulFetchMs` — is the hang detector. A chain that is
77
+ * caught up never calls `readData`, so its last-successful-fetch time is
78
+ * indistinguishable from that of a chain wedged inside a request that will
79
+ * never return. The loop still cycles in the first case and does not in the
80
+ * second, so a heartbeat that is stale by many polling intervals means the
81
+ * loop is stuck in `stateToInput`/`readData`.
82
+ */
83
+ public lastPollAtMs: number = 0;
72
84
  /** Number of consecutive errors since the last successful fetch. */
73
85
  public consecutiveErrors: number = 0;
74
86
  /** Timestamp of the most recent error, or 0 if no error has occurred. */
75
87
  public lastErrorTimestamp: number = 0;
76
88
 
89
+ // ── Streaming producer (see startAsync + orchestration/sync.ts) ──
90
+ /**
91
+ * Whether {@link SyncState#startAsync} runs a long-lived producer that this
92
+ * protocol depends on for its data (a subscription/stream), as opposed to the
93
+ * base-class no-op used by polled chains.
94
+ *
95
+ * `startSync` supervises and restarts the producer only when this is true.
96
+ * A polled chain must leave it false, or its no-op `startAsync` would be
97
+ * "restarted" forever.
98
+ */
99
+ public readonly hasAsyncProducer: boolean = false;
100
+ /**
101
+ * How many times the streaming producer has been restarted after dying —
102
+ * either by throwing or by returning (a producer that returns has ended its
103
+ * stream). Surfaced on `/health`; a climbing count means a flapping upstream.
104
+ */
105
+ public producerRestarts: number = 0;
106
+ /**
107
+ * Producer failures, tracked separately from {@link consecutiveErrors}.
108
+ *
109
+ * That counter belongs to the fetch loop and is only cleared by a successful
110
+ * `readData`, so sharing it conflated two independent signals: an idle
111
+ * streaming chain would report `erroring` long after its producer recovered,
112
+ * and a successful poll would erase the record of a flapping producer.
113
+ */
114
+ public producerErrors: number = 0;
115
+ /** Wall-clock time of the last producer failure, or 0 if none. */
116
+ public lastProducerErrorMs: number = 0;
117
+
118
+ /**
119
+ * Resolved polling interval for this protocol, stamped by `startSync`.
120
+ *
121
+ * Single source of truth: `/health` derives its wedged threshold from this
122
+ * rather than reaching back into `fetcher.config.syncProtocol`, which it can
123
+ * only do through an unchecked cast that would fail silently to a default if
124
+ * the config shape ever moved.
125
+ */
126
+ public pollingIntervalMs: number = 0;
127
+
77
128
  // ── Backpressure observability (see common/page-helpers.ts + README) ──
78
129
  /** Resolved fetch cap (`maxBufferedPages`); set on each backpressure check, 0 until the first. */
79
130
  public bufferCap: number = 0;
@@ -27,6 +27,7 @@ import type {
27
27
  BitcoinTxId,
28
28
  } from "@effectstream/utils";
29
29
  import { blockNumberRelation } from "../common/utils.ts";
30
+ import { DEFAULT_REQUEST_TIMEOUT_MS, fetchWithTimeout } from "../common/http.ts";
30
31
  import type { ConfigType } from "./types.ts";
31
32
 
32
33
  const SATOSHIS_PER_BTC = 100_000_000;
@@ -457,6 +458,8 @@ type BitcoinRpcClientOptions = {
457
458
  url: string;
458
459
  username?: string | null;
459
460
  password?: string | null;
461
+ /** Per-request deadline; defaults to {@link DEFAULT_REQUEST_TIMEOUT_MS}. */
462
+ requestTimeoutMs?: number;
460
463
  };
461
464
 
462
465
  type BitcoinRpcRequest = {
@@ -515,16 +518,21 @@ export class BitcoinRpcClient {
515
518
  private async call<TResult>(
516
519
  request: BitcoinRpcRequest,
517
520
  ): Promise<TResult> {
518
- const response = await fetch(this.options.url, {
519
- method: "POST",
520
- headers: this.buildHeaders(),
521
- body: JSON.stringify({
522
- jsonrpc: "2.0",
523
- id: ++this.nextId,
524
- method: request.method,
525
- params: request.params ?? [],
526
- }),
527
- });
521
+ const response = await fetchWithTimeout(
522
+ this.options.url,
523
+ {
524
+ method: "POST",
525
+ headers: this.buildHeaders(),
526
+ body: JSON.stringify({
527
+ jsonrpc: "2.0",
528
+ id: ++this.nextId,
529
+ method: request.method,
530
+ params: request.params ?? [],
531
+ }),
532
+ },
533
+ `Bitcoin ${request.method}`,
534
+ this.options.requestTimeoutMs,
535
+ );
528
536
  if (!response.ok) {
529
537
  const body = await response.text();
530
538
  throw new Error(
@@ -54,6 +54,11 @@ const CELESTIA_NODE_URL = ENV.getString(
54
54
  "http://localhost:26658",
55
55
  );
56
56
  const CELESTIA_AUTH_TOKEN = ENV.getString("CELESTIA_AUTH_TOKEN", "");
57
+ /**
58
+ * Fallback per-request deadline, kept for compatibility with deployments that
59
+ * set it. The protocol config's `requestTimeoutMs` takes precedence — see the
60
+ * constructor.
61
+ */
57
62
  const CELESTIA_RPC_TIMEOUT_MS = parseInt(
58
63
  ENV.getString("CELESTIA_RPC_TIMEOUT_MS", "15000"),
59
64
  );
@@ -77,9 +82,18 @@ export function celestiaNamespaceToBase64(hex: string): string {
77
82
 
78
83
  export class CelestiaClient {
79
84
  private readonly rpcUrl: string;
85
+ /**
86
+ * Per-attempt deadline. Unlike the other clients this one retries internally
87
+ * (see `rpc` below), so this bounds each attempt rather than the whole call.
88
+ */
89
+ private readonly requestTimeoutMs: number;
80
90
 
81
- constructor(rpcUrl: string = CELESTIA_NODE_URL) {
91
+ constructor(
92
+ rpcUrl: string = CELESTIA_NODE_URL,
93
+ requestTimeoutMs: number = CELESTIA_RPC_TIMEOUT_MS,
94
+ ) {
82
95
  this.rpcUrl = rpcUrl;
96
+ this.requestTimeoutMs = requestTimeoutMs;
83
97
  }
84
98
 
85
99
  private async rpc<T>(method: string, params: unknown[]): Promise<T | null> {
@@ -95,7 +109,7 @@ export class CelestiaClient {
95
109
  method: "POST",
96
110
  headers,
97
111
  body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
98
- signal: AbortSignal.timeout(CELESTIA_RPC_TIMEOUT_MS),
112
+ signal: AbortSignal.timeout(this.requestTimeoutMs),
99
113
  });
100
114
  const json = await res.json();
101
115
  if (json.error) {
@@ -2,6 +2,7 @@ import {
2
2
  ConfigSyncProtocolType,
3
3
  type PrimitiveEntry,
4
4
  } from "@effectstream/config";
5
+ import { requestTimeoutOf } from "../common/http.ts";
5
6
  import { BaseDataFetcher } from "../base/fetcher.ts";
6
7
  import type { DataFetched } from "../base/fetcher.ts";
7
8
  import type {
@@ -37,7 +38,10 @@ export class CelestiaFetcher extends BaseDataFetcher<
37
38
  readonly config: ConfigType,
38
39
  ) {
39
40
  super(config.syncProtocol.name);
40
- this.client = new CelestiaClient(config.network.rpcUrl);
41
+ this.client = new CelestiaClient(
42
+ config.network.rpcUrl,
43
+ requestTimeoutOf(config.syncProtocol),
44
+ );
41
45
  }
42
46
 
43
47
  @bound
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
10
10
  import { getPage } from "@effectstream/db";
11
11
  import { CelestiaClient } from "./CelestiaClient.ts";
12
12
  import { applyDelay } from "../common/utils.ts";
13
+ import { requestTimeoutOf } from "../common/http.ts";
13
14
  import { bufferAtCap } from "../common/page-helpers.ts";
14
15
 
15
16
  export class CelestiaSyncState extends SyncState<
@@ -135,7 +136,10 @@ export class CelestiaSyncState extends SyncState<
135
136
  page,
136
137
  config,
137
138
  fetcher,
138
- new CelestiaClient(config.network.rpcUrl),
139
+ new CelestiaClient(
140
+ config.network.rpcUrl,
141
+ requestTimeoutOf(config.syncProtocol),
142
+ ),
139
143
  dbConn,
140
144
  );
141
145
  }
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Bounded HTTP for sync-protocol RPC clients.
3
+ *
4
+ * `fetch` has no default timeout. Against a blackholed endpoint — a load
5
+ * balancer that drops its backend, a socket left half-open by a NAT rebind —
6
+ * the connection is established and simply never answers, so the promise never
7
+ * settles. For a sync protocol that is the worst possible failure: `readData`
8
+ * never returns, so the fetch loop in `orchestration/sync.ts` never reaches its
9
+ * `catch`, `consecutiveErrors` stays 0, `lastSuccessfulFetchMs` freezes, and
10
+ * the merge blocks on that chain's page forever. The node stops producing
11
+ * blocks and reports nothing wrong.
12
+ *
13
+ * Deliberately timeout-only, with NO internal retry. The fetch loop already
14
+ * retries: it catches, counts the error, sleeps `pollingInterval` and re-runs
15
+ * the same page range (fetches are idempotent — `lastPage` only advances on a
16
+ * complete `DataFetched`). Retrying inside the client as well would double the
17
+ * backoff and, worse, hide failures from `consecutiveErrors`, which is what the
18
+ * health endpoint reports on.
19
+ *
20
+ * Regression test: `sync/test/rpc-timeout.test.ts` points each client at a
21
+ * blackhole and asserts it rejects rather than hanging.
22
+ */
23
+
24
+ /** Used when a protocol config carries no `requestTimeoutMs`. */
25
+ export const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;
26
+
27
+ /** Thrown when a request exceeds its budget, so callers can tell it apart. */
28
+ export class RequestTimeoutError extends Error {
29
+ constructor(
30
+ readonly label: string,
31
+ readonly timeoutMs: number,
32
+ ) {
33
+ super(`[${label}] request timed out after ${timeoutMs}ms`);
34
+ this.name = "RequestTimeoutError";
35
+ }
36
+ }
37
+
38
+ /**
39
+ * `fetch` with a hard deadline.
40
+ *
41
+ * @param label Prefix for the thrown error, e.g. `"Bitcoin getblockhash"`.
42
+ * @param timeoutMs Falls back to {@link DEFAULT_REQUEST_TIMEOUT_MS}.
43
+ * @param signal Optional caller signal; aborting either one aborts the request.
44
+ */
45
+ export async function fetchWithTimeout(
46
+ url: string,
47
+ init: RequestInit,
48
+ label: string,
49
+ timeoutMs: number = DEFAULT_REQUEST_TIMEOUT_MS,
50
+ signal?: AbortSignal,
51
+ ): Promise<Response> {
52
+ const timeoutSignal = AbortSignal.timeout(timeoutMs);
53
+ const combined = signal
54
+ ? AbortSignal.any([signal, timeoutSignal])
55
+ : timeoutSignal;
56
+
57
+ try {
58
+ return await fetch(url, { ...init, signal: combined });
59
+ } catch (err) {
60
+ // Surface our own error for the timeout case; a caller-supplied abort and
61
+ // ordinary transport failures propagate unchanged.
62
+ if (timeoutSignal.aborted && !(signal?.aborted ?? false)) {
63
+ throw new RequestTimeoutError(label, timeoutMs);
64
+ }
65
+ throw err;
66
+ }
67
+ }
68
+
69
+ /**
70
+ * Resolve the timeout for a protocol config. Optional in the schema, so this
71
+ * centralises the fallback instead of repeating `?? DEFAULT` at each client.
72
+ */
73
+ export function requestTimeoutOf(
74
+ syncProtocol: { requestTimeoutMs?: number } | undefined,
75
+ ): number {
76
+ return syncProtocol?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
77
+ }
@@ -1,4 +1,5 @@
1
1
  import { BaseDataFetcher } from "../base/fetcher.ts";
2
+ import { requestTimeoutOf } from "../common/http.ts";
2
3
  import type {
3
4
  Block,
4
5
  ConfigType,
@@ -59,6 +60,7 @@ export class MidnightFetcher extends BaseDataFetcher<
59
60
  this.client = new MidnightClient(
60
61
  indexerHttp,
61
62
  this.networkId,
63
+ requestTimeoutOf(config.syncProtocol),
62
64
  );
63
65
  }
64
66
 
@@ -35,7 +35,7 @@ export type CommonFunnelArgs = {
35
35
  // Graphql Block
36
36
  // =============
37
37
 
38
- // Interfaces approximated from GraphQL schema served by indexer v1.3.1
38
+ // Interfaces approximated from GraphQL schema served by indexer v4.3.3 (/api/v3)
39
39
 
40
40
  interface ContractCallOrDeploy {
41
41
  address: MidnightAddress;
@@ -1,3 +1,5 @@
1
+ import { DEFAULT_REQUEST_TIMEOUT_MS, fetchWithTimeout } from "../common/http.ts";
2
+
1
3
  // ============================
2
4
  // NEAR JSON-RPC type responses
3
5
  // ============================
@@ -107,17 +109,25 @@ export function parseNep297Log(log: string): Nep297Event | null {
107
109
 
108
110
  export class NearClient {
109
111
  private readonly rpcUrl: string;
112
+ /** Per-request deadline; see `sync-protocols/common/http.ts`. */
113
+ private readonly requestTimeoutMs: number;
110
114
 
111
- constructor(rpcUrl: string) {
115
+ constructor(rpcUrl: string, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
112
116
  this.rpcUrl = rpcUrl;
117
+ this.requestTimeoutMs = requestTimeoutMs;
113
118
  }
114
119
 
115
120
  private async rpc<T>(method: string, params: Record<string, unknown>): Promise<T> {
116
- const res = await fetch(this.rpcUrl, {
117
- method: "POST",
118
- headers: { "Content-Type": "application/json" },
119
- body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
120
- });
121
+ const res = await fetchWithTimeout(
122
+ this.rpcUrl,
123
+ {
124
+ method: "POST",
125
+ headers: { "Content-Type": "application/json" },
126
+ body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
127
+ },
128
+ `NEAR ${method}`,
129
+ this.requestTimeoutMs,
130
+ );
121
131
 
122
132
  const json = await res.json();
123
133
  if (json.error) {
@@ -4,6 +4,7 @@ import {
4
4
  } from "@effectstream/config";
5
5
  import { BaseDataFetcher } from "../base/fetcher.ts";
6
6
  import type { DataFetched } from "../base/fetcher.ts";
7
+ import { requestTimeoutOf } from "../common/http.ts";
7
8
  import type {
8
9
  LastPage,
9
10
  OutputAndCleanup,
@@ -39,7 +40,10 @@ export class NearFetcher extends BaseDataFetcher<
39
40
  readonly config: ConfigType,
40
41
  ) {
41
42
  super(config.syncProtocol.name);
42
- this.client = new NearClient(config.network.rpcUrl);
43
+ this.client = new NearClient(
44
+ config.network.rpcUrl,
45
+ requestTimeoutOf(config.syncProtocol),
46
+ );
43
47
  }
44
48
 
45
49
  @bound
@@ -11,6 +11,7 @@ import { getPage } from "@effectstream/db";
11
11
  import { NearClient } from "./NearClient.ts";
12
12
  import { applyDelay } from "../common/utils.ts";
13
13
  import { bufferAtCap } from "../common/page-helpers.ts";
14
+ import { requestTimeoutOf } from "../common/http.ts";
14
15
 
15
16
  export class NearSyncState extends SyncState<
16
17
  Input,
@@ -134,7 +135,10 @@ export class NearSyncState extends SyncState<
134
135
  page,
135
136
  config,
136
137
  fetcher,
137
- new NearClient(config.network.rpcUrl),
138
+ new NearClient(
139
+ config.network.rpcUrl,
140
+ requestTimeoutOf(config.syncProtocol),
141
+ ),
138
142
  dbConn,
139
143
  );
140
144
  }
@@ -4,19 +4,113 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
4
4
  import type { AllSyncProtocols, ISyncProtocol } from "../types.ts";
5
5
  import { tryYield } from "@effectstream/utils";
6
6
 
7
+ /**
8
+ * Used when a protocol config carries no `pollingInterval`.
9
+ *
10
+ * Every polling protocol declares one via `PollingSyncProtocol`, so this should
11
+ * be unreachable — it exists because the cost of getting it wrong is not a slow
12
+ * loop but a dead node. A pass through this loop that neither fetches nor
13
+ * sleeps never yields to the macrotask queue, which starves the entire process:
14
+ * no timers, no HTTP server, no other chain's fetch loop, and (for streaming
15
+ * chains) none of the stream callbacks that would let this loop make progress.
16
+ * See `sync/test/poll-loop-spin.test.ts`.
17
+ */
18
+ const FALLBACK_POLLING_INTERVAL_MS = 1_000;
19
+
20
+ /** This protocol's polling interval, or the fallback above. */
21
+ function pollingIntervalOf(config: { syncProtocol: unknown }): number {
22
+ const syncProtocol = config.syncProtocol as { pollingInterval?: number };
23
+ return syncProtocol?.pollingInterval ?? FALLBACK_POLLING_INTERVAL_MS;
24
+ }
25
+
26
+ /** Backoff bounds for restarting a dead streaming producer. */
27
+ const PRODUCER_MIN_BACKOFF_MS = 1_000;
28
+ const PRODUCER_MAX_BACKOFF_MS = 30_000;
29
+
7
30
  export function* startSync(
8
31
  state: AllSyncProtocols,
9
32
  ): Operation<void> {
10
33
  const iState: ISyncProtocol = state as ISyncProtocol;
11
34
 
12
- // spawn async task
35
+ // Resolve once and publish it, so `/health` and anything else reads the same
36
+ // value this loop paces itself with instead of re-deriving it.
37
+ state.pollingIntervalMs = pollingIntervalOf(state.fetcher.config);
38
+
13
39
  yield* spawn(function* () {
14
- yield* iState.startAsync();
40
+ if (!iState.hasAsyncProducer) {
41
+ // Polled chain: startAsync is the base-class no-op. Run it once, as
42
+ // before — supervising it would mean restarting a no-op forever.
43
+ yield* iState.startAsync();
44
+ return;
45
+ }
46
+
47
+ // Streaming chain: all of this protocol's data arrives through the
48
+ // producer, so if it dies the chain goes silent and the merge blocks on its
49
+ // page forever. Both of its exit modes used to be unhandled — a throw tore
50
+ // down the enclosing scope (in production, `start()`, taking every other
51
+ // chain with it), and a clean return went entirely unnoticed. Supervise
52
+ // both. See `sync/test/start-async-supervision.test.ts`.
53
+ let attempt = 0;
54
+ while (true) {
55
+ const startedAt = Date.now();
56
+ const result = yield* tryYield(iState.startAsync());
57
+ const ranForMs = Date.now() - startedAt;
58
+
59
+ if (result.error != null) {
60
+ // Producer failures are tracked separately from `consecutiveErrors`.
61
+ // That counter belongs to the fetch loop and is only cleared by a
62
+ // successful `readData`, so bumping it here left an idle streaming chain
63
+ // reporting `erroring` until its next block arrived — and conversely, a
64
+ // successful poll would erase the record of a flapping producer.
65
+ state.producerErrors++;
66
+ state.lastProducerErrorMs = Date.now();
67
+ log.remote(
68
+ ComponentNames.EFFECTSTREAM_SYNC,
69
+ [...state.getNamespace(), "startAsync"],
70
+ SeverityNumber.ERROR,
71
+ (l) => l(result.error),
72
+ );
73
+ } else {
74
+ // Returning is a failure for a producer: its stream ended. Treated the
75
+ // same as a throw, because the observable effect is identical — no
76
+ // more data.
77
+ log.remote(
78
+ ComponentNames.EFFECTSTREAM_SYNC,
79
+ [...state.getNamespace(), "startAsync"],
80
+ SeverityNumber.WARN,
81
+ (l) => l("producer returned: stream ended, restarting"),
82
+ );
83
+ }
84
+
85
+ state.producerRestarts++;
86
+
87
+ // De-escalate after a producer that clearly worked. Without this `attempt`
88
+ // only ever grows, so a stream that ran healthily for a day between
89
+ // incidents would still be waiting the 30s cap on its next restart —
90
+ // penalising a healthy chain for ancient history. (It also keeps
91
+ // `2 ** attempt` from reaching Infinity after ~1024 restarts, which
92
+ // `Math.min` currently absorbs silently.)
93
+ if (ranForMs >= PRODUCER_MAX_BACKOFF_MS) {
94
+ attempt = 0;
95
+ }
96
+
97
+ const backoff = Math.min(
98
+ PRODUCER_MIN_BACKOFF_MS * 2 ** attempt,
99
+ PRODUCER_MAX_BACKOFF_MS,
100
+ );
101
+ attempt++;
102
+ yield* sleep(backoff);
103
+ }
15
104
  });
16
105
 
17
106
  // spawn polling task
18
107
  yield* spawn(function* () {
19
108
  while (true) {
109
+ // Liveness heartbeat: stamped every pass regardless of outcome, so a
110
+ // stale value means the loop is wedged inside a call that never returns
111
+ // rather than merely idle. See SyncState.lastPollAtMs.
112
+ state.lastPollAtMs = Date.now();
113
+
20
114
  const inputResult = yield* tryYield(iState.stateToInput());
21
115
  if (inputResult.error != null) {
22
116
  state.consecutiveErrors++;
@@ -27,18 +121,14 @@ export function* startSync(
27
121
  SeverityNumber.ERROR,
28
122
  (log) => log(inputResult.error),
29
123
  );
30
- const { config } = state.fetcher;
31
- if ("pollingInterval" in config.syncProtocol) {
32
- yield* sleep(config.syncProtocol.pollingInterval);
33
- }
124
+ yield* sleep(state.pollingIntervalMs);
34
125
  continue;
35
126
  }
36
127
  const input = inputResult.data;
37
- const { config } = state.fetcher;
38
128
  if (input == null) {
39
- if ("pollingInterval" in config.syncProtocol) {
40
- yield* sleep(config.syncProtocol.pollingInterval);
41
- }
129
+ // Caught up. This is the ordinary steady state, and the pass that must
130
+ // never skip its sleep — see FALLBACK_POLLING_INTERVAL_MS.
131
+ yield* sleep(state.pollingIntervalMs);
42
132
  continue;
43
133
  }
44
134
 
@@ -52,9 +142,7 @@ export function* startSync(
52
142
  SeverityNumber.ERROR,
53
143
  (l) => l(result.error),
54
144
  );
55
- if ("pollingInterval" in config.syncProtocol) {
56
- yield* sleep(config.syncProtocol.pollingInterval);
57
- }
145
+ yield* sleep(state.pollingIntervalMs);
58
146
  continue;
59
147
  }
60
148
  state.consecutiveErrors = 0;
@@ -0,0 +1,158 @@
1
+ import {
2
+ DEFAULT_REQUEST_TIMEOUT_MS,
3
+ fetchWithTimeout,
4
+ } from "../common/http.ts";
5
+
6
+ // ==========================
7
+ // Solana JSON-RPC type defs
8
+ // ==========================
9
+
10
+ export type SolanaBlock = {
11
+ blockhash: string;
12
+ blockTime: number | null;
13
+ blockHeight: number | null;
14
+ parentSlot: number;
15
+ previousBlockhash: string;
16
+ transactions: SolanaTransaction[];
17
+ };
18
+
19
+ export type SolanaTransaction = {
20
+ transaction: {
21
+ message: {
22
+ accountKeys: string[];
23
+ instructions: SolanaInstruction[];
24
+ };
25
+ signatures: string[];
26
+ };
27
+ /**
28
+ * Null when the RPC could not decode the transaction (e.g. a version newer
29
+ * than `maxSupportedTransactionVersion`). Callers must null-check.
30
+ */
31
+ meta: {
32
+ err: unknown | null;
33
+ logMessages: string[] | null;
34
+ preBalances: number[];
35
+ postBalances: number[];
36
+ /**
37
+ * Addresses a versioned (v0) transaction pulled in through an address
38
+ * lookup table. These are NOT in `message.accountKeys`, which carries only
39
+ * static keys — but `pre`/`postBalances` are indexed over the full list.
40
+ * See {@link resolveAccountKeys} for the ordering.
41
+ */
42
+ loadedAddresses?: {
43
+ writable: string[];
44
+ readonly: string[];
45
+ } | null;
46
+ } | null;
47
+ };
48
+
49
+ /**
50
+ * The account list `pre`/`postBalances` are indexed against: static message
51
+ * keys first, then lookup-table writable addresses, then lookup-table readonly
52
+ * ones. Legacy transactions have no `loadedAddresses`, so this is just the
53
+ * static keys.
54
+ */
55
+ export function resolveAccountKeys(
56
+ accountKeys: string[],
57
+ loadedAddresses?: { writable: string[]; readonly: string[] } | null,
58
+ ): string[] {
59
+ if (!loadedAddresses) return accountKeys;
60
+ return [
61
+ ...accountKeys,
62
+ ...(loadedAddresses.writable ?? []),
63
+ ...(loadedAddresses.readonly ?? []),
64
+ ];
65
+ }
66
+
67
+ export type SolanaInstruction = {
68
+ programId: string;
69
+ accounts: string[];
70
+ data: string;
71
+ };
72
+
73
+ // ===========
74
+ // RPC Client
75
+ // ===========
76
+
77
+ export class SolanaClient {
78
+ private readonly rpcUrl: string;
79
+ /** Per-request deadline; see `sync-protocols/common/http.ts`. */
80
+ private readonly requestTimeoutMs: number;
81
+
82
+ constructor(rpcUrl: string, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
83
+ this.rpcUrl = rpcUrl;
84
+ this.requestTimeoutMs = requestTimeoutMs;
85
+ }
86
+
87
+ private async rpc<T>(
88
+ method: string,
89
+ params: unknown[] = [],
90
+ ): Promise<T> {
91
+ // Bounded: a blackholed endpoint would otherwise hang readData forever,
92
+ // freezing block production with every health counter clean (sync
93
+ // CLAUDE.md finding #4).
94
+ const res = await fetchWithTimeout(
95
+ this.rpcUrl,
96
+ {
97
+ method: "POST",
98
+ headers: { "Content-Type": "application/json" },
99
+ body: JSON.stringify({
100
+ jsonrpc: "2.0",
101
+ id: 1,
102
+ method,
103
+ params,
104
+ }),
105
+ },
106
+ `Solana ${method}`,
107
+ this.requestTimeoutMs,
108
+ );
109
+
110
+ const json = await res.json();
111
+ if (json.error) {
112
+ const err = new Error(
113
+ `[Solana] RPC error [${method}]: ${json.error.message ?? JSON.stringify(json.error)}`,
114
+ ) as Error & { rpcCode?: number };
115
+ // Preserve the JSON-RPC code so callers can branch on it instead of
116
+ // pattern-matching human-readable messages.
117
+ err.rpcCode = typeof json.error.code === "number" ? json.error.code : undefined;
118
+ throw err;
119
+ }
120
+ return json.result as T;
121
+ }
122
+
123
+ async getSlot(): Promise<number> {
124
+ return this.rpc<number>("getSlot", [
125
+ { commitment: "confirmed" },
126
+ ]);
127
+ }
128
+
129
+ async getBlock(
130
+ slot: number,
131
+ ): Promise<SolanaBlock | null> {
132
+ try {
133
+ return await this.rpc<SolanaBlock | null>("getBlock", [
134
+ slot,
135
+ {
136
+ encoding: "json",
137
+ transactionDetails: "full",
138
+ rewards: false,
139
+ maxSupportedTransactionVersion: 0,
140
+ commitment: "confirmed",
141
+ },
142
+ ]);
143
+ } catch (e) {
144
+ // A skipped slot is normal on Solana: no block was produced. Branch on the
145
+ // JSON-RPC code rather than the message text —
146
+ // -32007 SLOT_SKIPPED, -32009 LONG_TERM_STORAGE_SLOT_SKIPPED.
147
+ // Deliberately NOT -32004 (block not available yet): that is a transient
148
+ // "ask again" and must keep throwing so the fetcher retries rather than
149
+ // treating the slot as permanently empty.
150
+ const code = (e as { rpcCode?: number }).rpcCode;
151
+ if (code === -32007 || code === -32009) return null;
152
+ // Fall back to the message for RPCs that omit or remap the code.
153
+ if (e instanceof Error && e.message.includes("was skipped")) return null;
154
+ throw e;
155
+ }
156
+ }
157
+
158
+ }