@effectstream/sync 0.101.0 → 0.102.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +82 -18
- package/LICENSE-APACHE +201 -0
- package/LICENSE-MIT +21 -0
- package/README.md +63 -4
- package/docs/ADDING-A-SYNC-PROTOCOL.md +16 -9
- package/package.json +8 -8
- package/src/sync-protocols/avail/AvailClient.ts +34 -6
- package/src/sync-protocols/avail/fetcher.ts +2 -0
- package/src/sync-protocols/avail/state.ts +2 -0
- package/src/sync-protocols/base/state.ts +51 -0
- package/src/sync-protocols/bitcoin/fetcher.ts +18 -10
- package/src/sync-protocols/celestia/CelestiaClient.ts +16 -2
- package/src/sync-protocols/celestia/fetcher.ts +5 -1
- package/src/sync-protocols/celestia/state.ts +5 -1
- package/src/sync-protocols/common/http.ts +77 -0
- package/src/sync-protocols/midnight/fetcher.ts +2 -0
- package/src/sync-protocols/midnight/types.ts +1 -1
- package/src/sync-protocols/near/NearClient.ts +16 -6
- package/src/sync-protocols/near/fetcher.ts +5 -1
- package/src/sync-protocols/near/state.ts +5 -1
- package/src/sync-protocols/orchestration/sync.ts +101 -13
- package/src/sync-protocols/solana/SolanaClient.ts +158 -0
- package/src/sync-protocols/solana/fetcher.ts +358 -0
- package/src/sync-protocols/solana/program-logs.test.ts +113 -0
- package/src/sync-protocols/solana/program-logs.ts +84 -0
- package/src/sync-protocols/solana/read-data-retry.test.ts +153 -0
- package/src/sync-protocols/solana/read-primitives.test.ts +188 -0
- package/src/sync-protocols/solana/root-page.test.ts +66 -0
- package/src/sync-protocols/solana/state.ts +167 -0
- package/src/sync-protocols/solana/types.ts +58 -0
- package/src/sync-protocols/types.ts +2 -0
- package/src/sync-protocols/utxorpc/WatchMultiplexer.ts +52 -2
- package/src/sync-protocols/utxorpc/state.ts +3 -0
- package/src/syncProtocolFactory.ts +15 -0
- package/test/poll-loop-spin.test.ts +112 -0
- package/test/rpc-timeout.test.ts +150 -0
- package/test/spin-probe.ts +102 -0
- package/test/start-async-supervision.test.ts +139 -0
|
@@ -69,11 +69,62 @@ export abstract class SyncState<
|
|
|
69
69
|
public lastPage: undefined | LastPage<Page, RootPage>;
|
|
70
70
|
/** Timestamp of the last successful fetch (readData completed without error). */
|
|
71
71
|
public lastSuccessfulFetchMs: number = 0;
|
|
72
|
+
/**
|
|
73
|
+
* Wall-clock time of the last completed pass through the fetch loop, whatever
|
|
74
|
+
* its outcome (fetched, caught up, or errored).
|
|
75
|
+
*
|
|
76
|
+
* This — not `lastSuccessfulFetchMs` — is the hang detector. A chain that is
|
|
77
|
+
* caught up never calls `readData`, so its last-successful-fetch time is
|
|
78
|
+
* indistinguishable from that of a chain wedged inside a request that will
|
|
79
|
+
* never return. The loop still cycles in the first case and does not in the
|
|
80
|
+
* second, so a heartbeat that is stale by many polling intervals means the
|
|
81
|
+
* loop is stuck in `stateToInput`/`readData`.
|
|
82
|
+
*/
|
|
83
|
+
public lastPollAtMs: number = 0;
|
|
72
84
|
/** Number of consecutive errors since the last successful fetch. */
|
|
73
85
|
public consecutiveErrors: number = 0;
|
|
74
86
|
/** Timestamp of the most recent error, or 0 if no error has occurred. */
|
|
75
87
|
public lastErrorTimestamp: number = 0;
|
|
76
88
|
|
|
89
|
+
// ── Streaming producer (see startAsync + orchestration/sync.ts) ──
|
|
90
|
+
/**
|
|
91
|
+
* Whether {@link SyncState#startAsync} runs a long-lived producer that this
|
|
92
|
+
* protocol depends on for its data (a subscription/stream), as opposed to the
|
|
93
|
+
* base-class no-op used by polled chains.
|
|
94
|
+
*
|
|
95
|
+
* `startSync` supervises and restarts the producer only when this is true.
|
|
96
|
+
* A polled chain must leave it false, or its no-op `startAsync` would be
|
|
97
|
+
* "restarted" forever.
|
|
98
|
+
*/
|
|
99
|
+
public readonly hasAsyncProducer: boolean = false;
|
|
100
|
+
/**
|
|
101
|
+
* How many times the streaming producer has been restarted after dying —
|
|
102
|
+
* either by throwing or by returning (a producer that returns has ended its
|
|
103
|
+
* stream). Surfaced on `/health`; a climbing count means a flapping upstream.
|
|
104
|
+
*/
|
|
105
|
+
public producerRestarts: number = 0;
|
|
106
|
+
/**
|
|
107
|
+
* Producer failures, tracked separately from {@link consecutiveErrors}.
|
|
108
|
+
*
|
|
109
|
+
* That counter belongs to the fetch loop and is only cleared by a successful
|
|
110
|
+
* `readData`, so sharing it conflated two independent signals: an idle
|
|
111
|
+
* streaming chain would report `erroring` long after its producer recovered,
|
|
112
|
+
* and a successful poll would erase the record of a flapping producer.
|
|
113
|
+
*/
|
|
114
|
+
public producerErrors: number = 0;
|
|
115
|
+
/** Wall-clock time of the last producer failure, or 0 if none. */
|
|
116
|
+
public lastProducerErrorMs: number = 0;
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Resolved polling interval for this protocol, stamped by `startSync`.
|
|
120
|
+
*
|
|
121
|
+
* Single source of truth: `/health` derives its wedged threshold from this
|
|
122
|
+
* rather than reaching back into `fetcher.config.syncProtocol`, which it can
|
|
123
|
+
* only do through an unchecked cast that would fail silently to a default if
|
|
124
|
+
* the config shape ever moved.
|
|
125
|
+
*/
|
|
126
|
+
public pollingIntervalMs: number = 0;
|
|
127
|
+
|
|
77
128
|
// ── Backpressure observability (see common/page-helpers.ts + README) ──
|
|
78
129
|
/** Resolved fetch cap (`maxBufferedPages`); set on each backpressure check, 0 until the first. */
|
|
79
130
|
public bufferCap: number = 0;
|
|
@@ -27,6 +27,7 @@ import type {
|
|
|
27
27
|
BitcoinTxId,
|
|
28
28
|
} from "@effectstream/utils";
|
|
29
29
|
import { blockNumberRelation } from "../common/utils.ts";
|
|
30
|
+
import { DEFAULT_REQUEST_TIMEOUT_MS, fetchWithTimeout } from "../common/http.ts";
|
|
30
31
|
import type { ConfigType } from "./types.ts";
|
|
31
32
|
|
|
32
33
|
const SATOSHIS_PER_BTC = 100_000_000;
|
|
@@ -457,6 +458,8 @@ type BitcoinRpcClientOptions = {
|
|
|
457
458
|
url: string;
|
|
458
459
|
username?: string | null;
|
|
459
460
|
password?: string | null;
|
|
461
|
+
/** Per-request deadline; defaults to {@link DEFAULT_REQUEST_TIMEOUT_MS}. */
|
|
462
|
+
requestTimeoutMs?: number;
|
|
460
463
|
};
|
|
461
464
|
|
|
462
465
|
type BitcoinRpcRequest = {
|
|
@@ -515,16 +518,21 @@ export class BitcoinRpcClient {
|
|
|
515
518
|
private async call<TResult>(
|
|
516
519
|
request: BitcoinRpcRequest,
|
|
517
520
|
): Promise<TResult> {
|
|
518
|
-
const response = await
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
521
|
+
const response = await fetchWithTimeout(
|
|
522
|
+
this.options.url,
|
|
523
|
+
{
|
|
524
|
+
method: "POST",
|
|
525
|
+
headers: this.buildHeaders(),
|
|
526
|
+
body: JSON.stringify({
|
|
527
|
+
jsonrpc: "2.0",
|
|
528
|
+
id: ++this.nextId,
|
|
529
|
+
method: request.method,
|
|
530
|
+
params: request.params ?? [],
|
|
531
|
+
}),
|
|
532
|
+
},
|
|
533
|
+
`Bitcoin ${request.method}`,
|
|
534
|
+
this.options.requestTimeoutMs,
|
|
535
|
+
);
|
|
528
536
|
if (!response.ok) {
|
|
529
537
|
const body = await response.text();
|
|
530
538
|
throw new Error(
|
|
@@ -54,6 +54,11 @@ const CELESTIA_NODE_URL = ENV.getString(
|
|
|
54
54
|
"http://localhost:26658",
|
|
55
55
|
);
|
|
56
56
|
const CELESTIA_AUTH_TOKEN = ENV.getString("CELESTIA_AUTH_TOKEN", "");
|
|
57
|
+
/**
|
|
58
|
+
* Fallback per-request deadline, kept for compatibility with deployments that
|
|
59
|
+
* set it. The protocol config's `requestTimeoutMs` takes precedence — see the
|
|
60
|
+
* constructor.
|
|
61
|
+
*/
|
|
57
62
|
const CELESTIA_RPC_TIMEOUT_MS = parseInt(
|
|
58
63
|
ENV.getString("CELESTIA_RPC_TIMEOUT_MS", "15000"),
|
|
59
64
|
);
|
|
@@ -77,9 +82,18 @@ export function celestiaNamespaceToBase64(hex: string): string {
|
|
|
77
82
|
|
|
78
83
|
export class CelestiaClient {
|
|
79
84
|
private readonly rpcUrl: string;
|
|
85
|
+
/**
|
|
86
|
+
* Per-attempt deadline. Unlike the other clients this one retries internally
|
|
87
|
+
* (see `rpc` below), so this bounds each attempt rather than the whole call.
|
|
88
|
+
*/
|
|
89
|
+
private readonly requestTimeoutMs: number;
|
|
80
90
|
|
|
81
|
-
constructor(
|
|
91
|
+
constructor(
|
|
92
|
+
rpcUrl: string = CELESTIA_NODE_URL,
|
|
93
|
+
requestTimeoutMs: number = CELESTIA_RPC_TIMEOUT_MS,
|
|
94
|
+
) {
|
|
82
95
|
this.rpcUrl = rpcUrl;
|
|
96
|
+
this.requestTimeoutMs = requestTimeoutMs;
|
|
83
97
|
}
|
|
84
98
|
|
|
85
99
|
private async rpc<T>(method: string, params: unknown[]): Promise<T | null> {
|
|
@@ -95,7 +109,7 @@ export class CelestiaClient {
|
|
|
95
109
|
method: "POST",
|
|
96
110
|
headers,
|
|
97
111
|
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
|
|
98
|
-
signal: AbortSignal.timeout(
|
|
112
|
+
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
|
99
113
|
});
|
|
100
114
|
const json = await res.json();
|
|
101
115
|
if (json.error) {
|
|
@@ -2,6 +2,7 @@ import {
|
|
|
2
2
|
ConfigSyncProtocolType,
|
|
3
3
|
type PrimitiveEntry,
|
|
4
4
|
} from "@effectstream/config";
|
|
5
|
+
import { requestTimeoutOf } from "../common/http.ts";
|
|
5
6
|
import { BaseDataFetcher } from "../base/fetcher.ts";
|
|
6
7
|
import type { DataFetched } from "../base/fetcher.ts";
|
|
7
8
|
import type {
|
|
@@ -37,7 +38,10 @@ export class CelestiaFetcher extends BaseDataFetcher<
|
|
|
37
38
|
readonly config: ConfigType,
|
|
38
39
|
) {
|
|
39
40
|
super(config.syncProtocol.name);
|
|
40
|
-
this.client = new CelestiaClient(
|
|
41
|
+
this.client = new CelestiaClient(
|
|
42
|
+
config.network.rpcUrl,
|
|
43
|
+
requestTimeoutOf(config.syncProtocol),
|
|
44
|
+
);
|
|
41
45
|
}
|
|
42
46
|
|
|
43
47
|
@bound
|
|
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
|
|
|
10
10
|
import { getPage } from "@effectstream/db";
|
|
11
11
|
import { CelestiaClient } from "./CelestiaClient.ts";
|
|
12
12
|
import { applyDelay } from "../common/utils.ts";
|
|
13
|
+
import { requestTimeoutOf } from "../common/http.ts";
|
|
13
14
|
import { bufferAtCap } from "../common/page-helpers.ts";
|
|
14
15
|
|
|
15
16
|
export class CelestiaSyncState extends SyncState<
|
|
@@ -135,7 +136,10 @@ export class CelestiaSyncState extends SyncState<
|
|
|
135
136
|
page,
|
|
136
137
|
config,
|
|
137
138
|
fetcher,
|
|
138
|
-
new CelestiaClient(
|
|
139
|
+
new CelestiaClient(
|
|
140
|
+
config.network.rpcUrl,
|
|
141
|
+
requestTimeoutOf(config.syncProtocol),
|
|
142
|
+
),
|
|
139
143
|
dbConn,
|
|
140
144
|
);
|
|
141
145
|
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bounded HTTP for sync-protocol RPC clients.
|
|
3
|
+
*
|
|
4
|
+
* `fetch` has no default timeout. Against a blackholed endpoint — a load
|
|
5
|
+
* balancer that drops its backend, a socket left half-open by a NAT rebind —
|
|
6
|
+
* the connection is established and simply never answers, so the promise never
|
|
7
|
+
* settles. For a sync protocol that is the worst possible failure: `readData`
|
|
8
|
+
* never returns, so the fetch loop in `orchestration/sync.ts` never reaches its
|
|
9
|
+
* `catch`, `consecutiveErrors` stays 0, `lastSuccessfulFetchMs` freezes, and
|
|
10
|
+
* the merge blocks on that chain's page forever. The node stops producing
|
|
11
|
+
* blocks and reports nothing wrong.
|
|
12
|
+
*
|
|
13
|
+
* Deliberately timeout-only, with NO internal retry. The fetch loop already
|
|
14
|
+
* retries: it catches, counts the error, sleeps `pollingInterval` and re-runs
|
|
15
|
+
* the same page range (fetches are idempotent — `lastPage` only advances on a
|
|
16
|
+
* complete `DataFetched`). Retrying inside the client as well would double the
|
|
17
|
+
* backoff and, worse, hide failures from `consecutiveErrors`, which is what the
|
|
18
|
+
* health endpoint reports on.
|
|
19
|
+
*
|
|
20
|
+
* Regression test: `sync/test/rpc-timeout.test.ts` points each client at a
|
|
21
|
+
* blackhole and asserts it rejects rather than hanging.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** Used when a protocol config carries no `requestTimeoutMs`. */
|
|
25
|
+
export const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;
|
|
26
|
+
|
|
27
|
+
/** Thrown when a request exceeds its budget, so callers can tell it apart. */
|
|
28
|
+
export class RequestTimeoutError extends Error {
|
|
29
|
+
constructor(
|
|
30
|
+
readonly label: string,
|
|
31
|
+
readonly timeoutMs: number,
|
|
32
|
+
) {
|
|
33
|
+
super(`[${label}] request timed out after ${timeoutMs}ms`);
|
|
34
|
+
this.name = "RequestTimeoutError";
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* `fetch` with a hard deadline.
|
|
40
|
+
*
|
|
41
|
+
* @param label Prefix for the thrown error, e.g. `"Bitcoin getblockhash"`.
|
|
42
|
+
* @param timeoutMs Falls back to {@link DEFAULT_REQUEST_TIMEOUT_MS}.
|
|
43
|
+
* @param signal Optional caller signal; aborting either one aborts the request.
|
|
44
|
+
*/
|
|
45
|
+
export async function fetchWithTimeout(
|
|
46
|
+
url: string,
|
|
47
|
+
init: RequestInit,
|
|
48
|
+
label: string,
|
|
49
|
+
timeoutMs: number = DEFAULT_REQUEST_TIMEOUT_MS,
|
|
50
|
+
signal?: AbortSignal,
|
|
51
|
+
): Promise<Response> {
|
|
52
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
53
|
+
const combined = signal
|
|
54
|
+
? AbortSignal.any([signal, timeoutSignal])
|
|
55
|
+
: timeoutSignal;
|
|
56
|
+
|
|
57
|
+
try {
|
|
58
|
+
return await fetch(url, { ...init, signal: combined });
|
|
59
|
+
} catch (err) {
|
|
60
|
+
// Surface our own error for the timeout case; a caller-supplied abort and
|
|
61
|
+
// ordinary transport failures propagate unchanged.
|
|
62
|
+
if (timeoutSignal.aborted && !(signal?.aborted ?? false)) {
|
|
63
|
+
throw new RequestTimeoutError(label, timeoutMs);
|
|
64
|
+
}
|
|
65
|
+
throw err;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Resolve the timeout for a protocol config. Optional in the schema, so this
|
|
71
|
+
* centralises the fallback instead of repeating `?? DEFAULT` at each client.
|
|
72
|
+
*/
|
|
73
|
+
export function requestTimeoutOf(
|
|
74
|
+
syncProtocol: { requestTimeoutMs?: number } | undefined,
|
|
75
|
+
): number {
|
|
76
|
+
return syncProtocol?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
77
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { BaseDataFetcher } from "../base/fetcher.ts";
|
|
2
|
+
import { requestTimeoutOf } from "../common/http.ts";
|
|
2
3
|
import type {
|
|
3
4
|
Block,
|
|
4
5
|
ConfigType,
|
|
@@ -59,6 +60,7 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
59
60
|
this.client = new MidnightClient(
|
|
60
61
|
indexerHttp,
|
|
61
62
|
this.networkId,
|
|
63
|
+
requestTimeoutOf(config.syncProtocol),
|
|
62
64
|
);
|
|
63
65
|
}
|
|
64
66
|
|
|
@@ -35,7 +35,7 @@ export type CommonFunnelArgs = {
|
|
|
35
35
|
// Graphql Block
|
|
36
36
|
// =============
|
|
37
37
|
|
|
38
|
-
// Interfaces approximated from GraphQL schema served by indexer
|
|
38
|
+
// Interfaces approximated from GraphQL schema served by indexer v4.3.3 (/api/v3)
|
|
39
39
|
|
|
40
40
|
interface ContractCallOrDeploy {
|
|
41
41
|
address: MidnightAddress;
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { DEFAULT_REQUEST_TIMEOUT_MS, fetchWithTimeout } from "../common/http.ts";
|
|
2
|
+
|
|
1
3
|
// ============================
|
|
2
4
|
// NEAR JSON-RPC type responses
|
|
3
5
|
// ============================
|
|
@@ -107,17 +109,25 @@ export function parseNep297Log(log: string): Nep297Event | null {
|
|
|
107
109
|
|
|
108
110
|
export class NearClient {
|
|
109
111
|
private readonly rpcUrl: string;
|
|
112
|
+
/** Per-request deadline; see `sync-protocols/common/http.ts`. */
|
|
113
|
+
private readonly requestTimeoutMs: number;
|
|
110
114
|
|
|
111
|
-
constructor(rpcUrl: string) {
|
|
115
|
+
constructor(rpcUrl: string, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
|
|
112
116
|
this.rpcUrl = rpcUrl;
|
|
117
|
+
this.requestTimeoutMs = requestTimeoutMs;
|
|
113
118
|
}
|
|
114
119
|
|
|
115
120
|
private async rpc<T>(method: string, params: Record<string, unknown>): Promise<T> {
|
|
116
|
-
const res = await
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
+
const res = await fetchWithTimeout(
|
|
122
|
+
this.rpcUrl,
|
|
123
|
+
{
|
|
124
|
+
method: "POST",
|
|
125
|
+
headers: { "Content-Type": "application/json" },
|
|
126
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
|
|
127
|
+
},
|
|
128
|
+
`NEAR ${method}`,
|
|
129
|
+
this.requestTimeoutMs,
|
|
130
|
+
);
|
|
121
131
|
|
|
122
132
|
const json = await res.json();
|
|
123
133
|
if (json.error) {
|
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
} from "@effectstream/config";
|
|
5
5
|
import { BaseDataFetcher } from "../base/fetcher.ts";
|
|
6
6
|
import type { DataFetched } from "../base/fetcher.ts";
|
|
7
|
+
import { requestTimeoutOf } from "../common/http.ts";
|
|
7
8
|
import type {
|
|
8
9
|
LastPage,
|
|
9
10
|
OutputAndCleanup,
|
|
@@ -39,7 +40,10 @@ export class NearFetcher extends BaseDataFetcher<
|
|
|
39
40
|
readonly config: ConfigType,
|
|
40
41
|
) {
|
|
41
42
|
super(config.syncProtocol.name);
|
|
42
|
-
this.client = new NearClient(
|
|
43
|
+
this.client = new NearClient(
|
|
44
|
+
config.network.rpcUrl,
|
|
45
|
+
requestTimeoutOf(config.syncProtocol),
|
|
46
|
+
);
|
|
43
47
|
}
|
|
44
48
|
|
|
45
49
|
@bound
|
|
@@ -11,6 +11,7 @@ import { getPage } from "@effectstream/db";
|
|
|
11
11
|
import { NearClient } from "./NearClient.ts";
|
|
12
12
|
import { applyDelay } from "../common/utils.ts";
|
|
13
13
|
import { bufferAtCap } from "../common/page-helpers.ts";
|
|
14
|
+
import { requestTimeoutOf } from "../common/http.ts";
|
|
14
15
|
|
|
15
16
|
export class NearSyncState extends SyncState<
|
|
16
17
|
Input,
|
|
@@ -134,7 +135,10 @@ export class NearSyncState extends SyncState<
|
|
|
134
135
|
page,
|
|
135
136
|
config,
|
|
136
137
|
fetcher,
|
|
137
|
-
new NearClient(
|
|
138
|
+
new NearClient(
|
|
139
|
+
config.network.rpcUrl,
|
|
140
|
+
requestTimeoutOf(config.syncProtocol),
|
|
141
|
+
),
|
|
138
142
|
dbConn,
|
|
139
143
|
);
|
|
140
144
|
}
|
|
@@ -4,19 +4,113 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
|
|
|
4
4
|
import type { AllSyncProtocols, ISyncProtocol } from "../types.ts";
|
|
5
5
|
import { tryYield } from "@effectstream/utils";
|
|
6
6
|
|
|
7
|
+
/**
|
|
8
|
+
* Used when a protocol config carries no `pollingInterval`.
|
|
9
|
+
*
|
|
10
|
+
* Every polling protocol declares one via `PollingSyncProtocol`, so this should
|
|
11
|
+
* be unreachable — it exists because the cost of getting it wrong is not a slow
|
|
12
|
+
* loop but a dead node. A pass through this loop that neither fetches nor
|
|
13
|
+
* sleeps never yields to the macrotask queue, which starves the entire process:
|
|
14
|
+
* no timers, no HTTP server, no other chain's fetch loop, and (for streaming
|
|
15
|
+
* chains) none of the stream callbacks that would let this loop make progress.
|
|
16
|
+
* See `sync/test/poll-loop-spin.test.ts`.
|
|
17
|
+
*/
|
|
18
|
+
const FALLBACK_POLLING_INTERVAL_MS = 1_000;
|
|
19
|
+
|
|
20
|
+
/** This protocol's polling interval, or the fallback above. */
|
|
21
|
+
function pollingIntervalOf(config: { syncProtocol: unknown }): number {
|
|
22
|
+
const syncProtocol = config.syncProtocol as { pollingInterval?: number };
|
|
23
|
+
return syncProtocol?.pollingInterval ?? FALLBACK_POLLING_INTERVAL_MS;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** Backoff bounds for restarting a dead streaming producer. */
|
|
27
|
+
const PRODUCER_MIN_BACKOFF_MS = 1_000;
|
|
28
|
+
const PRODUCER_MAX_BACKOFF_MS = 30_000;
|
|
29
|
+
|
|
7
30
|
export function* startSync(
|
|
8
31
|
state: AllSyncProtocols,
|
|
9
32
|
): Operation<void> {
|
|
10
33
|
const iState: ISyncProtocol = state as ISyncProtocol;
|
|
11
34
|
|
|
12
|
-
//
|
|
35
|
+
// Resolve once and publish it, so `/health` and anything else reads the same
|
|
36
|
+
// value this loop paces itself with instead of re-deriving it.
|
|
37
|
+
state.pollingIntervalMs = pollingIntervalOf(state.fetcher.config);
|
|
38
|
+
|
|
13
39
|
yield* spawn(function* () {
|
|
14
|
-
|
|
40
|
+
if (!iState.hasAsyncProducer) {
|
|
41
|
+
// Polled chain: startAsync is the base-class no-op. Run it once, as
|
|
42
|
+
// before — supervising it would mean restarting a no-op forever.
|
|
43
|
+
yield* iState.startAsync();
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// Streaming chain: all of this protocol's data arrives through the
|
|
48
|
+
// producer, so if it dies the chain goes silent and the merge blocks on its
|
|
49
|
+
// page forever. Both of its exit modes used to be unhandled — a throw tore
|
|
50
|
+
// down the enclosing scope (in production, `start()`, taking every other
|
|
51
|
+
// chain with it), and a clean return went entirely unnoticed. Supervise
|
|
52
|
+
// both. See `sync/test/start-async-supervision.test.ts`.
|
|
53
|
+
let attempt = 0;
|
|
54
|
+
while (true) {
|
|
55
|
+
const startedAt = Date.now();
|
|
56
|
+
const result = yield* tryYield(iState.startAsync());
|
|
57
|
+
const ranForMs = Date.now() - startedAt;
|
|
58
|
+
|
|
59
|
+
if (result.error != null) {
|
|
60
|
+
// Producer failures are tracked separately from `consecutiveErrors`.
|
|
61
|
+
// That counter belongs to the fetch loop and is only cleared by a
|
|
62
|
+
// successful `readData`, so bumping it here left an idle streaming chain
|
|
63
|
+
// reporting `erroring` until its next block arrived — and conversely, a
|
|
64
|
+
// successful poll would erase the record of a flapping producer.
|
|
65
|
+
state.producerErrors++;
|
|
66
|
+
state.lastProducerErrorMs = Date.now();
|
|
67
|
+
log.remote(
|
|
68
|
+
ComponentNames.EFFECTSTREAM_SYNC,
|
|
69
|
+
[...state.getNamespace(), "startAsync"],
|
|
70
|
+
SeverityNumber.ERROR,
|
|
71
|
+
(l) => l(result.error),
|
|
72
|
+
);
|
|
73
|
+
} else {
|
|
74
|
+
// Returning is a failure for a producer: its stream ended. Treated the
|
|
75
|
+
// same as a throw, because the observable effect is identical — no
|
|
76
|
+
// more data.
|
|
77
|
+
log.remote(
|
|
78
|
+
ComponentNames.EFFECTSTREAM_SYNC,
|
|
79
|
+
[...state.getNamespace(), "startAsync"],
|
|
80
|
+
SeverityNumber.WARN,
|
|
81
|
+
(l) => l("producer returned: stream ended, restarting"),
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
state.producerRestarts++;
|
|
86
|
+
|
|
87
|
+
// De-escalate after a producer that clearly worked. Without this `attempt`
|
|
88
|
+
// only ever grows, so a stream that ran healthily for a day between
|
|
89
|
+
// incidents would still be waiting the 30s cap on its next restart —
|
|
90
|
+
// penalising a healthy chain for ancient history. (It also keeps
|
|
91
|
+
// `2 ** attempt` from reaching Infinity after ~1024 restarts, which
|
|
92
|
+
// `Math.min` currently absorbs silently.)
|
|
93
|
+
if (ranForMs >= PRODUCER_MAX_BACKOFF_MS) {
|
|
94
|
+
attempt = 0;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const backoff = Math.min(
|
|
98
|
+
PRODUCER_MIN_BACKOFF_MS * 2 ** attempt,
|
|
99
|
+
PRODUCER_MAX_BACKOFF_MS,
|
|
100
|
+
);
|
|
101
|
+
attempt++;
|
|
102
|
+
yield* sleep(backoff);
|
|
103
|
+
}
|
|
15
104
|
});
|
|
16
105
|
|
|
17
106
|
// spawn polling task
|
|
18
107
|
yield* spawn(function* () {
|
|
19
108
|
while (true) {
|
|
109
|
+
// Liveness heartbeat: stamped every pass regardless of outcome, so a
|
|
110
|
+
// stale value means the loop is wedged inside a call that never returns
|
|
111
|
+
// rather than merely idle. See SyncState.lastPollAtMs.
|
|
112
|
+
state.lastPollAtMs = Date.now();
|
|
113
|
+
|
|
20
114
|
const inputResult = yield* tryYield(iState.stateToInput());
|
|
21
115
|
if (inputResult.error != null) {
|
|
22
116
|
state.consecutiveErrors++;
|
|
@@ -27,18 +121,14 @@ export function* startSync(
|
|
|
27
121
|
SeverityNumber.ERROR,
|
|
28
122
|
(log) => log(inputResult.error),
|
|
29
123
|
);
|
|
30
|
-
|
|
31
|
-
if ("pollingInterval" in config.syncProtocol) {
|
|
32
|
-
yield* sleep(config.syncProtocol.pollingInterval);
|
|
33
|
-
}
|
|
124
|
+
yield* sleep(state.pollingIntervalMs);
|
|
34
125
|
continue;
|
|
35
126
|
}
|
|
36
127
|
const input = inputResult.data;
|
|
37
|
-
const { config } = state.fetcher;
|
|
38
128
|
if (input == null) {
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
129
|
+
// Caught up. This is the ordinary steady state, and the pass that must
|
|
130
|
+
// never skip its sleep — see FALLBACK_POLLING_INTERVAL_MS.
|
|
131
|
+
yield* sleep(state.pollingIntervalMs);
|
|
42
132
|
continue;
|
|
43
133
|
}
|
|
44
134
|
|
|
@@ -52,9 +142,7 @@ export function* startSync(
|
|
|
52
142
|
SeverityNumber.ERROR,
|
|
53
143
|
(l) => l(result.error),
|
|
54
144
|
);
|
|
55
|
-
|
|
56
|
-
yield* sleep(config.syncProtocol.pollingInterval);
|
|
57
|
-
}
|
|
145
|
+
yield* sleep(state.pollingIntervalMs);
|
|
58
146
|
continue;
|
|
59
147
|
}
|
|
60
148
|
state.consecutiveErrors = 0;
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DEFAULT_REQUEST_TIMEOUT_MS,
|
|
3
|
+
fetchWithTimeout,
|
|
4
|
+
} from "../common/http.ts";
|
|
5
|
+
|
|
6
|
+
// ==========================
|
|
7
|
+
// Solana JSON-RPC type defs
|
|
8
|
+
// ==========================
|
|
9
|
+
|
|
10
|
+
export type SolanaBlock = {
|
|
11
|
+
blockhash: string;
|
|
12
|
+
blockTime: number | null;
|
|
13
|
+
blockHeight: number | null;
|
|
14
|
+
parentSlot: number;
|
|
15
|
+
previousBlockhash: string;
|
|
16
|
+
transactions: SolanaTransaction[];
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
export type SolanaTransaction = {
|
|
20
|
+
transaction: {
|
|
21
|
+
message: {
|
|
22
|
+
accountKeys: string[];
|
|
23
|
+
instructions: SolanaInstruction[];
|
|
24
|
+
};
|
|
25
|
+
signatures: string[];
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Null when the RPC could not decode the transaction (e.g. a version newer
|
|
29
|
+
* than `maxSupportedTransactionVersion`). Callers must null-check.
|
|
30
|
+
*/
|
|
31
|
+
meta: {
|
|
32
|
+
err: unknown | null;
|
|
33
|
+
logMessages: string[] | null;
|
|
34
|
+
preBalances: number[];
|
|
35
|
+
postBalances: number[];
|
|
36
|
+
/**
|
|
37
|
+
* Addresses a versioned (v0) transaction pulled in through an address
|
|
38
|
+
* lookup table. These are NOT in `message.accountKeys`, which carries only
|
|
39
|
+
* static keys — but `pre`/`postBalances` are indexed over the full list.
|
|
40
|
+
* See {@link resolveAccountKeys} for the ordering.
|
|
41
|
+
*/
|
|
42
|
+
loadedAddresses?: {
|
|
43
|
+
writable: string[];
|
|
44
|
+
readonly: string[];
|
|
45
|
+
} | null;
|
|
46
|
+
} | null;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* The account list `pre`/`postBalances` are indexed against: static message
|
|
51
|
+
* keys first, then lookup-table writable addresses, then lookup-table readonly
|
|
52
|
+
* ones. Legacy transactions have no `loadedAddresses`, so this is just the
|
|
53
|
+
* static keys.
|
|
54
|
+
*/
|
|
55
|
+
export function resolveAccountKeys(
|
|
56
|
+
accountKeys: string[],
|
|
57
|
+
loadedAddresses?: { writable: string[]; readonly: string[] } | null,
|
|
58
|
+
): string[] {
|
|
59
|
+
if (!loadedAddresses) return accountKeys;
|
|
60
|
+
return [
|
|
61
|
+
...accountKeys,
|
|
62
|
+
...(loadedAddresses.writable ?? []),
|
|
63
|
+
...(loadedAddresses.readonly ?? []),
|
|
64
|
+
];
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export type SolanaInstruction = {
|
|
68
|
+
programId: string;
|
|
69
|
+
accounts: string[];
|
|
70
|
+
data: string;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
// ===========
|
|
74
|
+
// RPC Client
|
|
75
|
+
// ===========
|
|
76
|
+
|
|
77
|
+
export class SolanaClient {
|
|
78
|
+
private readonly rpcUrl: string;
|
|
79
|
+
/** Per-request deadline; see `sync-protocols/common/http.ts`. */
|
|
80
|
+
private readonly requestTimeoutMs: number;
|
|
81
|
+
|
|
82
|
+
constructor(rpcUrl: string, requestTimeoutMs = DEFAULT_REQUEST_TIMEOUT_MS) {
|
|
83
|
+
this.rpcUrl = rpcUrl;
|
|
84
|
+
this.requestTimeoutMs = requestTimeoutMs;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
private async rpc<T>(
|
|
88
|
+
method: string,
|
|
89
|
+
params: unknown[] = [],
|
|
90
|
+
): Promise<T> {
|
|
91
|
+
// Bounded: a blackholed endpoint would otherwise hang readData forever,
|
|
92
|
+
// freezing block production with every health counter clean (sync
|
|
93
|
+
// CLAUDE.md finding #4).
|
|
94
|
+
const res = await fetchWithTimeout(
|
|
95
|
+
this.rpcUrl,
|
|
96
|
+
{
|
|
97
|
+
method: "POST",
|
|
98
|
+
headers: { "Content-Type": "application/json" },
|
|
99
|
+
body: JSON.stringify({
|
|
100
|
+
jsonrpc: "2.0",
|
|
101
|
+
id: 1,
|
|
102
|
+
method,
|
|
103
|
+
params,
|
|
104
|
+
}),
|
|
105
|
+
},
|
|
106
|
+
`Solana ${method}`,
|
|
107
|
+
this.requestTimeoutMs,
|
|
108
|
+
);
|
|
109
|
+
|
|
110
|
+
const json = await res.json();
|
|
111
|
+
if (json.error) {
|
|
112
|
+
const err = new Error(
|
|
113
|
+
`[Solana] RPC error [${method}]: ${json.error.message ?? JSON.stringify(json.error)}`,
|
|
114
|
+
) as Error & { rpcCode?: number };
|
|
115
|
+
// Preserve the JSON-RPC code so callers can branch on it instead of
|
|
116
|
+
// pattern-matching human-readable messages.
|
|
117
|
+
err.rpcCode = typeof json.error.code === "number" ? json.error.code : undefined;
|
|
118
|
+
throw err;
|
|
119
|
+
}
|
|
120
|
+
return json.result as T;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
async getSlot(): Promise<number> {
|
|
124
|
+
return this.rpc<number>("getSlot", [
|
|
125
|
+
{ commitment: "confirmed" },
|
|
126
|
+
]);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
async getBlock(
|
|
130
|
+
slot: number,
|
|
131
|
+
): Promise<SolanaBlock | null> {
|
|
132
|
+
try {
|
|
133
|
+
return await this.rpc<SolanaBlock | null>("getBlock", [
|
|
134
|
+
slot,
|
|
135
|
+
{
|
|
136
|
+
encoding: "json",
|
|
137
|
+
transactionDetails: "full",
|
|
138
|
+
rewards: false,
|
|
139
|
+
maxSupportedTransactionVersion: 0,
|
|
140
|
+
commitment: "confirmed",
|
|
141
|
+
},
|
|
142
|
+
]);
|
|
143
|
+
} catch (e) {
|
|
144
|
+
// A skipped slot is normal on Solana: no block was produced. Branch on the
|
|
145
|
+
// JSON-RPC code rather than the message text —
|
|
146
|
+
// -32007 SLOT_SKIPPED, -32009 LONG_TERM_STORAGE_SLOT_SKIPPED.
|
|
147
|
+
// Deliberately NOT -32004 (block not available yet): that is a transient
|
|
148
|
+
// "ask again" and must keep throwing so the fetcher retries rather than
|
|
149
|
+
// treating the slot as permanently empty.
|
|
150
|
+
const code = (e as { rpcCode?: number }).rpcCode;
|
|
151
|
+
if (code === -32007 || code === -32009) return null;
|
|
152
|
+
// Fall back to the message for RPCs that omit or remap the code.
|
|
153
|
+
if (e instanceof Error && e.message.includes("was skipped")) return null;
|
|
154
|
+
throw e;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
}
|