@effectstream/sync 0.101.1 → 0.103.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +82 -18
- package/LICENSE-APACHE +201 -0
- package/LICENSE-MIT +21 -0
- package/README.md +63 -4
- package/docs/ADDING-A-SYNC-PROTOCOL.md +16 -9
- package/package.json +8 -8
- package/src/sync-protocols/avail/AvailClient.ts +34 -6
- package/src/sync-protocols/avail/fetcher.ts +2 -0
- package/src/sync-protocols/avail/state.ts +2 -0
- package/src/sync-protocols/base/state.ts +51 -0
- package/src/sync-protocols/bitcoin/fetcher.ts +18 -10
- package/src/sync-protocols/celestia/CelestiaClient.ts +16 -2
- package/src/sync-protocols/celestia/fetcher.ts +5 -1
- package/src/sync-protocols/celestia/state.ts +5 -1
- package/src/sync-protocols/common/http.ts +77 -0
- package/src/sync-protocols/midnight/MidnightClient.ts +11 -3
- package/src/sync-protocols/midnight/fetcher.ts +18 -11
- package/src/sync-protocols/midnight/types.ts +1 -1
- package/src/sync-protocols/midnight/zswap-decoder.ts +99 -101
- package/src/sync-protocols/near/NearClient.ts +16 -6
- package/src/sync-protocols/near/fetcher.ts +5 -1
- package/src/sync-protocols/near/state.ts +5 -1
- package/src/sync-protocols/orchestration/sync.ts +101 -13
- package/src/sync-protocols/solana/SolanaClient.ts +188 -0
- package/src/sync-protocols/solana/fetcher.ts +456 -0
- package/src/sync-protocols/solana/program-logs.test.ts +113 -0
- package/src/sync-protocols/solana/program-logs.ts +84 -0
- package/src/sync-protocols/solana/read-data-retry.test.ts +153 -0
- package/src/sync-protocols/solana/read-primitives.test.ts +435 -0
- package/src/sync-protocols/solana/root-page.test.ts +66 -0
- package/src/sync-protocols/solana/state.ts +167 -0
- package/src/sync-protocols/solana/types.ts +58 -0
- package/src/sync-protocols/types.ts +2 -0
- package/src/sync-protocols/utxorpc/WatchMultiplexer.ts +52 -2
- package/src/sync-protocols/utxorpc/state.ts +3 -0
- package/src/syncProtocolFactory.ts +15 -0
- package/test/poll-loop-spin.test.ts +112 -0
- package/test/rpc-timeout.test.ts +150 -0
- package/test/spin-probe.ts +102 -0
- package/test/start-async-supervision.test.ts +139 -0
- package/test/zswap-decoder.test.ts +74 -0
|
@@ -69,11 +69,62 @@ export abstract class SyncState<
|
|
|
69
69
|
public lastPage: undefined | LastPage<Page, RootPage>;
|
|
70
70
|
/** Timestamp of the last successful fetch (readData completed without error). */
|
|
71
71
|
public lastSuccessfulFetchMs: number = 0;
|
|
72
|
+
/**
|
|
73
|
+
* Wall-clock time of the last completed pass through the fetch loop, whatever
|
|
74
|
+
* its outcome (fetched, caught up, or errored).
|
|
75
|
+
*
|
|
76
|
+
* This — not `lastSuccessfulFetchMs` — is the hang detector. A chain that is
|
|
77
|
+
* caught up never calls `readData`, so its last-successful-fetch time is
|
|
78
|
+
* indistinguishable from that of a chain wedged inside a request that will
|
|
79
|
+
* never return. The loop still cycles in the first case and does not in the
|
|
80
|
+
* second, so a heartbeat that is stale by many polling intervals means the
|
|
81
|
+
* loop is stuck in `stateToInput`/`readData`.
|
|
82
|
+
*/
|
|
83
|
+
public lastPollAtMs: number = 0;
|
|
72
84
|
/** Number of consecutive errors since the last successful fetch. */
|
|
73
85
|
public consecutiveErrors: number = 0;
|
|
74
86
|
/** Timestamp of the most recent error, or 0 if no error has occurred. */
|
|
75
87
|
public lastErrorTimestamp: number = 0;
|
|
76
88
|
|
|
89
|
+
// ── Streaming producer (see startAsync + orchestration/sync.ts) ──
|
|
90
|
+
/**
|
|
91
|
+
* Whether {@link SyncState#startAsync} runs a long-lived producer that this
|
|
92
|
+
* protocol depends on for its data (a subscription/stream), as opposed to the
|
|
93
|
+
* base-class no-op used by polled chains.
|
|
94
|
+
*
|
|
95
|
+
* `startSync` supervises and restarts the producer only when this is true.
|
|
96
|
+
* A polled chain must leave it false, or its no-op `startAsync` would be
|
|
97
|
+
* "restarted" forever.
|
|
98
|
+
*/
|
|
99
|
+
public readonly hasAsyncProducer: boolean = false;
|
|
100
|
+
/**
|
|
101
|
+
* How many times the streaming producer has been restarted after dying —
|
|
102
|
+
* either by throwing or by returning (a producer that returns has ended its
|
|
103
|
+
* stream). Surfaced on `/health`; a climbing count means a flapping upstream.
|
|
104
|
+
*/
|
|
105
|
+
public producerRestarts: number = 0;
|
|
106
|
+
/**
|
|
107
|
+
* Producer failures, tracked separately from {@link consecutiveErrors}.
|
|
108
|
+
*
|
|
109
|
+
* That counter belongs to the fetch loop and is only cleared by a successful
|
|
110
|
+
* `readData`, so sharing it conflated two independent signals: an idle
|
|
111
|
+
* streaming chain would report `erroring` long after its producer recovered,
|
|
112
|
+
* and a successful poll would erase the record of a flapping producer.
|
|
113
|
+
*/
|
|
114
|
+
public producerErrors: number = 0;
|
|
115
|
+
/** Wall-clock time of the last producer failure, or 0 if none. */
|
|
116
|
+
public lastProducerErrorMs: number = 0;
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Resolved polling interval for this protocol, stamped by `startSync`.
|
|
120
|
+
*
|
|
121
|
+
* Single source of truth: `/health` derives its wedged threshold from this
|
|
122
|
+
* rather than reaching back into `fetcher.config.syncProtocol`, which it can
|
|
123
|
+
* only do through an unchecked cast that would fail silently to a default if
|
|
124
|
+
* the config shape ever moved.
|
|
125
|
+
*/
|
|
126
|
+
public pollingIntervalMs: number = 0;
|
|
127
|
+
|
|
77
128
|
// ── Backpressure observability (see common/page-helpers.ts + README) ──
|
|
78
129
|
/** Resolved fetch cap (`maxBufferedPages`); set on each backpressure check, 0 until the first. */
|
|
79
130
|
public bufferCap: number = 0;
|
|
@@ -27,6 +27,7 @@ import type {
|
|
|
27
27
|
BitcoinTxId,
|
|
28
28
|
} from "@effectstream/utils";
|
|
29
29
|
import { blockNumberRelation } from "../common/utils.ts";
|
|
30
|
+
import { DEFAULT_REQUEST_TIMEOUT_MS, fetchWithTimeout } from "../common/http.ts";
|
|
30
31
|
import type { ConfigType } from "./types.ts";
|
|
31
32
|
|
|
32
33
|
const SATOSHIS_PER_BTC = 100_000_000;
|
|
@@ -457,6 +458,8 @@ type BitcoinRpcClientOptions = {
|
|
|
457
458
|
url: string;
|
|
458
459
|
username?: string | null;
|
|
459
460
|
password?: string | null;
|
|
461
|
+
/** Per-request deadline; defaults to {@link DEFAULT_REQUEST_TIMEOUT_MS}. */
|
|
462
|
+
requestTimeoutMs?: number;
|
|
460
463
|
};
|
|
461
464
|
|
|
462
465
|
type BitcoinRpcRequest = {
|
|
@@ -515,16 +518,21 @@ export class BitcoinRpcClient {
|
|
|
515
518
|
private async call<TResult>(
|
|
516
519
|
request: BitcoinRpcRequest,
|
|
517
520
|
): Promise<TResult> {
|
|
518
|
-
const response = await
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
521
|
+
const response = await fetchWithTimeout(
|
|
522
|
+
this.options.url,
|
|
523
|
+
{
|
|
524
|
+
method: "POST",
|
|
525
|
+
headers: this.buildHeaders(),
|
|
526
|
+
body: JSON.stringify({
|
|
527
|
+
jsonrpc: "2.0",
|
|
528
|
+
id: ++this.nextId,
|
|
529
|
+
method: request.method,
|
|
530
|
+
params: request.params ?? [],
|
|
531
|
+
}),
|
|
532
|
+
},
|
|
533
|
+
`Bitcoin ${request.method}`,
|
|
534
|
+
this.options.requestTimeoutMs,
|
|
535
|
+
);
|
|
528
536
|
if (!response.ok) {
|
|
529
537
|
const body = await response.text();
|
|
530
538
|
throw new Error(
|
|
@@ -54,6 +54,11 @@ const CELESTIA_NODE_URL = ENV.getString(
|
|
|
54
54
|
"http://localhost:26658",
|
|
55
55
|
);
|
|
56
56
|
const CELESTIA_AUTH_TOKEN = ENV.getString("CELESTIA_AUTH_TOKEN", "");
|
|
57
|
+
/**
|
|
58
|
+
* Fallback per-request deadline, kept for compatibility with deployments that
|
|
59
|
+
* set it. The protocol config's `requestTimeoutMs` takes precedence — see the
|
|
60
|
+
* constructor.
|
|
61
|
+
*/
|
|
57
62
|
const CELESTIA_RPC_TIMEOUT_MS = parseInt(
|
|
58
63
|
ENV.getString("CELESTIA_RPC_TIMEOUT_MS", "15000"),
|
|
59
64
|
);
|
|
@@ -77,9 +82,18 @@ export function celestiaNamespaceToBase64(hex: string): string {
|
|
|
77
82
|
|
|
78
83
|
export class CelestiaClient {
|
|
79
84
|
private readonly rpcUrl: string;
|
|
85
|
+
/**
|
|
86
|
+
* Per-attempt deadline. Unlike the other clients this one retries internally
|
|
87
|
+
* (see `rpc` below), so this bounds each attempt rather than the whole call.
|
|
88
|
+
*/
|
|
89
|
+
private readonly requestTimeoutMs: number;
|
|
80
90
|
|
|
81
|
-
constructor(
|
|
91
|
+
constructor(
|
|
92
|
+
rpcUrl: string = CELESTIA_NODE_URL,
|
|
93
|
+
requestTimeoutMs: number = CELESTIA_RPC_TIMEOUT_MS,
|
|
94
|
+
) {
|
|
82
95
|
this.rpcUrl = rpcUrl;
|
|
96
|
+
this.requestTimeoutMs = requestTimeoutMs;
|
|
83
97
|
}
|
|
84
98
|
|
|
85
99
|
private async rpc<T>(method: string, params: unknown[]): Promise<T | null> {
|
|
@@ -95,7 +109,7 @@ export class CelestiaClient {
|
|
|
95
109
|
method: "POST",
|
|
96
110
|
headers,
|
|
97
111
|
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
|
|
98
|
-
signal: AbortSignal.timeout(
|
|
112
|
+
signal: AbortSignal.timeout(this.requestTimeoutMs),
|
|
99
113
|
});
|
|
100
114
|
const json = await res.json();
|
|
101
115
|
if (json.error) {
|
|
@@ -2,6 +2,7 @@ import {
|
|
|
2
2
|
ConfigSyncProtocolType,
|
|
3
3
|
type PrimitiveEntry,
|
|
4
4
|
} from "@effectstream/config";
|
|
5
|
+
import { requestTimeoutOf } from "../common/http.ts";
|
|
5
6
|
import { BaseDataFetcher } from "../base/fetcher.ts";
|
|
6
7
|
import type { DataFetched } from "../base/fetcher.ts";
|
|
7
8
|
import type {
|
|
@@ -37,7 +38,10 @@ export class CelestiaFetcher extends BaseDataFetcher<
|
|
|
37
38
|
readonly config: ConfigType,
|
|
38
39
|
) {
|
|
39
40
|
super(config.syncProtocol.name);
|
|
40
|
-
this.client = new CelestiaClient(
|
|
41
|
+
this.client = new CelestiaClient(
|
|
42
|
+
config.network.rpcUrl,
|
|
43
|
+
requestTimeoutOf(config.syncProtocol),
|
|
44
|
+
);
|
|
41
45
|
}
|
|
42
46
|
|
|
43
47
|
@bound
|
|
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
|
|
|
10
10
|
import { getPage } from "@effectstream/db";
|
|
11
11
|
import { CelestiaClient } from "./CelestiaClient.ts";
|
|
12
12
|
import { applyDelay } from "../common/utils.ts";
|
|
13
|
+
import { requestTimeoutOf } from "../common/http.ts";
|
|
13
14
|
import { bufferAtCap } from "../common/page-helpers.ts";
|
|
14
15
|
|
|
15
16
|
export class CelestiaSyncState extends SyncState<
|
|
@@ -135,7 +136,10 @@ export class CelestiaSyncState extends SyncState<
|
|
|
135
136
|
page,
|
|
136
137
|
config,
|
|
137
138
|
fetcher,
|
|
138
|
-
new CelestiaClient(
|
|
139
|
+
new CelestiaClient(
|
|
140
|
+
config.network.rpcUrl,
|
|
141
|
+
requestTimeoutOf(config.syncProtocol),
|
|
142
|
+
),
|
|
139
143
|
dbConn,
|
|
140
144
|
);
|
|
141
145
|
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Bounded HTTP for sync-protocol RPC clients.
|
|
3
|
+
*
|
|
4
|
+
* `fetch` has no default timeout. Against a blackholed endpoint — a load
|
|
5
|
+
* balancer that drops its backend, a socket left half-open by a NAT rebind —
|
|
6
|
+
* the connection is established and simply never answers, so the promise never
|
|
7
|
+
* settles. For a sync protocol that is the worst possible failure: `readData`
|
|
8
|
+
* never returns, so the fetch loop in `orchestration/sync.ts` never reaches its
|
|
9
|
+
* `catch`, `consecutiveErrors` stays 0, `lastSuccessfulFetchMs` freezes, and
|
|
10
|
+
* the merge blocks on that chain's page forever. The node stops producing
|
|
11
|
+
* blocks and reports nothing wrong.
|
|
12
|
+
*
|
|
13
|
+
* Deliberately timeout-only, with NO internal retry. The fetch loop already
|
|
14
|
+
* retries: it catches, counts the error, sleeps `pollingInterval` and re-runs
|
|
15
|
+
* the same page range (fetches are idempotent — `lastPage` only advances on a
|
|
16
|
+
* complete `DataFetched`). Retrying inside the client as well would double the
|
|
17
|
+
* backoff and, worse, hide failures from `consecutiveErrors`, which is what the
|
|
18
|
+
* health endpoint reports on.
|
|
19
|
+
*
|
|
20
|
+
* Regression test: `sync/test/rpc-timeout.test.ts` points each client at a
|
|
21
|
+
* blackhole and asserts it rejects rather than hanging.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
/** Used when a protocol config carries no `requestTimeoutMs`. */
|
|
25
|
+
export const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;
|
|
26
|
+
|
|
27
|
+
/** Thrown when a request exceeds its budget, so callers can tell it apart. */
|
|
28
|
+
export class RequestTimeoutError extends Error {
|
|
29
|
+
constructor(
|
|
30
|
+
readonly label: string,
|
|
31
|
+
readonly timeoutMs: number,
|
|
32
|
+
) {
|
|
33
|
+
super(`[${label}] request timed out after ${timeoutMs}ms`);
|
|
34
|
+
this.name = "RequestTimeoutError";
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* `fetch` with a hard deadline.
|
|
40
|
+
*
|
|
41
|
+
* @param label Prefix for the thrown error, e.g. `"Bitcoin getblockhash"`.
|
|
42
|
+
* @param timeoutMs Falls back to {@link DEFAULT_REQUEST_TIMEOUT_MS}.
|
|
43
|
+
* @param signal Optional caller signal; aborting either one aborts the request.
|
|
44
|
+
*/
|
|
45
|
+
export async function fetchWithTimeout(
|
|
46
|
+
url: string,
|
|
47
|
+
init: RequestInit,
|
|
48
|
+
label: string,
|
|
49
|
+
timeoutMs: number = DEFAULT_REQUEST_TIMEOUT_MS,
|
|
50
|
+
signal?: AbortSignal,
|
|
51
|
+
): Promise<Response> {
|
|
52
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
53
|
+
const combined = signal
|
|
54
|
+
? AbortSignal.any([signal, timeoutSignal])
|
|
55
|
+
: timeoutSignal;
|
|
56
|
+
|
|
57
|
+
try {
|
|
58
|
+
return await fetch(url, { ...init, signal: combined });
|
|
59
|
+
} catch (err) {
|
|
60
|
+
// Surface our own error for the timeout case; a caller-supplied abort and
|
|
61
|
+
// ordinary transport failures propagate unchanged.
|
|
62
|
+
if (timeoutSignal.aborted && !(signal?.aborted ?? false)) {
|
|
63
|
+
throw new RequestTimeoutError(label, timeoutMs);
|
|
64
|
+
}
|
|
65
|
+
throw err;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Resolve the timeout for a protocol config. Optional in the schema, so this
|
|
71
|
+
* centralises the fallback instead of repeating `?? DEFAULT` at each client.
|
|
72
|
+
*/
|
|
73
|
+
export function requestTimeoutOf(
|
|
74
|
+
syncProtocol: { requestTimeoutMs?: number } | undefined,
|
|
75
|
+
): number {
|
|
76
|
+
return syncProtocol?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
77
|
+
}
|
|
@@ -30,11 +30,19 @@ export interface MidnightGqlBlockState {
|
|
|
30
30
|
intentHash: string;
|
|
31
31
|
outputIndex: number;
|
|
32
32
|
owner: string;
|
|
33
|
+
/** u128 value as a decimal string */
|
|
34
|
+
value: string;
|
|
35
|
+
/** hex-encoded serialized token type */
|
|
36
|
+
tokenType: string;
|
|
33
37
|
}[];
|
|
34
38
|
unshieldedCreatedOutputs?: {
|
|
35
39
|
intentHash: string;
|
|
36
40
|
outputIndex: number;
|
|
37
41
|
owner: string;
|
|
42
|
+
/** u128 value as a decimal string */
|
|
43
|
+
value: string;
|
|
44
|
+
/** hex-encoded serialized token type */
|
|
45
|
+
tokenType: string;
|
|
38
46
|
}[];
|
|
39
47
|
// Present only on RegularTransaction (selected via inline fragment): the
|
|
40
48
|
// coin-commitment Merkle tree root after this transaction.
|
|
@@ -54,7 +62,7 @@ export interface MidnightGqlBlockState {
|
|
|
54
62
|
export interface BlockFetchOptions {
|
|
55
63
|
/** Include contractActions in the transaction fields (needed for MidnightGenericPrimitive). Default: true */
|
|
56
64
|
contractActions?: boolean;
|
|
57
|
-
/** Include zswapLedgerEvents in the transaction fields (needed for
|
|
65
|
+
/** Include zswapLedgerEvents in the transaction fields (needed for MidnightNullifierAndCommitmentPrimitive). Default: true */
|
|
58
66
|
zswapLedgerEvents?: boolean;
|
|
59
67
|
/** Include unshieldedSpentOutputs (needed for MidnightUnshieldedSpendPrimitive). Default: false */
|
|
60
68
|
unshieldedSpentOutputs?: boolean;
|
|
@@ -150,10 +158,10 @@ export class MidnightClient {
|
|
|
150
158
|
? `zswapLedgerEvents { id raw maxId }`
|
|
151
159
|
: "";
|
|
152
160
|
const unshieldedSpentField = unshieldedSpentOutputs
|
|
153
|
-
? `unshieldedSpentOutputs { intentHash outputIndex owner }`
|
|
161
|
+
? `unshieldedSpentOutputs { intentHash outputIndex owner value tokenType }`
|
|
154
162
|
: "";
|
|
155
163
|
const unshieldedCreatedField = unshieldedCreatedOutputs
|
|
156
|
-
? `unshieldedCreatedOutputs { intentHash outputIndex owner }`
|
|
164
|
+
? `unshieldedCreatedOutputs { intentHash outputIndex owner value tokenType }`
|
|
157
165
|
: "";
|
|
158
166
|
// zswapMerkleTreeRoot lives on RegularTransaction, not the Transaction
|
|
159
167
|
// interface, so it must be selected through an inline fragment.
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { BaseDataFetcher } from "../base/fetcher.ts";
|
|
2
|
+
import { requestTimeoutOf } from "../common/http.ts";
|
|
2
3
|
import type {
|
|
3
4
|
Block,
|
|
4
5
|
ConfigType,
|
|
@@ -18,7 +19,7 @@ import type { RootOutput, RootPage } from "../types.ts";
|
|
|
18
19
|
import { bound } from "@effectstream/utils";
|
|
19
20
|
import { MidnightClient, type BlockFetchOptions, type MidnightGqlBlockState } from "./MidnightClient.ts";
|
|
20
21
|
import { ContractState, StateValue } from "@midnight-ntwrk/onchain-runtime";
|
|
21
|
-
import {
|
|
22
|
+
import { decodeZswapEvent } from "./zswap-decoder.ts";
|
|
22
23
|
import { decodeTokenMints } from "./mint-decoder.ts";
|
|
23
24
|
|
|
24
25
|
export class MidnightFetcher extends BaseDataFetcher<
|
|
@@ -59,6 +60,7 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
59
60
|
this.client = new MidnightClient(
|
|
60
61
|
indexerHttp,
|
|
61
62
|
this.networkId,
|
|
63
|
+
requestTimeoutOf(config.syncProtocol),
|
|
62
64
|
);
|
|
63
65
|
}
|
|
64
66
|
|
|
@@ -76,14 +78,14 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
76
78
|
const blockFetchOptions: BlockFetchOptions = {
|
|
77
79
|
contractActions: this.config.primitives.some(
|
|
78
80
|
(p) =>
|
|
79
|
-
p.primitive.type !== "Midnight:
|
|
81
|
+
p.primitive.type !== "Midnight:NullifierAndCommitment" &&
|
|
80
82
|
p.primitive.type !== "Midnight:UnshieldedSpend" &&
|
|
81
83
|
p.primitive.type !== "Midnight:UnshieldedCreate" &&
|
|
82
84
|
p.primitive.type !== "Midnight:ZswapRoot" &&
|
|
83
85
|
p.primitive.type !== "Midnight:TokenMint",
|
|
84
86
|
),
|
|
85
87
|
zswapLedgerEvents: this.config.primitives.some(
|
|
86
|
-
(p) => p.primitive.type === "Midnight:
|
|
88
|
+
(p) => p.primitive.type === "Midnight:NullifierAndCommitment",
|
|
87
89
|
),
|
|
88
90
|
unshieldedSpentOutputs: this.config.primitives.some(
|
|
89
91
|
(p) => p.primitive.type === "Midnight:UnshieldedSpend",
|
|
@@ -181,8 +183,8 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
181
183
|
const syncResults: PrimitiveType[] = [];
|
|
182
184
|
|
|
183
185
|
for (const primitiveEntry of primitiveEntries) {
|
|
184
|
-
if (primitiveEntry.primitive.type === "Midnight:
|
|
185
|
-
syncResults.push(...this.
|
|
186
|
+
if (primitiveEntry.primitive.type === "Midnight:NullifierAndCommitment") {
|
|
187
|
+
syncResults.push(...this.fetchZswapEvents(height, primitiveEntry, block));
|
|
186
188
|
} else if (primitiveEntry.primitive.type === "Midnight:UnshieldedSpend") {
|
|
187
189
|
syncResults.push(...this.fetchUnshieldedSpends(height, primitiveEntry, block));
|
|
188
190
|
} else if (primitiveEntry.primitive.type === "Midnight:UnshieldedCreate") {
|
|
@@ -203,16 +205,19 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
203
205
|
}
|
|
204
206
|
|
|
205
207
|
@bound
|
|
206
|
-
|
|
208
|
+
fetchZswapEvents(
|
|
207
209
|
height: number,
|
|
208
210
|
primitiveEntry: PrimitiveEntryType,
|
|
209
211
|
block: MidnightGqlBlockState,
|
|
210
212
|
): PrimitiveType[] {
|
|
213
|
+
const capture = primitiveEntry.primitive.capture ?? "both";
|
|
211
214
|
const results: PrimitiveType[] = [];
|
|
212
215
|
for (const tx of block.block.transactions) {
|
|
213
216
|
for (const event of tx.zswapLedgerEvents ?? []) {
|
|
214
|
-
const decoded =
|
|
217
|
+
const decoded = decodeZswapEvent(event.raw);
|
|
215
218
|
if (!decoded) continue;
|
|
219
|
+
if (decoded.kind === "nullifier" && capture === "commitments") continue;
|
|
220
|
+
if (decoded.kind === "commitment" && capture === "nullifiers") continue;
|
|
216
221
|
results.push({
|
|
217
222
|
syncProtocol: {
|
|
218
223
|
name: primitiveEntry.syncProtocol,
|
|
@@ -222,12 +227,10 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
222
227
|
},
|
|
223
228
|
primitive: primitiveEntry.primitive.name,
|
|
224
229
|
output: {
|
|
225
|
-
payloadType:
|
|
230
|
+
payloadType: `midnight-${decoded.kind}`,
|
|
226
231
|
payload: {
|
|
227
|
-
|
|
228
|
-
txHash: decoded.txHash,
|
|
232
|
+
...decoded,
|
|
229
233
|
eventId: event.id,
|
|
230
|
-
logicalSegment: decoded.logicalSegment,
|
|
231
234
|
},
|
|
232
235
|
},
|
|
233
236
|
});
|
|
@@ -259,6 +262,8 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
259
262
|
owner: spend.owner,
|
|
260
263
|
intentHash: spend.intentHash,
|
|
261
264
|
outputIndex: spend.outputIndex,
|
|
265
|
+
value: spend.value,
|
|
266
|
+
tokenType: spend.tokenType,
|
|
262
267
|
txHash: tx.hash,
|
|
263
268
|
},
|
|
264
269
|
},
|
|
@@ -294,6 +299,8 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
294
299
|
owner: created.owner,
|
|
295
300
|
intentHash: created.intentHash,
|
|
296
301
|
outputIndex: created.outputIndex,
|
|
302
|
+
value: created.value,
|
|
303
|
+
tokenType: created.tokenType,
|
|
297
304
|
txHash: tx.hash,
|
|
298
305
|
},
|
|
299
306
|
},
|
|
@@ -35,7 +35,7 @@ export type CommonFunnelArgs = {
|
|
|
35
35
|
// Graphql Block
|
|
36
36
|
// =============
|
|
37
37
|
|
|
38
|
-
// Interfaces approximated from GraphQL schema served by indexer
|
|
38
|
+
// Interfaces approximated from GraphQL schema served by indexer v4.3.3 (/api/v3)
|
|
39
39
|
|
|
40
40
|
interface ContractCallOrDeploy {
|
|
41
41
|
address: MidnightAddress;
|
|
@@ -1,58 +1,26 @@
|
|
|
1
|
-
import { Buffer } from "node:buffer";
|
|
2
1
|
import * as ledger from '@midnight-ntwrk/ledger-v8';
|
|
3
2
|
|
|
4
|
-
//
|
|
5
|
-
|
|
6
|
-
try {
|
|
7
|
-
const bytes = Uint8Array.from(
|
|
8
|
-
Buffer.from(rawHex.replace(/^0x/, ""), "hex"),
|
|
9
|
-
);
|
|
10
|
-
const event = ledger.Event.deserialize(bytes);
|
|
11
|
-
console.log("event", event);
|
|
12
|
-
// if (event.type === "ZswapInput") {
|
|
13
|
-
// event.nullifier — Uint8Array (32 bytes)
|
|
14
|
-
// event.source.transactionHash
|
|
15
|
-
// event.source.logicalSegment
|
|
16
|
-
// } else if (event.type === "ZswapOutput") {
|
|
17
|
-
// event.commitment, etc.
|
|
18
|
-
// }
|
|
19
|
-
} catch {
|
|
20
|
-
return null;
|
|
21
|
-
}
|
|
22
|
-
return null;
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
// ─── SCALE compact integer decoder ───────────────────────────────────────────
|
|
3
|
+
// Wire format of a raw zswap ledger event, for reference (midnight-ledger 8.x,
|
|
4
|
+
// `midnight-ledger/ledger/src/events.rs` + `storage-core/src/arena.rs`):
|
|
26
5
|
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
6
|
+
// 19 bytes outer tag "midnight:event[v9]:"
|
|
7
|
+
// then a TopoSortedNodes envelope (NOT the struct directly):
|
|
8
|
+
// compact_u32 node count, and per node: compact_u32 child count,
|
|
9
|
+
// compact_u64 child indices, compact_u32 data length, data bytes.
|
|
10
|
+
// The root node's data is:
|
|
11
|
+
// 32 bytes EventSource.transaction_hash
|
|
12
|
+
// 2 bytes logical_segment (LE u16)
|
|
13
|
+
// 2 bytes physical_segment (LE u16)
|
|
14
|
+
// 1 byte EventDetails variant: 0=ZswapInput, 1=ZswapOutput, 2+=other
|
|
15
|
+
// ZswapInput: 32-byte nullifier + Option<Sp<ContractAddress>> byte
|
|
16
|
+
// ZswapOutput: 32-byte commitment + variable-length preimage evidence
|
|
17
|
+
// + Option<Sp<ContractAddress>> byte + compact_u64 mt_index
|
|
18
|
+
// A Some(..) contract address is a SEPARATE node in the envelope, shifting
|
|
19
|
+
// all offsets — so positional parsing is not viable; always deserialize via
|
|
20
|
+
// the ledger bindings.
|
|
32
21
|
|
|
33
|
-
function
|
|
34
|
-
|
|
35
|
-
const mode = first & 0b11;
|
|
36
|
-
if (mode === 0b00) {
|
|
37
|
-
return [(first & 0xFC) >> 2, pos + 1];
|
|
38
|
-
} else if (mode === 0b01) {
|
|
39
|
-
const second = b[pos + 1];
|
|
40
|
-
const v0 = ((first & 0xFC) >> 2) | ((second & 0x03) << 6);
|
|
41
|
-
const v1 = (second & 0xFC) >> 2;
|
|
42
|
-
return [(v0 | (v1 << 8)) >>> 0, pos + 2];
|
|
43
|
-
} else if (mode === 0b10) {
|
|
44
|
-
const s = b[pos + 1], t = b[pos + 2], u = b[pos + 3];
|
|
45
|
-
const v0 = ((first & 0xFC) >> 2) | ((s & 0x03) << 6);
|
|
46
|
-
const v1 = ((s & 0xFC) >> 2) | ((t & 0x03) << 6);
|
|
47
|
-
const v2 = ((t & 0xFC) >> 2) | ((u & 0x03) << 6);
|
|
48
|
-
const v3 = (u & 0xFC) >> 2;
|
|
49
|
-
return [(v0 | (v1 << 8) | (v2 << 16) | (v3 << 24)) >>> 0, pos + 4];
|
|
50
|
-
} else {
|
|
51
|
-
const n = ((first & 0xFC) >> 2) + 4;
|
|
52
|
-
let val = 0;
|
|
53
|
-
for (let i = 0; i < n; i++) val |= b[pos + 1 + i] << (i * 8);
|
|
54
|
-
return [val >>> 0, pos + 1 + n];
|
|
55
|
-
}
|
|
22
|
+
function normalizeHex(hex: string): string {
|
|
23
|
+
return (hex.startsWith("0x") ? hex.slice(2) : hex).toLowerCase();
|
|
56
24
|
}
|
|
57
25
|
|
|
58
26
|
function hexToBytes(hex: string): Uint8Array {
|
|
@@ -64,60 +32,90 @@ function hexToBytes(hex: string): Uint8Array {
|
|
|
64
32
|
return arr;
|
|
65
33
|
}
|
|
66
34
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
35
|
+
/** A zswap ledger event decoded via the native ledger-v8 bindings. */
|
|
36
|
+
export type DecodedZswapEvent =
|
|
37
|
+
| {
|
|
38
|
+
kind: "nullifier";
|
|
39
|
+
/** 32-byte nullifier, hex */
|
|
40
|
+
nullifier: string;
|
|
41
|
+
/** 32-byte transaction hash, hex */
|
|
42
|
+
txHash: string;
|
|
43
|
+
/** Logical segment index */
|
|
44
|
+
logicalSegment: number;
|
|
45
|
+
/** Contract address that spent the coin, hex — absent for user coins */
|
|
46
|
+
contract?: string;
|
|
47
|
+
}
|
|
48
|
+
| {
|
|
49
|
+
kind: "commitment";
|
|
50
|
+
/** 32-byte coin commitment, hex */
|
|
51
|
+
commitment: string;
|
|
52
|
+
/** Index of the commitment in the zswap Merkle tree, u64 as decimal string */
|
|
53
|
+
mtIndex: string;
|
|
54
|
+
/** 32-byte transaction hash, hex */
|
|
55
|
+
txHash: string;
|
|
56
|
+
/** Logical segment index */
|
|
57
|
+
logicalSegment: number;
|
|
58
|
+
/** Contract address that received the coin, hex — absent for user coins */
|
|
59
|
+
contract?: string;
|
|
60
|
+
};
|
|
90
61
|
|
|
91
62
|
/**
|
|
92
|
-
* Decode a raw
|
|
93
|
-
*
|
|
94
|
-
*
|
|
63
|
+
* Decode a raw zswap ledger event (ZswapInput or ZswapOutput) using the
|
|
64
|
+
* native ledger-v8 WASM deserializer. The indexer's `zswapLedgerEvents`
|
|
65
|
+
* field only ever carries these two variants, so a `null` return means the
|
|
66
|
+
* event could not be decoded (e.g. a ledger event-format version bump) and
|
|
67
|
+
* is logged.
|
|
95
68
|
*/
|
|
96
|
-
export function
|
|
97
|
-
|
|
98
|
-
): DecodedZswapInput | null {
|
|
69
|
+
export function decodeZswapEvent(rawHex: string): DecodedZswapEvent | null {
|
|
70
|
+
let event: ledger.Event;
|
|
99
71
|
try {
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
pos += 2;
|
|
107
|
-
pos += 2; // skip physical segment
|
|
108
|
-
|
|
109
|
-
const [variant, p1] = readScaleU32(b, pos);
|
|
110
|
-
pos = p1;
|
|
111
|
-
|
|
112
|
-
if (variant !== 0) {
|
|
113
|
-
return null; // ZswapOutput or other — not a nullifier event
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
pos += 4; // skip 4-byte contract_id
|
|
117
|
-
|
|
118
|
-
const nullifier = bytesToHex(b.slice(pos, pos + 32));
|
|
119
|
-
return { txHash, logicalSegment, nullifier };
|
|
120
|
-
} catch {
|
|
72
|
+
event = ledger.Event.deserialize(hexToBytes(rawHex));
|
|
73
|
+
} catch (err) {
|
|
74
|
+
console.warn(
|
|
75
|
+
"[midnight] failed to deserialize zswap ledger event — ledger event format may have changed",
|
|
76
|
+
{ raw: rawHex, error: String(err) },
|
|
77
|
+
);
|
|
121
78
|
return null;
|
|
122
79
|
}
|
|
80
|
+
// EventDetails is an open union ({ tag: string } catch-all), so narrow manually.
|
|
81
|
+
const content = event.content as {
|
|
82
|
+
tag: string;
|
|
83
|
+
nullifier?: string;
|
|
84
|
+
commitment?: string;
|
|
85
|
+
contract?: string;
|
|
86
|
+
mtIndex?: bigint;
|
|
87
|
+
};
|
|
88
|
+
const source = {
|
|
89
|
+
txHash: normalizeHex(event.source.transactionHash),
|
|
90
|
+
logicalSegment: event.source.logicalSegment,
|
|
91
|
+
};
|
|
92
|
+
const contract = content.contract != null
|
|
93
|
+
? { contract: normalizeHex(content.contract) }
|
|
94
|
+
: {};
|
|
95
|
+
if (content.tag === "zswapInput" && content.nullifier != null) {
|
|
96
|
+
return {
|
|
97
|
+
kind: "nullifier",
|
|
98
|
+
nullifier: normalizeHex(content.nullifier),
|
|
99
|
+
...contract,
|
|
100
|
+
...source,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
if (
|
|
104
|
+
content.tag === "zswapOutput" &&
|
|
105
|
+
content.commitment != null &&
|
|
106
|
+
content.mtIndex != null
|
|
107
|
+
) {
|
|
108
|
+
return {
|
|
109
|
+
kind: "commitment",
|
|
110
|
+
commitment: normalizeHex(content.commitment),
|
|
111
|
+
mtIndex: content.mtIndex.toString(),
|
|
112
|
+
...contract,
|
|
113
|
+
...source,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
console.warn(
|
|
117
|
+
"[midnight] unexpected zswap ledger event variant",
|
|
118
|
+
{ raw: rawHex, tag: content.tag },
|
|
119
|
+
);
|
|
120
|
+
return null;
|
|
123
121
|
}
|