@effectstream/sync 0.101.1 → 0.103.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CLAUDE.md +82 -18
  2. package/LICENSE-APACHE +201 -0
  3. package/LICENSE-MIT +21 -0
  4. package/README.md +63 -4
  5. package/docs/ADDING-A-SYNC-PROTOCOL.md +16 -9
  6. package/package.json +8 -8
  7. package/src/sync-protocols/avail/AvailClient.ts +34 -6
  8. package/src/sync-protocols/avail/fetcher.ts +2 -0
  9. package/src/sync-protocols/avail/state.ts +2 -0
  10. package/src/sync-protocols/base/state.ts +51 -0
  11. package/src/sync-protocols/bitcoin/fetcher.ts +18 -10
  12. package/src/sync-protocols/celestia/CelestiaClient.ts +16 -2
  13. package/src/sync-protocols/celestia/fetcher.ts +5 -1
  14. package/src/sync-protocols/celestia/state.ts +5 -1
  15. package/src/sync-protocols/common/http.ts +77 -0
  16. package/src/sync-protocols/midnight/MidnightClient.ts +11 -3
  17. package/src/sync-protocols/midnight/fetcher.ts +18 -11
  18. package/src/sync-protocols/midnight/types.ts +1 -1
  19. package/src/sync-protocols/midnight/zswap-decoder.ts +99 -101
  20. package/src/sync-protocols/near/NearClient.ts +16 -6
  21. package/src/sync-protocols/near/fetcher.ts +5 -1
  22. package/src/sync-protocols/near/state.ts +5 -1
  23. package/src/sync-protocols/orchestration/sync.ts +101 -13
  24. package/src/sync-protocols/solana/SolanaClient.ts +188 -0
  25. package/src/sync-protocols/solana/fetcher.ts +456 -0
  26. package/src/sync-protocols/solana/program-logs.test.ts +113 -0
  27. package/src/sync-protocols/solana/program-logs.ts +84 -0
  28. package/src/sync-protocols/solana/read-data-retry.test.ts +153 -0
  29. package/src/sync-protocols/solana/read-primitives.test.ts +435 -0
  30. package/src/sync-protocols/solana/root-page.test.ts +66 -0
  31. package/src/sync-protocols/solana/state.ts +167 -0
  32. package/src/sync-protocols/solana/types.ts +58 -0
  33. package/src/sync-protocols/types.ts +2 -0
  34. package/src/sync-protocols/utxorpc/WatchMultiplexer.ts +52 -2
  35. package/src/sync-protocols/utxorpc/state.ts +3 -0
  36. package/src/syncProtocolFactory.ts +15 -0
  37. package/test/poll-loop-spin.test.ts +112 -0
  38. package/test/rpc-timeout.test.ts +150 -0
  39. package/test/spin-probe.ts +102 -0
  40. package/test/start-async-supervision.test.ts +139 -0
  41. package/test/zswap-decoder.test.ts +74 -0
@@ -69,11 +69,62 @@ export abstract class SyncState<
69
69
  public lastPage: undefined | LastPage<Page, RootPage>;
70
70
  /** Timestamp of the last successful fetch (readData completed without error). */
71
71
  public lastSuccessfulFetchMs: number = 0;
72
+ /**
73
+ * Wall-clock time of the last completed pass through the fetch loop, whatever
74
+ * its outcome (fetched, caught up, or errored).
75
+ *
76
+ * This — not `lastSuccessfulFetchMs` — is the hang detector. A chain that is
77
+ * caught up never calls `readData`, so its last-successful-fetch time is
78
+ * indistinguishable from that of a chain wedged inside a request that will
79
+ * never return. The loop still cycles in the first case and does not in the
80
+ * second, so a heartbeat that is stale by many polling intervals means the
81
+ * loop is stuck in `stateToInput`/`readData`.
82
+ */
83
+ public lastPollAtMs: number = 0;
72
84
  /** Number of consecutive errors since the last successful fetch. */
73
85
  public consecutiveErrors: number = 0;
74
86
  /** Timestamp of the most recent error, or 0 if no error has occurred. */
75
87
  public lastErrorTimestamp: number = 0;
76
88
 
89
+ // ── Streaming producer (see startAsync + orchestration/sync.ts) ──
90
+ /**
91
+ * Whether {@link SyncState#startAsync} runs a long-lived producer that this
92
+ * protocol depends on for its data (a subscription/stream), as opposed to the
93
+ * base-class no-op used by polled chains.
94
+ *
95
+ * `startSync` supervises and restarts the producer only when this is true.
96
+ * A polled chain must leave it false, or its no-op `startAsync` would be
97
+ * "restarted" forever.
98
+ */
99
+ public readonly hasAsyncProducer: boolean = false;
100
+ /**
101
+ * How many times the streaming producer has been restarted after dying —
102
+ * either by throwing or by returning (a producer that returns has ended its
103
+ * stream). Surfaced on `/health`; a climbing count means a flapping upstream.
104
+ */
105
+ public producerRestarts: number = 0;
106
+ /**
107
+ * Producer failures, tracked separately from {@link consecutiveErrors}.
108
+ *
109
+ * That counter belongs to the fetch loop and is only cleared by a successful
110
+ * `readData`, so sharing it conflated two independent signals: an idle
111
+ * streaming chain would report `erroring` long after its producer recovered,
112
+ * and a successful poll would erase the record of a flapping producer.
113
+ */
114
+ public producerErrors: number = 0;
115
+ /** Wall-clock time of the last producer failure, or 0 if none. */
116
+ public lastProducerErrorMs: number = 0;
117
+
118
+ /**
119
+ * Resolved polling interval for this protocol, stamped by `startSync`.
120
+ *
121
+ * Single source of truth: `/health` derives its wedged threshold from this
122
+ * rather than reaching back into `fetcher.config.syncProtocol`, which it can
123
+ * only do through an unchecked cast that would fail silently to a default if
124
+ * the config shape ever moved.
125
+ */
126
+ public pollingIntervalMs: number = 0;
127
+
77
128
  // ── Backpressure observability (see common/page-helpers.ts + README) ──
78
129
  /** Resolved fetch cap (`maxBufferedPages`); set on each backpressure check, 0 until the first. */
79
130
  public bufferCap: number = 0;
@@ -27,6 +27,7 @@ import type {
27
27
  BitcoinTxId,
28
28
  } from "@effectstream/utils";
29
29
  import { blockNumberRelation } from "../common/utils.ts";
30
+ import { DEFAULT_REQUEST_TIMEOUT_MS, fetchWithTimeout } from "../common/http.ts";
30
31
  import type { ConfigType } from "./types.ts";
31
32
 
32
33
  const SATOSHIS_PER_BTC = 100_000_000;
@@ -457,6 +458,8 @@ type BitcoinRpcClientOptions = {
457
458
  url: string;
458
459
  username?: string | null;
459
460
  password?: string | null;
461
+ /** Per-request deadline; defaults to {@link DEFAULT_REQUEST_TIMEOUT_MS}. */
462
+ requestTimeoutMs?: number;
460
463
  };
461
464
 
462
465
  type BitcoinRpcRequest = {
@@ -515,16 +518,21 @@ export class BitcoinRpcClient {
515
518
  private async call<TResult>(
516
519
  request: BitcoinRpcRequest,
517
520
  ): Promise<TResult> {
518
- const response = await fetch(this.options.url, {
519
- method: "POST",
520
- headers: this.buildHeaders(),
521
- body: JSON.stringify({
522
- jsonrpc: "2.0",
523
- id: ++this.nextId,
524
- method: request.method,
525
- params: request.params ?? [],
526
- }),
527
- });
521
+ const response = await fetchWithTimeout(
522
+ this.options.url,
523
+ {
524
+ method: "POST",
525
+ headers: this.buildHeaders(),
526
+ body: JSON.stringify({
527
+ jsonrpc: "2.0",
528
+ id: ++this.nextId,
529
+ method: request.method,
530
+ params: request.params ?? [],
531
+ }),
532
+ },
533
+ `Bitcoin ${request.method}`,
534
+ this.options.requestTimeoutMs,
535
+ );
528
536
  if (!response.ok) {
529
537
  const body = await response.text();
530
538
  throw new Error(
@@ -54,6 +54,11 @@ const CELESTIA_NODE_URL = ENV.getString(
54
54
  "http://localhost:26658",
55
55
  );
56
56
  const CELESTIA_AUTH_TOKEN = ENV.getString("CELESTIA_AUTH_TOKEN", "");
57
+ /**
58
+ * Fallback per-request deadline, kept for compatibility with deployments that
59
+ * set it. The protocol config's `requestTimeoutMs` takes precedence — see the
60
+ * constructor.
61
+ */
57
62
  const CELESTIA_RPC_TIMEOUT_MS = parseInt(
58
63
  ENV.getString("CELESTIA_RPC_TIMEOUT_MS", "15000"),
59
64
  );
@@ -77,9 +82,18 @@ export function celestiaNamespaceToBase64(hex: string): string {
77
82
 
78
83
  export class CelestiaClient {
79
84
  private readonly rpcUrl: string;
85
+ /**
86
+ * Per-attempt deadline. Unlike the other clients this one retries internally
87
+ * (see `rpc` below), so this bounds each attempt rather than the whole call.
88
+ */
89
+ private readonly requestTimeoutMs: number;
80
90
 
81
- constructor(rpcUrl: string = CELESTIA_NODE_URL) {
91
+ constructor(
92
+ rpcUrl: string = CELESTIA_NODE_URL,
93
+ requestTimeoutMs: number = CELESTIA_RPC_TIMEOUT_MS,
94
+ ) {
82
95
  this.rpcUrl = rpcUrl;
96
+ this.requestTimeoutMs = requestTimeoutMs;
83
97
  }
84
98
 
85
99
  private async rpc<T>(method: string, params: unknown[]): Promise<T | null> {
@@ -95,7 +109,7 @@ export class CelestiaClient {
95
109
  method: "POST",
96
110
  headers,
97
111
  body: JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }),
98
- signal: AbortSignal.timeout(CELESTIA_RPC_TIMEOUT_MS),
112
+ signal: AbortSignal.timeout(this.requestTimeoutMs),
99
113
  });
100
114
  const json = await res.json();
101
115
  if (json.error) {
@@ -2,6 +2,7 @@ import {
2
2
  ConfigSyncProtocolType,
3
3
  type PrimitiveEntry,
4
4
  } from "@effectstream/config";
5
+ import { requestTimeoutOf } from "../common/http.ts";
5
6
  import { BaseDataFetcher } from "../base/fetcher.ts";
6
7
  import type { DataFetched } from "../base/fetcher.ts";
7
8
  import type {
@@ -37,7 +38,10 @@ export class CelestiaFetcher extends BaseDataFetcher<
37
38
  readonly config: ConfigType,
38
39
  ) {
39
40
  super(config.syncProtocol.name);
40
- this.client = new CelestiaClient(config.network.rpcUrl);
41
+ this.client = new CelestiaClient(
42
+ config.network.rpcUrl,
43
+ requestTimeoutOf(config.syncProtocol),
44
+ );
41
45
  }
42
46
 
43
47
  @bound
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
10
10
  import { getPage } from "@effectstream/db";
11
11
  import { CelestiaClient } from "./CelestiaClient.ts";
12
12
  import { applyDelay } from "../common/utils.ts";
13
+ import { requestTimeoutOf } from "../common/http.ts";
13
14
  import { bufferAtCap } from "../common/page-helpers.ts";
14
15
 
15
16
  export class CelestiaSyncState extends SyncState<
@@ -135,7 +136,10 @@ export class CelestiaSyncState extends SyncState<
135
136
  page,
136
137
  config,
137
138
  fetcher,
138
- new CelestiaClient(config.network.rpcUrl),
139
+ new CelestiaClient(
140
+ config.network.rpcUrl,
141
+ requestTimeoutOf(config.syncProtocol),
142
+ ),
139
143
  dbConn,
140
144
  );
141
145
  }
@@ -0,0 +1,77 @@
1
+ /**
2
+ * Bounded HTTP for sync-protocol RPC clients.
3
+ *
4
+ * `fetch` has no default timeout. Against a blackholed endpoint — a load
5
+ * balancer that drops its backend, a socket left half-open by a NAT rebind —
6
+ * the connection is established and simply never answers, so the promise never
7
+ * settles. For a sync protocol that is the worst possible failure: `readData`
8
+ * never returns, so the fetch loop in `orchestration/sync.ts` never reaches its
9
+ * `catch`, `consecutiveErrors` stays 0, `lastSuccessfulFetchMs` freezes, and
10
+ * the merge blocks on that chain's page forever. The node stops producing
11
+ * blocks and reports nothing wrong.
12
+ *
13
+ * Deliberately timeout-only, with NO internal retry. The fetch loop already
14
+ * retries: it catches, counts the error, sleeps `pollingInterval` and re-runs
15
+ * the same page range (fetches are idempotent — `lastPage` only advances on a
16
+ * complete `DataFetched`). Retrying inside the client as well would double the
17
+ * backoff and, worse, hide failures from `consecutiveErrors`, which is what the
18
+ * health endpoint reports on.
19
+ *
20
+ * Regression test: `sync/test/rpc-timeout.test.ts` points each client at a
21
+ * blackhole and asserts it rejects rather than hanging.
22
+ */
23
+
24
+ /** Used when a protocol config carries no `requestTimeoutMs`. */
25
+ export const DEFAULT_REQUEST_TIMEOUT_MS = 15_000;
26
+
27
+ /** Thrown when a request exceeds its budget, so callers can tell it apart. */
28
+ export class RequestTimeoutError extends Error {
29
+ constructor(
30
+ readonly label: string,
31
+ readonly timeoutMs: number,
32
+ ) {
33
+ super(`[${label}] request timed out after ${timeoutMs}ms`);
34
+ this.name = "RequestTimeoutError";
35
+ }
36
+ }
37
+
38
+ /**
39
+ * `fetch` with a hard deadline.
40
+ *
41
+ * @param label Prefix for the thrown error, e.g. `"Bitcoin getblockhash"`.
42
+ * @param timeoutMs Falls back to {@link DEFAULT_REQUEST_TIMEOUT_MS}.
43
+ * @param signal Optional caller signal; aborting either one aborts the request.
44
+ */
45
+ export async function fetchWithTimeout(
46
+ url: string,
47
+ init: RequestInit,
48
+ label: string,
49
+ timeoutMs: number = DEFAULT_REQUEST_TIMEOUT_MS,
50
+ signal?: AbortSignal,
51
+ ): Promise<Response> {
52
+ const timeoutSignal = AbortSignal.timeout(timeoutMs);
53
+ const combined = signal
54
+ ? AbortSignal.any([signal, timeoutSignal])
55
+ : timeoutSignal;
56
+
57
+ try {
58
+ return await fetch(url, { ...init, signal: combined });
59
+ } catch (err) {
60
+ // Surface our own error for the timeout case; a caller-supplied abort and
61
+ // ordinary transport failures propagate unchanged.
62
+ if (timeoutSignal.aborted && !(signal?.aborted ?? false)) {
63
+ throw new RequestTimeoutError(label, timeoutMs);
64
+ }
65
+ throw err;
66
+ }
67
+ }
68
+
69
+ /**
70
+ * Resolve the timeout for a protocol config. Optional in the schema, so this
71
+ * centralises the fallback instead of repeating `?? DEFAULT` at each client.
72
+ */
73
+ export function requestTimeoutOf(
74
+ syncProtocol: { requestTimeoutMs?: number } | undefined,
75
+ ): number {
76
+ return syncProtocol?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
77
+ }
@@ -30,11 +30,19 @@ export interface MidnightGqlBlockState {
30
30
  intentHash: string;
31
31
  outputIndex: number;
32
32
  owner: string;
33
+ /** u128 value as a decimal string */
34
+ value: string;
35
+ /** hex-encoded serialized token type */
36
+ tokenType: string;
33
37
  }[];
34
38
  unshieldedCreatedOutputs?: {
35
39
  intentHash: string;
36
40
  outputIndex: number;
37
41
  owner: string;
42
+ /** u128 value as a decimal string */
43
+ value: string;
44
+ /** hex-encoded serialized token type */
45
+ tokenType: string;
38
46
  }[];
39
47
  // Present only on RegularTransaction (selected via inline fragment): the
40
48
  // coin-commitment Merkle tree root after this transaction.
@@ -54,7 +62,7 @@ export interface MidnightGqlBlockState {
54
62
  export interface BlockFetchOptions {
55
63
  /** Include contractActions in the transaction fields (needed for MidnightGenericPrimitive). Default: true */
56
64
  contractActions?: boolean;
57
- /** Include zswapLedgerEvents in the transaction fields (needed for MidnightNullifierPrimitive). Default: true */
65
+ /** Include zswapLedgerEvents in the transaction fields (needed for MidnightNullifierAndCommitmentPrimitive). Default: true */
58
66
  zswapLedgerEvents?: boolean;
59
67
  /** Include unshieldedSpentOutputs (needed for MidnightUnshieldedSpendPrimitive). Default: false */
60
68
  unshieldedSpentOutputs?: boolean;
@@ -150,10 +158,10 @@ export class MidnightClient {
150
158
  ? `zswapLedgerEvents { id raw maxId }`
151
159
  : "";
152
160
  const unshieldedSpentField = unshieldedSpentOutputs
153
- ? `unshieldedSpentOutputs { intentHash outputIndex owner }`
161
+ ? `unshieldedSpentOutputs { intentHash outputIndex owner value tokenType }`
154
162
  : "";
155
163
  const unshieldedCreatedField = unshieldedCreatedOutputs
156
- ? `unshieldedCreatedOutputs { intentHash outputIndex owner }`
164
+ ? `unshieldedCreatedOutputs { intentHash outputIndex owner value tokenType }`
157
165
  : "";
158
166
  // zswapMerkleTreeRoot lives on RegularTransaction, not the Transaction
159
167
  // interface, so it must be selected through an inline fragment.
@@ -1,4 +1,5 @@
1
1
  import { BaseDataFetcher } from "../base/fetcher.ts";
2
+ import { requestTimeoutOf } from "../common/http.ts";
2
3
  import type {
3
4
  Block,
4
5
  ConfigType,
@@ -18,7 +19,7 @@ import type { RootOutput, RootPage } from "../types.ts";
18
19
  import { bound } from "@effectstream/utils";
19
20
  import { MidnightClient, type BlockFetchOptions, type MidnightGqlBlockState } from "./MidnightClient.ts";
20
21
  import { ContractState, StateValue } from "@midnight-ntwrk/onchain-runtime";
21
- import { decodeZswapInputEvent } from "./zswap-decoder.ts";
22
+ import { decodeZswapEvent } from "./zswap-decoder.ts";
22
23
  import { decodeTokenMints } from "./mint-decoder.ts";
23
24
 
24
25
  export class MidnightFetcher extends BaseDataFetcher<
@@ -59,6 +60,7 @@ export class MidnightFetcher extends BaseDataFetcher<
59
60
  this.client = new MidnightClient(
60
61
  indexerHttp,
61
62
  this.networkId,
63
+ requestTimeoutOf(config.syncProtocol),
62
64
  );
63
65
  }
64
66
 
@@ -76,14 +78,14 @@ export class MidnightFetcher extends BaseDataFetcher<
76
78
  const blockFetchOptions: BlockFetchOptions = {
77
79
  contractActions: this.config.primitives.some(
78
80
  (p) =>
79
- p.primitive.type !== "Midnight:Nullifier" &&
81
+ p.primitive.type !== "Midnight:NullifierAndCommitment" &&
80
82
  p.primitive.type !== "Midnight:UnshieldedSpend" &&
81
83
  p.primitive.type !== "Midnight:UnshieldedCreate" &&
82
84
  p.primitive.type !== "Midnight:ZswapRoot" &&
83
85
  p.primitive.type !== "Midnight:TokenMint",
84
86
  ),
85
87
  zswapLedgerEvents: this.config.primitives.some(
86
- (p) => p.primitive.type === "Midnight:Nullifier",
88
+ (p) => p.primitive.type === "Midnight:NullifierAndCommitment",
87
89
  ),
88
90
  unshieldedSpentOutputs: this.config.primitives.some(
89
91
  (p) => p.primitive.type === "Midnight:UnshieldedSpend",
@@ -181,8 +183,8 @@ export class MidnightFetcher extends BaseDataFetcher<
181
183
  const syncResults: PrimitiveType[] = [];
182
184
 
183
185
  for (const primitiveEntry of primitiveEntries) {
184
- if (primitiveEntry.primitive.type === "Midnight:Nullifier") {
185
- syncResults.push(...this.fetchNullifiers(height, primitiveEntry, block));
186
+ if (primitiveEntry.primitive.type === "Midnight:NullifierAndCommitment") {
187
+ syncResults.push(...this.fetchZswapEvents(height, primitiveEntry, block));
186
188
  } else if (primitiveEntry.primitive.type === "Midnight:UnshieldedSpend") {
187
189
  syncResults.push(...this.fetchUnshieldedSpends(height, primitiveEntry, block));
188
190
  } else if (primitiveEntry.primitive.type === "Midnight:UnshieldedCreate") {
@@ -203,16 +205,19 @@ export class MidnightFetcher extends BaseDataFetcher<
203
205
  }
204
206
 
205
207
  @bound
206
- fetchNullifiers(
208
+ fetchZswapEvents(
207
209
  height: number,
208
210
  primitiveEntry: PrimitiveEntryType,
209
211
  block: MidnightGqlBlockState,
210
212
  ): PrimitiveType[] {
213
+ const capture = primitiveEntry.primitive.capture ?? "both";
211
214
  const results: PrimitiveType[] = [];
212
215
  for (const tx of block.block.transactions) {
213
216
  for (const event of tx.zswapLedgerEvents ?? []) {
214
- const decoded = decodeZswapInputEvent(event.raw);
217
+ const decoded = decodeZswapEvent(event.raw);
215
218
  if (!decoded) continue;
219
+ if (decoded.kind === "nullifier" && capture === "commitments") continue;
220
+ if (decoded.kind === "commitment" && capture === "nullifiers") continue;
216
221
  results.push({
217
222
  syncProtocol: {
218
223
  name: primitiveEntry.syncProtocol,
@@ -222,12 +227,10 @@ export class MidnightFetcher extends BaseDataFetcher<
222
227
  },
223
228
  primitive: primitiveEntry.primitive.name,
224
229
  output: {
225
- payloadType: "midnight-nullifier",
230
+ payloadType: `midnight-${decoded.kind}`,
226
231
  payload: {
227
- nullifier: decoded.nullifier,
228
- txHash: decoded.txHash,
232
+ ...decoded,
229
233
  eventId: event.id,
230
- logicalSegment: decoded.logicalSegment,
231
234
  },
232
235
  },
233
236
  });
@@ -259,6 +262,8 @@ export class MidnightFetcher extends BaseDataFetcher<
259
262
  owner: spend.owner,
260
263
  intentHash: spend.intentHash,
261
264
  outputIndex: spend.outputIndex,
265
+ value: spend.value,
266
+ tokenType: spend.tokenType,
262
267
  txHash: tx.hash,
263
268
  },
264
269
  },
@@ -294,6 +299,8 @@ export class MidnightFetcher extends BaseDataFetcher<
294
299
  owner: created.owner,
295
300
  intentHash: created.intentHash,
296
301
  outputIndex: created.outputIndex,
302
+ value: created.value,
303
+ tokenType: created.tokenType,
297
304
  txHash: tx.hash,
298
305
  },
299
306
  },
@@ -35,7 +35,7 @@ export type CommonFunnelArgs = {
35
35
  // Graphql Block
36
36
  // =============
37
37
 
38
- // Interfaces approximated from GraphQL schema served by indexer v1.3.1
38
+ // Interfaces approximated from GraphQL schema served by indexer v4.3.3 (/api/v3)
39
39
 
40
40
  interface ContractCallOrDeploy {
41
41
  address: MidnightAddress;
@@ -1,58 +1,26 @@
1
- import { Buffer } from "node:buffer";
2
1
  import * as ledger from '@midnight-ntwrk/ledger-v8';
3
2
 
4
- // TODO This alternative implementation should be the native way to decode the event.
5
- function new_decodeZswapInputEvent(rawHex: string): ledger.Event | null {
6
- try {
7
- const bytes = Uint8Array.from(
8
- Buffer.from(rawHex.replace(/^0x/, ""), "hex"),
9
- );
10
- const event = ledger.Event.deserialize(bytes);
11
- console.log("event", event);
12
- // if (event.type === "ZswapInput") {
13
- // event.nullifier — Uint8Array (32 bytes)
14
- // event.source.transactionHash
15
- // event.source.logicalSegment
16
- // } else if (event.type === "ZswapOutput") {
17
- // event.commitment, etc.
18
- // }
19
- } catch {
20
- return null;
21
- }
22
- return null;
23
- }
24
-
25
- // ─── SCALE compact integer decoder ───────────────────────────────────────────
3
+ // Wire format of a raw zswap ledger event, for reference (midnight-ledger 8.x,
4
+ // `midnight-ledger/ledger/src/events.rs` + `storage-core/src/arena.rs`):
26
5
  //
27
- // Bottom 2 bits of the first byte are the mode:
28
- // 00 = 1-byte (value in bits [7:2])
29
- // 01 = 2-byte
30
- // 10 = 4-byte
31
- // 11 = n-byte length prefix
6
+ // 19 bytes outer tag "midnight:event[v9]:"
7
+ // then a TopoSortedNodes envelope (NOT the struct directly):
8
+ // compact_u32 node count, and per node: compact_u32 child count,
9
+ // compact_u64 child indices, compact_u32 data length, data bytes.
10
+ // The root node's data is:
11
+ // 32 bytes EventSource.transaction_hash
12
+ // 2 bytes logical_segment (LE u16)
13
+ // 2 bytes physical_segment (LE u16)
14
+ // 1 byte EventDetails variant: 0=ZswapInput, 1=ZswapOutput, 2+=other
15
+ // ZswapInput: 32-byte nullifier + Option<Sp<ContractAddress>> byte
16
+ // ZswapOutput: 32-byte commitment + variable-length preimage evidence
17
+ // + Option<Sp<ContractAddress>> byte + compact_u64 mt_index
18
+ // A Some(..) contract address is a SEPARATE node in the envelope, shifting
19
+ // all offsets — so positional parsing is not viable; always deserialize via
20
+ // the ledger bindings.
32
21
 
33
- function readScaleU32(b: Uint8Array, pos: number): [number, number] {
34
- const first = b[pos];
35
- const mode = first & 0b11;
36
- if (mode === 0b00) {
37
- return [(first & 0xFC) >> 2, pos + 1];
38
- } else if (mode === 0b01) {
39
- const second = b[pos + 1];
40
- const v0 = ((first & 0xFC) >> 2) | ((second & 0x03) << 6);
41
- const v1 = (second & 0xFC) >> 2;
42
- return [(v0 | (v1 << 8)) >>> 0, pos + 2];
43
- } else if (mode === 0b10) {
44
- const s = b[pos + 1], t = b[pos + 2], u = b[pos + 3];
45
- const v0 = ((first & 0xFC) >> 2) | ((s & 0x03) << 6);
46
- const v1 = ((s & 0xFC) >> 2) | ((t & 0x03) << 6);
47
- const v2 = ((t & 0xFC) >> 2) | ((u & 0x03) << 6);
48
- const v3 = (u & 0xFC) >> 2;
49
- return [(v0 | (v1 << 8) | (v2 << 16) | (v3 << 24)) >>> 0, pos + 4];
50
- } else {
51
- const n = ((first & 0xFC) >> 2) + 4;
52
- let val = 0;
53
- for (let i = 0; i < n; i++) val |= b[pos + 1 + i] << (i * 8);
54
- return [val >>> 0, pos + 1 + n];
55
- }
22
+ function normalizeHex(hex: string): string {
23
+ return (hex.startsWith("0x") ? hex.slice(2) : hex).toLowerCase();
56
24
  }
57
25
 
58
26
  function hexToBytes(hex: string): Uint8Array {
@@ -64,60 +32,90 @@ function hexToBytes(hex: string): Uint8Array {
64
32
  return arr;
65
33
  }
66
34
 
67
- function bytesToHex(bytes: Uint8Array): string {
68
- return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
69
- }
70
-
71
- // ─── Binary layout for "midnight:event[v9]:" ─────────────────────────────────
72
- // 19 bytes outer tag "midnight:event[v9]:"
73
- // 32 bytes EventSource.transaction_hash
74
- // 2 bytes EventSource.logical_segment (LE u16)
75
- // 2 bytes EventSource.physical_segment (LE u16, skipped)
76
- // SCALE u32 EventDetails variant: 0=ZswapInput, 1=ZswapOutput, 2+=other
77
- //
78
- // ZswapInput: 4-byte contract_id + 32-byte CoinNullifier
79
-
80
- const OUTER_TAG = "midnight:event[v9]:";
81
-
82
- export interface DecodedZswapInput {
83
- /** 32-byte transaction hash, hex */
84
- txHash: string;
85
- /** Logical segment index */
86
- logicalSegment: number;
87
- /** 32-byte nullifier, hex */
88
- nullifier: string;
89
- }
35
+ /** A zswap ledger event decoded via the native ledger-v8 bindings. */
36
+ export type DecodedZswapEvent =
37
+ | {
38
+ kind: "nullifier";
39
+ /** 32-byte nullifier, hex */
40
+ nullifier: string;
41
+ /** 32-byte transaction hash, hex */
42
+ txHash: string;
43
+ /** Logical segment index */
44
+ logicalSegment: number;
45
+ /** Contract address that spent the coin, hex — absent for user coins */
46
+ contract?: string;
47
+ }
48
+ | {
49
+ kind: "commitment";
50
+ /** 32-byte coin commitment, hex */
51
+ commitment: string;
52
+ /** Index of the commitment in the zswap Merkle tree, u64 as decimal string */
53
+ mtIndex: string;
54
+ /** 32-byte transaction hash, hex */
55
+ txHash: string;
56
+ /** Logical segment index */
57
+ logicalSegment: number;
58
+ /** Contract address that received the coin, hex — absent for user coins */
59
+ contract?: string;
60
+ };
90
61
 
91
62
  /**
92
- * Decode a raw SCALE-encoded zswap ledger event.
93
- * Returns the decoded data if the event is a ZswapInput (nullifier spend),
94
- * or `null` for ZswapOutput / other variants.
63
+ * Decode a raw zswap ledger event (ZswapInput or ZswapOutput) using the
64
+ * native ledger-v8 WASM deserializer. The indexer's `zswapLedgerEvents`
65
+ * field only ever carries these two variants, so a `null` return means the
66
+ * event could not be decoded (e.g. a ledger event-format version bump) and
67
+ * is logged.
95
68
  */
96
- export function decodeZswapInputEvent(
97
- rawHex: string,
98
- ): DecodedZswapInput | null {
69
+ export function decodeZswapEvent(rawHex: string): DecodedZswapEvent | null {
70
+ let event: ledger.Event;
99
71
  try {
100
- const b = hexToBytes(rawHex);
101
- let pos = OUTER_TAG.length;
102
-
103
- const txHash = bytesToHex(b.slice(pos, pos + 32));
104
- pos += 32;
105
- const logicalSegment = b[pos] | (b[pos + 1] << 8);
106
- pos += 2;
107
- pos += 2; // skip physical segment
108
-
109
- const [variant, p1] = readScaleU32(b, pos);
110
- pos = p1;
111
-
112
- if (variant !== 0) {
113
- return null; // ZswapOutput or other — not a nullifier event
114
- }
115
-
116
- pos += 4; // skip 4-byte contract_id
117
-
118
- const nullifier = bytesToHex(b.slice(pos, pos + 32));
119
- return { txHash, logicalSegment, nullifier };
120
- } catch {
72
+ event = ledger.Event.deserialize(hexToBytes(rawHex));
73
+ } catch (err) {
74
+ console.warn(
75
+ "[midnight] failed to deserialize zswap ledger event — ledger event format may have changed",
76
+ { raw: rawHex, error: String(err) },
77
+ );
121
78
  return null;
122
79
  }
80
+ // EventDetails is an open union ({ tag: string } catch-all), so narrow manually.
81
+ const content = event.content as {
82
+ tag: string;
83
+ nullifier?: string;
84
+ commitment?: string;
85
+ contract?: string;
86
+ mtIndex?: bigint;
87
+ };
88
+ const source = {
89
+ txHash: normalizeHex(event.source.transactionHash),
90
+ logicalSegment: event.source.logicalSegment,
91
+ };
92
+ const contract = content.contract != null
93
+ ? { contract: normalizeHex(content.contract) }
94
+ : {};
95
+ if (content.tag === "zswapInput" && content.nullifier != null) {
96
+ return {
97
+ kind: "nullifier",
98
+ nullifier: normalizeHex(content.nullifier),
99
+ ...contract,
100
+ ...source,
101
+ };
102
+ }
103
+ if (
104
+ content.tag === "zswapOutput" &&
105
+ content.commitment != null &&
106
+ content.mtIndex != null
107
+ ) {
108
+ return {
109
+ kind: "commitment",
110
+ commitment: normalizeHex(content.commitment),
111
+ mtIndex: content.mtIndex.toString(),
112
+ ...contract,
113
+ ...source,
114
+ };
115
+ }
116
+ console.warn(
117
+ "[midnight] unexpected zswap ledger event variant",
118
+ { raw: rawHex, tag: content.tag },
119
+ );
120
+ return null;
123
121
  }