@effectstream/sync 0.100.22 → 0.100.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +73 -32
- package/README.md +62 -0
- package/package.json +5 -5
- package/src/sync-protocols/avail/state.ts +11 -0
- package/src/sync-protocols/base/state.ts +49 -9
- package/src/sync-protocols/bitcoin/state.ts +13 -1
- package/src/sync-protocols/celestia/state.ts +11 -0
- package/src/sync-protocols/common/page-helpers.ts +41 -0
- package/src/sync-protocols/common/root.ts +9 -0
- package/src/sync-protocols/evm/state.ts +12 -1
- package/src/sync-protocols/midnight/MidnightClient.ts +20 -0
- package/src/sync-protocols/midnight/fetcher.ts +44 -1
- package/src/sync-protocols/midnight/mint-decoder.ts +151 -0
- package/src/sync-protocols/midnight/state.ts +11 -0
- package/src/sync-protocols/near/state.ts +11 -0
- package/src/sync-protocols/ntp/state.ts +13 -1
- package/src/sync-protocols/orchestration/merge.ts +23 -0
- package/src/sync-protocols/test/state.ts +13 -1
- package/src/sync-protocols/utxorpc/state.ts +18 -0
- package/test/mint-decoder.test.ts +28 -0
package/CLAUDE.md
CHANGED
|
@@ -35,7 +35,7 @@ config (syncProtocols.main + syncProtocols.parallel)
|
|
|
35
35
|
│ PER-CHAIN FETCH LOOPS one spawn per protocol │
|
|
36
36
|
│ startSync(state): stateToInput → readData → updateState │
|
|
37
37
|
│ → push Output to bufferedData (Deque) + wake newData/newPageCondVar│
|
|
38
|
-
│ →
|
|
38
|
+
│ → lastPage kept IN-MEMORY only (no DB write; runtime owns resume) │
|
|
39
39
|
│ → producerChannel.send(...) ← NOTE: nothing consumes this today │
|
|
40
40
|
└────────────────────────────────────────┬─────────────────────────────┘
|
|
41
41
|
▼
|
|
@@ -47,8 +47,9 @@ config (syncProtocols.main + syncProtocols.parallel)
|
|
|
47
47
|
└────────────────────────────────────────┬─────────────────────────────┘
|
|
48
48
|
▼
|
|
49
49
|
runtime/src/main.ts: for (block of each(finalizedBlockStream))
|
|
50
|
-
acquire DB mutex → processFinalizedBlock (STF)
|
|
51
|
-
→ removePages(<
|
|
50
|
+
acquire DB mutex → processFinalizedBlock (STF)
|
|
51
|
+
→ upsertPage(block.resumePages) + removePages(< committed) [in txn] → COMMIT
|
|
52
|
+
→ fire-and-forget MQTT events → each.next()
|
|
52
53
|
```
|
|
53
54
|
|
|
54
55
|
## The three per-chain abstractions (`src/sync-protocols/<chain>/`)
|
|
@@ -112,12 +113,34 @@ iteration it walks protocols in order:
|
|
|
112
113
|
a parallel chain's timestamp so older blocks merge into more-recent root slots.
|
|
113
114
|
4. **Mutate-after-commit.** `OutputAndCleanup` keeps data in the Deque until the block is
|
|
114
115
|
safely produced; the consumer commits to Postgres *before* emitting events.
|
|
115
|
-
5. **Durable,
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
116
|
+
5. **Durable, block-accurate resume queue.** *(Single source of truth for how restart/resume
|
|
117
|
+
works — code comments point here; see also Finding #2 for the bug this fixes.)*
|
|
118
|
+
|
|
119
|
+
`effectstream.sync_protocol_pagination` is keyed `(protocol_name, page_number)` and the
|
|
120
|
+
**runtime is the sole writer** of the resume position (Fix D). End to end:
|
|
121
|
+
- **Fetch loop** (`base/state.ts:updateState`) keeps `lastPage` in memory only — it does
|
|
122
|
+
**not** persist (it runs ahead of commits, so its chunk-end page is not a safe resume
|
|
123
|
+
point).
|
|
124
|
+
- **Merge** (`orchestration/merge.ts`) tags every `ChainBlock` with `resumePages`: one
|
|
125
|
+
marker per protocol = `outputToLastPage(lastConsumed)`, the `LastPage`
|
|
126
|
+
(`{ own, ownBlockNumber, root }`) of the **highest datum merged into the block**. Each
|
|
127
|
+
protocol implements `outputToLastPage` (`base/state.ts` + per-chain `state.ts`) because
|
|
128
|
+
`own` is a structured page for object-paged chains and can't be rebuilt from the block
|
|
129
|
+
number. Pending data above the committed timestamp is **not** marked, so it is always
|
|
130
|
+
re-fetched, never skipped.
|
|
131
|
+
- **Commit** (`runtime/process-blocks.ts`, STEP 7) writes each marker **inside the block's
|
|
132
|
+
transaction**: `upsertPage(page_number = ownBlockNumber)` then
|
|
133
|
+
`removePages(page_number < ownBlockNumber)` — leaving exactly one row per protocol = the
|
|
134
|
+
committed watermark, atomic with the block.
|
|
135
|
+
- **Boot** (`restoreState`) reads `getPage` = `ORDER BY page_number ASC LIMIT 1` and
|
|
136
|
+
resumes from `nextInterval(own).from` = the next uncommitted block.
|
|
137
|
+
|
|
138
|
+
**Re-scan on restart is bounded by `stepSize`, not a chain's whole quiet tail.** Every
|
|
139
|
+
fetched chunk's boundary page (`data.to`) is buffered and consumed even when empty, so the
|
|
140
|
+
marker advances to within one `stepSize` of the committed frontier — and reaches the
|
|
141
|
+
fetched tip exactly at full catch-up. So a long-quiet chain re-fetches at most ~`stepSize`
|
|
142
|
+
blocks after a restart, regardless of how long ago its last data was. Trade-off:
|
|
143
|
+
`getSyncAndLastPage` now reports `synced_page == fetched_page`.
|
|
121
144
|
|
|
122
145
|
---
|
|
123
146
|
|
|
@@ -144,26 +167,42 @@ Consequences during catch-up:
|
|
|
144
167
|
(`mergeIntoRoot`, `merge.ts`), so a slow/stalled chain stalls all block production while
|
|
145
168
|
the others keep ballooning (head-of-line blocking).
|
|
146
169
|
|
|
147
|
-
→ Fix: **(C) backpressure** —
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
170
|
+
→ Fix: **(C) backpressure** — ✅ SHIPPED. `bufferAtCap` (`common/page-helpers.ts`) caps
|
|
171
|
+
`bufferedData` and pauses the fetch loop (each chain's `stateToInput` returns `undefined`)
|
|
172
|
+
until the merge drains. See README "Backpressure (`maxBufferedPages`)".
|
|
173
|
+
|
|
174
|
+
> **Merge→apply queue cap (Fix C′) — ✅ SHIPPED.** The Deque cap above bounds *fetch*, but the
|
|
175
|
+
> merge then sends finalized blocks into an effection channel whose subscriber queue buffers
|
|
176
|
+
> unbounded — so the merge (no I/O) could still race ahead of the serial apply loop and grow that
|
|
177
|
+
> queue toward the whole backlog, especially with empty-block coalescing (Fix E) removing the
|
|
178
|
+
> per-block apply brake. The runtime now gates the merge's `send` (`runtime/src/main.ts`): it blocks
|
|
179
|
+
> once `EFFECTSTREAM_FINALIZED_STREAM_CAP` (default 2048) produced blocks are unconsumed, and the
|
|
180
|
+
> apply loop wakes it as it drains. `inFlight = produced − consumed` is surfaced on
|
|
181
|
+
> `/debug/metrics`. Repro/guard: `runtime/test/reproduction/coalesce-memory.test.ts`.
|
|
182
|
+
|
|
183
|
+
> **Merge-demand exemption (safeguard).** A naive cap can deadlock the merge: it drains a
|
|
184
|
+
> parallel chain's buffer only *after* its page passes the root timestamp `τ`, so if the cap
|
|
185
|
+
> pauses the fetcher while its page is still `≤ τ` (a far skip-ahead, or a chain finer-grained
|
|
186
|
+
> than the cap), merge waits for the page ↔ the page waits for a fetch ↔ the fetch waits for
|
|
187
|
+
> the buffer to drain — circular. Guard: `mergeIntoRoot` sets `SyncState.mergeWaitingForPage`
|
|
188
|
+
> while blocked on a chain's page, and `bufferAtCap` returns `false` for that chain while the
|
|
189
|
+
> flag is set — so the fetcher advances past `τ`. Bounded by necessity: only
|
|
190
|
+
> `(lastPage.root, τ]` is buffered above the cap. Repros: `buffering.test.ts` 1c/1d.
|
|
191
|
+
|
|
192
|
+
### 2. Silent data gap on restart (correctness) — ✅ FIXED (D)
|
|
193
|
+
|
|
194
|
+
**The bug (historical).** The page queue was **chunk-granular** but commits are
|
|
195
|
+
**single-block-granular**: the fetch loop persisted `upsertPage(page_number = chunk_end)`,
|
|
196
|
+
commit deleted `removePages(page_number < merged_block_N)`, and restart resumed from
|
|
197
|
+
`getPage` = oldest remaining chunk at `nextInterval(own).from = chunk_end + 1`. During
|
|
198
|
+
catch-up chunks are full `stepSize`, so the committed watermark `N` sat **inside** the
|
|
199
|
+
oldest retained chunk `[start, end]` (`end ≥ N ⇒ not deleted`); resume jumped to `end + 1`,
|
|
200
|
+
**silently skipping `(N, end]`** → missing primitives → divergent app state, with no error.
|
|
201
|
+
|
|
202
|
+
**The fix (D) — block-accurate resume, runtime is the sole writer.** The runtime persists a
|
|
203
|
+
block-accurate marker per protocol on commit instead of the fetch loop persisting chunk-end
|
|
204
|
+
pages. Full mechanism in **design idea #5** above. Regression test: the
|
|
205
|
+
`consistency.test.ts` "mid-chunk restart" case (Postgres-only).
|
|
167
206
|
|
|
168
207
|
### 3. Serial replay is the catch-up bottleneck
|
|
169
208
|
|
|
@@ -182,6 +221,8 @@ block-hash chain.
|
|
|
182
221
|
> survives a restart. See `docs/ADDING-A-SYNC-PROTOCOL.md` for how to add a chain
|
|
183
222
|
> (with `test` as the worked example).
|
|
184
223
|
>
|
|
185
|
-
> Deterministic reproductions of #1/#2/#3 and their fixes land
|
|
186
|
-
>
|
|
187
|
-
>
|
|
224
|
+
> Deterministic reproductions of #1/#2/#3 and their fixes land alongside each fix:
|
|
225
|
+
> **#2 → block-accurate resume is DONE** (`consistency.test.ts`); **#1 → fetch
|
|
226
|
+
> backpressure is DONE** (`buffering.test.ts`, incl. 1c/1d for the merge-demand
|
|
227
|
+
> exemption). Still in follow-up PRs: #3 → opt-in empty-block commit batching during
|
|
228
|
+
> catch-up.
|
package/README.md
CHANGED
|
@@ -46,6 +46,68 @@ const protocols = await genSyncProtocols(config);
|
|
|
46
46
|
// protocols.parallelEvmRPC_fast.runOne() // poll one block
|
|
47
47
|
```
|
|
48
48
|
|
|
49
|
+
## Backpressure (`maxBufferedPages`)
|
|
50
|
+
|
|
51
|
+
During deep catch-up a chain's fetch loop races to its tip far faster than the
|
|
52
|
+
merge can drain (the merge applies one block per DB transaction). Without a bound
|
|
53
|
+
the in-memory buffer (`SyncState.bufferedData`) grows toward the **entire backlog**
|
|
54
|
+
— hundreds of thousands of block objects — which is an OOM risk.
|
|
55
|
+
|
|
56
|
+
**The cap.** Every chain's `stateToInput` calls `bufferAtCap(state, syncProtocol)`
|
|
57
|
+
first (`sync-protocols/common/page-helpers.ts`): when
|
|
58
|
+
`bufferedData.size() >= maxBufferedPages` it returns `undefined`, so the chain stops
|
|
59
|
+
fetching — exactly as if it had caught up to the tip — and the polling loop sleeps
|
|
60
|
+
`pollingInterval` and retries. The merge keeps draining the buffer; once it drops
|
|
61
|
+
below the cap the next poll resumes. Peak in-memory buffering is therefore bounded
|
|
62
|
+
to **≈ `maxBufferedPages + stepSize`** per chain (one in-flight chunk can overshoot
|
|
63
|
+
the cap), instead of the whole backlog.
|
|
64
|
+
|
|
65
|
+
**Config.** `maxBufferedPages` is an optional field on every sync-protocol config
|
|
66
|
+
(declared once on the shared `PollingSyncProtocol` schema). When unset it defaults to
|
|
67
|
+
**`4 × stepSize`** (the `MAX_BUFFER_MULTIPLE` constant in `page-helpers.ts`), always
|
|
68
|
+
clamped to **`≥ stepSize + 1`** so a chain can always fetch at least one chunk to feed
|
|
69
|
+
the merge. ~4 chunks of look-ahead never starves the merge while keeping memory
|
|
70
|
+
bounded; raise it to trade memory for more fetch look-ahead. The cap only bites during
|
|
71
|
+
catch-up — in steady state the buffer sits near zero and the cap is never reached.
|
|
72
|
+
|
|
73
|
+
**Deadlock-safety (merge-demand exemption).** A naive cap can deadlock the merge: it
|
|
74
|
+
produces a root block at timestamp `τ` only once a parallel chain's page passes `τ`, and
|
|
75
|
+
drains that chain's buffer only *after* the page passes `τ`. If the cap pauses the
|
|
76
|
+
fetcher while its page is still `≤ τ` (the data up to `τ` exceeds the cap — e.g. a far
|
|
77
|
+
skip-ahead or a parallel chain finer-grained than the cap), neither side can proceed: a
|
|
78
|
+
circular wait, not a lost wakeup.
|
|
79
|
+
|
|
80
|
+
The safeguard is a **merge-demand exemption**: while the merge is blocked on a chain's
|
|
81
|
+
page (`mergeIntoRoot` sets `SyncState.mergeWaitingForPage`), `bufferAtCap` returns
|
|
82
|
+
`false` for that chain so the fetcher advances until its page crosses `τ`. Memory stays
|
|
83
|
+
**bounded by necessity** — only `(lastPage.root, τ]` is buffered above the cap, exactly
|
|
84
|
+
what the merge must hold to build block `τ`; once the page passes `τ` the normal cap
|
|
85
|
+
re-engages. A chain that is merely tip-limited (caught up, not gated by the merge) is
|
|
86
|
+
*not* exempted, so steady-state and head-of-line bounds are unaffected. Reproductions:
|
|
87
|
+
`buffering.test.ts` 1c (skip-ahead) and 1d (density).
|
|
88
|
+
|
|
89
|
+
**Scope.** The guard runs in **every** chain's `stateToInput`, so all sync chains are
|
|
90
|
+
covered: EVM, NTP, Bitcoin, Avail, Celestia, NEAR, Midnight, Cardano (UTXO-RPC), and
|
|
91
|
+
the synthetic `test` chain. Two notes:
|
|
92
|
+
- **Cardano (UTXO-RPC)** has no `stepSize` (it streams one block per pass), so the cap
|
|
93
|
+
falls back to a default chunk size of 1000 (⇒ default cap 4000); set `maxBufferedPages`
|
|
94
|
+
explicitly to tune it.
|
|
95
|
+
- The cap bounds `SyncState.bufferedData` (the merge-facing Deque the backpressure
|
|
96
|
+
feature protects). The UTXO-RPC fetcher additionally keeps its own internal FollowTip stream
|
|
97
|
+
buffer; pausing `stateToInput` stops draining it into `bufferedData`, but bounding
|
|
98
|
+
that lower-level stream is a separate, fetcher-specific concern.
|
|
99
|
+
|
|
100
|
+
**Observability.** Each `SyncState` tracks, and the runtime's `/debug/metrics`
|
|
101
|
+
endpoint reports per protocol: `cap` (resolved `maxBufferedPages`), `buf` (current
|
|
102
|
+
size), `bufHighWater` (peak since boot — catches spikes between samples), `pausedNow`,
|
|
103
|
+
`pauses` (rising-edge count — **the "backpressure engaged" signal**), `pausedMs`
|
|
104
|
+
(total time paused), and `mergeWaiting`/`mergeDemandRoot` (merge-demand exemption:
|
|
105
|
+
`mergeWaiting=true` means the merge is gated on this chain's page and the cap is lifted).
|
|
106
|
+
`pauses > 0` means the cap actively bounded memory; `0` means it
|
|
107
|
+
was never needed in that run. Steady-state these sit at `0`; during a real deep
|
|
108
|
+
catch-up (or under the perf harness's `PERF_APPLY_DELAY_MS` drain throttle) they climb
|
|
109
|
+
as `buf` pins to `cap`.
|
|
110
|
+
|
|
49
111
|
## Key exports
|
|
50
112
|
|
|
51
113
|
- `genSyncProtocols(dbConn, syncInfo)` - Effection generator that instantiates a runtime fetcher + state pair for every protocol in `syncInfo` (from `config.syncProtocols`). Called from the runtime's process-blocks loop.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@effectstream/sync",
|
|
3
|
-
"version": "0.100.
|
|
3
|
+
"version": "0.100.23",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"homepage": "https://effectstream.github.io/docs/",
|
|
6
6
|
"repository": {
|
|
@@ -13,10 +13,10 @@
|
|
|
13
13
|
},
|
|
14
14
|
"exports": "./src/mod.ts",
|
|
15
15
|
"dependencies": {
|
|
16
|
-
"@effectstream/config": "0.100.
|
|
17
|
-
"@effectstream/db": "0.100.
|
|
18
|
-
"@effectstream/log": "0.100.
|
|
19
|
-
"@effectstream/utils": "0.100.
|
|
16
|
+
"@effectstream/config": "0.100.23",
|
|
17
|
+
"@effectstream/db": "0.100.23",
|
|
18
|
+
"@effectstream/log": "0.100.23",
|
|
19
|
+
"@effectstream/utils": "0.100.23",
|
|
20
20
|
"@midnight-ntwrk/ledger-v8": "8.0.3",
|
|
21
21
|
"@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.4",
|
|
22
22
|
"@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",
|
|
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
|
|
|
10
10
|
import { getPage } from "@effectstream/db";
|
|
11
11
|
import { AvailClient } from "./AvailClient.ts";
|
|
12
12
|
import { applyDelay } from "../common/utils.ts";
|
|
13
|
+
import { bufferAtCap } from "../common/page-helpers.ts";
|
|
13
14
|
|
|
14
15
|
export class AvailSyncState extends SyncState<
|
|
15
16
|
Input,
|
|
@@ -63,6 +64,7 @@ export class AvailSyncState extends SyncState<
|
|
|
63
64
|
|
|
64
65
|
@bound
|
|
65
66
|
override *stateToInput(): Operation<Input | undefined> {
|
|
67
|
+
if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
|
|
66
68
|
const latestHeight = yield* call(() => this.client.getLatestBlockHeight());
|
|
67
69
|
const startHeight = this.lastPage?.own.height ??
|
|
68
70
|
this.config.syncProtocol.startBlockHeight - 1;
|
|
@@ -99,6 +101,15 @@ export class AvailSyncState extends SyncState<
|
|
|
99
101
|
rootOutput.primitives.push(...primitives);
|
|
100
102
|
}
|
|
101
103
|
|
|
104
|
+
@bound
|
|
105
|
+
override outputToLastPage(data: Output): LastPage<Page, RootPage> {
|
|
106
|
+
return {
|
|
107
|
+
own: { height: data.raw.number, hash: data.raw.hash },
|
|
108
|
+
ownBlockNumber: data.raw.number,
|
|
109
|
+
root: this.toRootPage(data),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
102
113
|
@bound
|
|
103
114
|
override getNamespace(): string[] {
|
|
104
115
|
return [this.config.network.name, this.config.syncProtocol.name];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Operation
|
|
1
|
+
import { type Operation } from "effection";
|
|
2
2
|
import Deque from "denque";
|
|
3
3
|
import {
|
|
4
4
|
type BlockNumber,
|
|
@@ -11,7 +11,6 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
|
|
|
11
11
|
import type { BaseDataFetcher, DataFetched } from "./fetcher.ts";
|
|
12
12
|
import type { PageRelation } from "./page.ts";
|
|
13
13
|
import type { PoolClient } from "pg";
|
|
14
|
-
import { acquireDBMutex, releaseDBMutex, upsertPage } from "@effectstream/db";
|
|
15
14
|
|
|
16
15
|
export type LastPage<Page, RootPage> = {
|
|
17
16
|
own: Page;
|
|
@@ -75,6 +74,32 @@ export abstract class SyncState<
|
|
|
75
74
|
/** Timestamp of the most recent error, or 0 if no error has occurred. */
|
|
76
75
|
public lastErrorTimestamp: number = 0;
|
|
77
76
|
|
|
77
|
+
// ── Backpressure observability (see common/page-helpers.ts + README) ──
|
|
78
|
+
/** Resolved fetch cap (`maxBufferedPages`); set on each backpressure check, 0 until the first. */
|
|
79
|
+
public bufferCap: number = 0;
|
|
80
|
+
/** Peak `bufferedData.size()` seen since boot — catches spikes the periodic sampler misses. */
|
|
81
|
+
public bufferHighWater: number = 0;
|
|
82
|
+
/** Whether fetching is currently paused because the buffer is at/over the cap. */
|
|
83
|
+
public pausedNow: boolean = false;
|
|
84
|
+
/** How many times the cap engaged (rising edges) — the "backpressure fired" counter. */
|
|
85
|
+
public backpressurePauses: number = 0;
|
|
86
|
+
/** Total time fetching has been paused by the cap, in ms. */
|
|
87
|
+
public backpressurePausedMs: number = 0;
|
|
88
|
+
/** Start of the current pause, for accumulating `backpressurePausedMs`. */
|
|
89
|
+
private pauseStartMs: number = 0;
|
|
90
|
+
|
|
91
|
+
// ── Merge-demand exemption (see common/page-helpers.ts + README) ──
|
|
92
|
+
/**
|
|
93
|
+
* True while the merge is blocked waiting for THIS chain's page to pass the
|
|
94
|
+
* current root timestamp. The cap must not pause the fetcher here: the merge
|
|
95
|
+
* drains this chain's buffer only after its page passes the target, so pausing
|
|
96
|
+
* would be a circular wait. While exempt the fetcher buffers only the data the
|
|
97
|
+
* merge must hold to build that block anyway (bounded by necessity).
|
|
98
|
+
*/
|
|
99
|
+
public mergeWaitingForPage: boolean = false;
|
|
100
|
+
/** Root page (ms timestamp) the merge is waiting to exceed — observability only. */
|
|
101
|
+
public mergeDemandRoot: RootPage | undefined = undefined;
|
|
102
|
+
|
|
78
103
|
constructor(
|
|
79
104
|
public readonly name: string,
|
|
80
105
|
lastPage: undefined | LastPage<Page, RootPage>,
|
|
@@ -106,6 +131,24 @@ export abstract class SyncState<
|
|
|
106
131
|
*/
|
|
107
132
|
abstract mergeDatum(ourOutput: Output, rootOutput: RootOutput): void;
|
|
108
133
|
|
|
134
|
+
recordBackpressure(atCap: boolean, cap: number): void {
|
|
135
|
+
this.bufferCap = cap;
|
|
136
|
+
const now = Date.now();
|
|
137
|
+
if (atCap && !this.pausedNow) {
|
|
138
|
+
this.pausedNow = true;
|
|
139
|
+
this.pauseStartMs = now;
|
|
140
|
+
this.backpressurePauses += 1;
|
|
141
|
+
} else if (!atCap && this.pausedNow) {
|
|
142
|
+
this.pausedNow = false;
|
|
143
|
+
this.backpressurePausedMs += now - this.pauseStartMs;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Build the resume marker for a single output: the {@link LastPage} to resume
|
|
148
|
+
* fetching *after* it. See CLAUDE.md design idea #5 (restart/resume).
|
|
149
|
+
*/
|
|
150
|
+
abstract outputToLastPage(data: Output): LastPage<Page, RootPage>;
|
|
151
|
+
|
|
109
152
|
@bound
|
|
110
153
|
*updateState(
|
|
111
154
|
_input: Input,
|
|
@@ -115,6 +158,8 @@ export abstract class SyncState<
|
|
|
115
158
|
for (const datum of data.output) {
|
|
116
159
|
this.bufferedData.push(datum);
|
|
117
160
|
}
|
|
161
|
+
// Peak right after a push — the local max before the merge drains it.
|
|
162
|
+
this.bufferHighWater = Math.max(this.bufferHighWater, this.bufferedData.size());
|
|
118
163
|
this.newDataCondVar.wake();
|
|
119
164
|
}
|
|
120
165
|
|
|
@@ -129,13 +174,8 @@ export abstract class SyncState<
|
|
|
129
174
|
),
|
|
130
175
|
);
|
|
131
176
|
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
protocol_name: this.name,
|
|
135
|
-
page_number: data.lastPage.ownBlockNumber,
|
|
136
|
-
page: JSON.stringify(data.lastPage),
|
|
137
|
-
}, this.dbConn as any)); // Client,
|
|
138
|
-
releaseDBMutex(`update-state-${this.name}`);
|
|
177
|
+
// lastPage is in-memory only; the runtime is the sole writer of the
|
|
178
|
+
// persisted resume marker (see CLAUDE.md design idea #5).
|
|
139
179
|
this.lastPage = data.lastPage;
|
|
140
180
|
this.newPageCondVar.wake();
|
|
141
181
|
}
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
type Output,
|
|
17
17
|
type Page,
|
|
18
18
|
} from "./types.ts";
|
|
19
|
-
import { genInputRange } from "../common/page-helpers.ts";
|
|
19
|
+
import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
|
|
20
20
|
import type {
|
|
21
21
|
ConfigNetworkType,
|
|
22
22
|
SyncProtocolWithNetwork,
|
|
@@ -58,6 +58,7 @@ export class BitcoinSyncState extends SyncState<
|
|
|
58
58
|
block_number: Number(data.raw.height),
|
|
59
59
|
blockHash: hash,
|
|
60
60
|
})),
|
|
61
|
+
resumePages: [],
|
|
61
62
|
primitives: data.primitives.map((primitive) => ({
|
|
62
63
|
...primitive,
|
|
63
64
|
source: this.config.syncProtocol.name,
|
|
@@ -75,6 +76,7 @@ export class BitcoinSyncState extends SyncState<
|
|
|
75
76
|
|
|
76
77
|
@bound
|
|
77
78
|
override *stateToInput(): Operation<Input | undefined> {
|
|
79
|
+
if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
|
|
78
80
|
return yield* genInputRange(
|
|
79
81
|
this as BitcoinSyncState,
|
|
80
82
|
this.config.syncProtocol.startBlockHeight as Page,
|
|
@@ -112,6 +114,16 @@ export class BitcoinSyncState extends SyncState<
|
|
|
112
114
|
rootOutput.blockInfo.push(...blockInfo);
|
|
113
115
|
}
|
|
114
116
|
|
|
117
|
+
@bound
|
|
118
|
+
override outputToLastPage(data: Output): LastPage<Page, RootPage> {
|
|
119
|
+
const block = Number(data.raw.height);
|
|
120
|
+
return {
|
|
121
|
+
own: block,
|
|
122
|
+
ownBlockNumber: block,
|
|
123
|
+
root: this.toRootPage(data),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
115
127
|
@bound
|
|
116
128
|
override getNamespace(): string[] {
|
|
117
129
|
return [this.config.network.name, this.config.syncProtocol.name];
|
|
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
|
|
|
10
10
|
import { getPage } from "@effectstream/db";
|
|
11
11
|
import { CelestiaClient } from "./CelestiaClient.ts";
|
|
12
12
|
import { applyDelay } from "../common/utils.ts";
|
|
13
|
+
import { bufferAtCap } from "../common/page-helpers.ts";
|
|
13
14
|
|
|
14
15
|
export class CelestiaSyncState extends SyncState<
|
|
15
16
|
Input,
|
|
@@ -61,6 +62,7 @@ export class CelestiaSyncState extends SyncState<
|
|
|
61
62
|
|
|
62
63
|
@bound
|
|
63
64
|
override *stateToInput(): Operation<Input | undefined> {
|
|
65
|
+
if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
|
|
64
66
|
const latestHeight = yield* call(() =>
|
|
65
67
|
this.client.getLatestBlockHeight()
|
|
66
68
|
);
|
|
@@ -99,6 +101,15 @@ export class CelestiaSyncState extends SyncState<
|
|
|
99
101
|
rootOutput.primitives.push(...primitives);
|
|
100
102
|
}
|
|
101
103
|
|
|
104
|
+
@bound
|
|
105
|
+
override outputToLastPage(data: Output): LastPage<Page, RootPage> {
|
|
106
|
+
return {
|
|
107
|
+
own: { height: data.height, hash: data.hash },
|
|
108
|
+
ownBlockNumber: data.height,
|
|
109
|
+
root: this.toRootPage(data),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
102
113
|
@bound
|
|
103
114
|
override getNamespace(): string[] {
|
|
104
115
|
return [this.config.network.name, this.config.syncProtocol.name];
|
|
@@ -5,6 +5,47 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
|
|
|
5
5
|
import type { PaginatedSyncProtocols } from "../types.ts";
|
|
6
6
|
import type { PageTypeOf } from "../base/state.ts";
|
|
7
7
|
|
|
8
|
+
// Fetch backpressure. See README "Backpressure (`maxBufferedPages`)".
|
|
9
|
+
const MAX_BUFFER_MULTIPLE = 4;
|
|
10
|
+
// Fallback chunk size for chains that fetch without an explicit stepSize
|
|
11
|
+
// (e.g. utxorpc streams one block per pass).
|
|
12
|
+
const DEFAULT_STEP_SIZE = 1000;
|
|
13
|
+
|
|
14
|
+
/** Resolved fetch-backpressure cap for a chain. See README "Backpressure". */
|
|
15
|
+
export function bufferCapFor(
|
|
16
|
+
syncProtocol: { maxBufferedPages?: number; stepSize?: number },
|
|
17
|
+
): number {
|
|
18
|
+
const step = syncProtocol.stepSize ?? DEFAULT_STEP_SIZE;
|
|
19
|
+
return Math.max(
|
|
20
|
+
syncProtocol.maxBufferedPages ?? MAX_BUFFER_MULTIPLE * step,
|
|
21
|
+
step + 1,
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Whether a chain should pause fetching because its in-memory buffer is at/over
|
|
27
|
+
* the cap. Each chain's `stateToInput` calls this first and returns `undefined`
|
|
28
|
+
* when true, so the fetch loop sleeps + retries while the merge drains. Also
|
|
29
|
+
* records backpressure metrics on `state` (for `/debug/metrics`).
|
|
30
|
+
* See README "Backpressure (`maxBufferedPages`)".
|
|
31
|
+
*/
|
|
32
|
+
export function bufferAtCap(
|
|
33
|
+
state: {
|
|
34
|
+
bufferedData: { size(): number };
|
|
35
|
+
recordBackpressure(atCap: boolean, cap: number): void;
|
|
36
|
+
mergeWaitingForPage: boolean;
|
|
37
|
+
},
|
|
38
|
+
syncProtocol: { maxBufferedPages?: number; stepSize?: number },
|
|
39
|
+
): boolean {
|
|
40
|
+
const cap = bufferCapFor(syncProtocol);
|
|
41
|
+
// Exempt this chain from the cap while the merge is blocked on its page (pausing
|
|
42
|
+
// there would deadlock — see SyncState.mergeWaitingForPage). Reporting
|
|
43
|
+
// atCap=false while exempt is correct: the chain is genuinely not paused.
|
|
44
|
+
const atCap = !state.mergeWaitingForPage && state.bufferedData.size() >= cap;
|
|
45
|
+
state.recordBackpressure(atCap, cap);
|
|
46
|
+
return atCap;
|
|
47
|
+
}
|
|
48
|
+
|
|
8
49
|
type ConfigSubset<Page> = {
|
|
9
50
|
name: string;
|
|
10
51
|
/**
|
|
@@ -5,6 +5,7 @@ import type {
|
|
|
5
5
|
TimestampMs,
|
|
6
6
|
} from "@effectstream/utils";
|
|
7
7
|
import type { PageRelation } from "../base/page.ts";
|
|
8
|
+
import type { LastPage } from "../base/state.ts";
|
|
8
9
|
import type {
|
|
9
10
|
ConfigSyncProtocolType,
|
|
10
11
|
FlattenSyncProtocolIOFor,
|
|
@@ -19,6 +20,14 @@ export type ChainBlock = {
|
|
|
19
20
|
block_number: BlockNumber;
|
|
20
21
|
blockHash: BlockHash;
|
|
21
22
|
}[];
|
|
23
|
+
/**
|
|
24
|
+
* Per-protocol block-accurate resume markers the runtime persists on commit.
|
|
25
|
+
* See `@effectstream/sync` CLAUDE.md, design idea #5 (restart/resume).
|
|
26
|
+
*/
|
|
27
|
+
resumePages: {
|
|
28
|
+
protocol_name: string;
|
|
29
|
+
lastPage: LastPage<unknown, unknown>;
|
|
30
|
+
}[];
|
|
22
31
|
primitives: (
|
|
23
32
|
& FlattenSyncProtocolIOFor<
|
|
24
33
|
ConfigSyncProtocolType
|
|
@@ -7,7 +7,7 @@ import { bound, type EvmBlockNumber } from "@effectstream/utils";
|
|
|
7
7
|
import { type LastPage, SyncState } from "../base/state.ts";
|
|
8
8
|
import type { RootOutput, RootPage } from "../types.ts";
|
|
9
9
|
import type { EvmFetcher } from "./fetcher.ts";
|
|
10
|
-
import { genInputRange } from "../common/page-helpers.ts";
|
|
10
|
+
import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
|
|
11
11
|
import type { Input, Output, Page } from "./types.ts";
|
|
12
12
|
import { toMsTimestamp } from "./types.ts";
|
|
13
13
|
import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
|
|
@@ -53,6 +53,7 @@ export class EvmSyncState extends SyncState<
|
|
|
53
53
|
|
|
54
54
|
@bound
|
|
55
55
|
override *stateToInput(): Operation<Input | undefined> {
|
|
56
|
+
if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
|
|
56
57
|
return yield* genInputRange(
|
|
57
58
|
this as EvmSyncState,
|
|
58
59
|
this.config.syncProtocol.startBlockHeight as EvmBlockNumber,
|
|
@@ -90,6 +91,16 @@ export class EvmSyncState extends SyncState<
|
|
|
90
91
|
rootOutput.blockInfo.push(...blockInfo);
|
|
91
92
|
}
|
|
92
93
|
|
|
94
|
+
@bound
|
|
95
|
+
override outputToLastPage(data: Output): LastPage<Page, RootPage> {
|
|
96
|
+
const block = Number(data.raw.number);
|
|
97
|
+
return {
|
|
98
|
+
own: block,
|
|
99
|
+
ownBlockNumber: block,
|
|
100
|
+
root: this.toRootPage(data),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
93
104
|
@bound
|
|
94
105
|
override getNamespace(): string[] {
|
|
95
106
|
return [this.config.network.name, this.config.syncProtocol.name];
|
|
@@ -40,6 +40,14 @@ export interface MidnightGqlBlockState {
|
|
|
40
40
|
// Present only on RegularTransaction (selected via inline fragment): the
|
|
41
41
|
// coin-commitment Merkle tree root after this transaction.
|
|
42
42
|
zswapMerkleTreeRoot?: string;
|
|
43
|
+
// Selected only for MidnightTokenMintPrimitive: the serialized
|
|
44
|
+
// transaction bytes (Transaction interface) and the apply result
|
|
45
|
+
// (RegularTransaction only — absent on system transactions).
|
|
46
|
+
raw?: string;
|
|
47
|
+
transactionResult?: {
|
|
48
|
+
status: "SUCCESS" | "PARTIAL_SUCCESS" | "FAILURE";
|
|
49
|
+
segments?: { id: number; success: boolean }[] | null;
|
|
50
|
+
};
|
|
43
51
|
}[];
|
|
44
52
|
};
|
|
45
53
|
};
|
|
@@ -55,6 +63,8 @@ export interface BlockFetchOptions {
|
|
|
55
63
|
unshieldedCreatedOutputs?: boolean;
|
|
56
64
|
/** Include `... on RegularTransaction { zswapMerkleTreeRoot }` (needed for MidnightZswapRootPrimitive). Default: false */
|
|
57
65
|
zswapRoots?: boolean;
|
|
66
|
+
/** Include tx `raw` + `... on RegularTransaction { transactionResult }` (needed for MidnightTokenMintPrimitive). Default: false */
|
|
67
|
+
tokenMints?: boolean;
|
|
58
68
|
}
|
|
59
69
|
|
|
60
70
|
type PublicDataProvider = ReturnType<typeof indexerPublicDataProvider>;
|
|
@@ -182,6 +192,7 @@ export class MidnightClient {
|
|
|
182
192
|
unshieldedSpentOutputs = false,
|
|
183
193
|
unshieldedCreatedOutputs = false,
|
|
184
194
|
zswapRoots = false,
|
|
195
|
+
tokenMints = false,
|
|
185
196
|
} = options;
|
|
186
197
|
const contractActionsField = contractActions
|
|
187
198
|
? `contractActions { address state }`
|
|
@@ -200,6 +211,14 @@ export class MidnightClient {
|
|
|
200
211
|
const zswapRootFragment = zswapRoots
|
|
201
212
|
? `... on RegularTransaction { zswapMerkleTreeRoot }`
|
|
202
213
|
: "";
|
|
214
|
+
// `raw` is on the Transaction interface; `transactionResult` is
|
|
215
|
+
// RegularTransaction-only (system transactions have neither result nor
|
|
216
|
+
// contract calls, so they are skipped downstream). Duplicate inline
|
|
217
|
+
// fragments merge legally in GraphQL.
|
|
218
|
+
const tokenMintFields = tokenMints
|
|
219
|
+
? `raw
|
|
220
|
+
... on RegularTransaction { transactionResult { status segments { id success } } }`
|
|
221
|
+
: "";
|
|
203
222
|
const query = `query {
|
|
204
223
|
block(offset: { height: ${blockHeight} }) {
|
|
205
224
|
hash
|
|
@@ -216,6 +235,7 @@ export class MidnightClient {
|
|
|
216
235
|
${unshieldedSpentField}
|
|
217
236
|
${unshieldedCreatedField}
|
|
218
237
|
${zswapRootFragment}
|
|
238
|
+
${tokenMintFields}
|
|
219
239
|
}
|
|
220
240
|
}
|
|
221
241
|
}`;
|
|
@@ -19,6 +19,7 @@ import { bound } from "@effectstream/utils";
|
|
|
19
19
|
import { MidnightClient, type BlockFetchOptions, type MidnightGqlBlockState } from "./MidnightClient.ts";
|
|
20
20
|
import { ContractState, StateValue } from "@midnight-ntwrk/onchain-runtime";
|
|
21
21
|
import { decodeZswapInputEvent } from "./zswap-decoder.ts";
|
|
22
|
+
import { decodeTokenMints } from "./mint-decoder.ts";
|
|
22
23
|
|
|
23
24
|
export class MidnightFetcher extends BaseDataFetcher<
|
|
24
25
|
Input,
|
|
@@ -86,7 +87,8 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
86
87
|
p.primitive.type !== "Midnight:Nullifier" &&
|
|
87
88
|
p.primitive.type !== "Midnight:UnshieldedSpend" &&
|
|
88
89
|
p.primitive.type !== "Midnight:UnshieldedCreate" &&
|
|
89
|
-
p.primitive.type !== "Midnight:ZswapRoot"
|
|
90
|
+
p.primitive.type !== "Midnight:ZswapRoot" &&
|
|
91
|
+
p.primitive.type !== "Midnight:TokenMint",
|
|
90
92
|
),
|
|
91
93
|
zswapLedgerEvents: this.config.primitives.some(
|
|
92
94
|
(p) => p.primitive.type === "Midnight:Nullifier",
|
|
@@ -100,6 +102,9 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
100
102
|
zswapRoots: this.config.primitives.some(
|
|
101
103
|
(p) => p.primitive.type === "Midnight:ZswapRoot",
|
|
102
104
|
),
|
|
105
|
+
tokenMints: this.config.primitives.some(
|
|
106
|
+
(p) => p.primitive.type === "Midnight:TokenMint",
|
|
107
|
+
),
|
|
103
108
|
};
|
|
104
109
|
const self = this;
|
|
105
110
|
const heights = Array.from(
|
|
@@ -192,6 +197,8 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
192
197
|
syncResults.push(...this.fetchUnshieldedCreates(height, primitiveEntry, block));
|
|
193
198
|
} else if (primitiveEntry.primitive.type === "Midnight:ZswapRoot") {
|
|
194
199
|
syncResults.push(...this.fetchZswapRoots(height, primitiveEntry, block));
|
|
200
|
+
} else if (primitiveEntry.primitive.type === "Midnight:TokenMint") {
|
|
201
|
+
syncResults.push(...this.fetchTokenMints(height, primitiveEntry, block));
|
|
195
202
|
} else {
|
|
196
203
|
asyncOps.push(
|
|
197
204
|
this.fetchContractState(height, client, primitiveEntry, block),
|
|
@@ -339,6 +346,42 @@ export class MidnightFetcher extends BaseDataFetcher<
|
|
|
339
346
|
];
|
|
340
347
|
}
|
|
341
348
|
|
|
349
|
+
// Decode custom token mints out of each regular transaction's raw bytes
|
|
350
|
+
// (ledger-v8 deserialize → contract call transcript effects). One input per
|
|
351
|
+
// (tx, call, domainSep, kind) that actually applied on chain — see
|
|
352
|
+
// mint-decoder.ts for the segment semantics. System transactions carry no
|
|
353
|
+
// transactionResult and are skipped.
|
|
354
|
+
@bound
|
|
355
|
+
fetchTokenMints(
|
|
356
|
+
height: number,
|
|
357
|
+
primitiveEntry: PrimitiveEntryType,
|
|
358
|
+
block: MidnightGqlBlockState,
|
|
359
|
+
): PrimitiveType[] {
|
|
360
|
+
const results: PrimitiveType[] = [];
|
|
361
|
+
for (const tx of block.block.transactions) {
|
|
362
|
+
if (!tx.raw || !tx.transactionResult) continue;
|
|
363
|
+
for (const mint of decodeTokenMints(tx.raw, tx.transactionResult)) {
|
|
364
|
+
results.push({
|
|
365
|
+
syncProtocol: {
|
|
366
|
+
name: primitiveEntry.syncProtocol,
|
|
367
|
+
blockNumber: height,
|
|
368
|
+
transactionHash: tx.hash,
|
|
369
|
+
contractAddress: mint.contractAddress,
|
|
370
|
+
},
|
|
371
|
+
primitive: primitiveEntry.primitive.name,
|
|
372
|
+
output: {
|
|
373
|
+
payloadType: "midnight-token-mint",
|
|
374
|
+
payload: {
|
|
375
|
+
...mint,
|
|
376
|
+
txHash: tx.hash,
|
|
377
|
+
},
|
|
378
|
+
},
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
return results;
|
|
383
|
+
}
|
|
384
|
+
|
|
342
385
|
@bound
|
|
343
386
|
*fetchContractState(
|
|
344
387
|
height: number,
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { Buffer } from "node:buffer";
|
|
2
|
+
import * as ledger from "@midnight-ntwrk/ledger-v8";
|
|
3
|
+
|
|
4
|
+
// Decodes custom token mints out of a raw Midnight transaction. A contract
|
|
5
|
+
// call's transcript effects record mints as `domain_sep → amount` maps
|
|
6
|
+
// (`shieldedMints` / `unshieldedMints`); the resulting token type ("color")
|
|
7
|
+
// is `rawTokenType(domain_sep, contract_address)`. Neither the indexer's
|
|
8
|
+
// GraphQL nor the wallet expose this mapping, so the only route is
|
|
9
|
+
// deserializing the transaction bytes with ledger-v8 — same approach as
|
|
10
|
+
// zswap-decoder.ts for nullifier events.
|
|
11
|
+
//
|
|
12
|
+
// Mirrors zswap-decoder.ts tolerance: never throws — an undecodable
|
|
13
|
+
// transaction logs a warning and contributes nothing.
|
|
14
|
+
|
|
15
|
+
export interface TokenMintRecord {
|
|
16
|
+
/** Address of the minting contract, hex */
|
|
17
|
+
contractAddress: string;
|
|
18
|
+
/** 32-byte domain separator pre-image, hex */
|
|
19
|
+
domainSep: string;
|
|
20
|
+
/** Derived token type ("color") = rawTokenType(domainSep, contractAddress), hex */
|
|
21
|
+
rawTokenType: string;
|
|
22
|
+
kind: "shielded" | "unshielded";
|
|
23
|
+
/** Total minted for this (call, domainSep, kind), as a decimal string (u64 exceeds MAX_SAFE_INTEGER) */
|
|
24
|
+
amount: string;
|
|
25
|
+
/** Contract entry point that minted, when printable */
|
|
26
|
+
entryPoint?: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface TxResultLike {
|
|
30
|
+
status: "SUCCESS" | "PARTIAL_SUCCESS" | "FAILURE";
|
|
31
|
+
segments?: { id: number; success: boolean }[] | null;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function hexToBytes(hex: string): Uint8Array {
|
|
35
|
+
const clean = hex.startsWith("0x") ? hex.slice(2) : hex;
|
|
36
|
+
return Uint8Array.from(Buffer.from(clean, "hex"));
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function bytesToHex(bytes: Uint8Array): string {
|
|
40
|
+
return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function normalizeHex(value: string): string {
|
|
44
|
+
return value.replace(/^0x/, "").toLowerCase();
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Extract every applied token mint from a raw serialized transaction.
|
|
49
|
+
*
|
|
50
|
+
* Segment semantics (failed segments roll back on chain):
|
|
51
|
+
* - no `result` (system transactions) or status FAILURE → nothing applied;
|
|
52
|
+
* - SUCCESS → guaranteed + all fallible transcripts applied;
|
|
53
|
+
* - PARTIAL_SUCCESS → guaranteed transcripts applied, fallible only where
|
|
54
|
+
* the intent's segment (the `intents` map key) succeeded.
|
|
55
|
+
*/
|
|
56
|
+
export function decodeTokenMints(
|
|
57
|
+
rawHex: string,
|
|
58
|
+
result?: TxResultLike | null,
|
|
59
|
+
): TokenMintRecord[] {
|
|
60
|
+
if (!result || result.status === "FAILURE") return [];
|
|
61
|
+
|
|
62
|
+
let tx: any;
|
|
63
|
+
try {
|
|
64
|
+
tx = ledger.Transaction.deserialize(
|
|
65
|
+
"signature",
|
|
66
|
+
"proof",
|
|
67
|
+
"binding",
|
|
68
|
+
hexToBytes(rawHex),
|
|
69
|
+
);
|
|
70
|
+
} catch (e) {
|
|
71
|
+
console.warn(
|
|
72
|
+
`[mint-decoder] undecodable transaction, skipping: ${
|
|
73
|
+
(e as Error)?.message ?? e
|
|
74
|
+
}`,
|
|
75
|
+
);
|
|
76
|
+
return [];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const records: TokenMintRecord[] = [];
|
|
80
|
+
for (const [segmentId, intent] of tx.intents ?? new Map()) {
|
|
81
|
+
const fallibleApplied = result.status === "SUCCESS" ||
|
|
82
|
+
(result.segments?.some((s) => s.id === Number(segmentId) && s.success) ??
|
|
83
|
+
false);
|
|
84
|
+
for (const action of intent.actions ?? []) {
|
|
85
|
+
// Only contract calls carry transcripts; deploys and maintenance
|
|
86
|
+
// updates cannot mint.
|
|
87
|
+
if (!(action instanceof ledger.ContractCall)) continue;
|
|
88
|
+
try {
|
|
89
|
+
records.push(...mintsOfCall(action, fallibleApplied));
|
|
90
|
+
} catch (e) {
|
|
91
|
+
console.warn(
|
|
92
|
+
`[mint-decoder] failed to read mints of call to ${
|
|
93
|
+
(action as any)?.address ?? "?"
|
|
94
|
+
}: ${(e as Error)?.message ?? e}`,
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return records;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function mintsOfCall(
|
|
103
|
+
call: any,
|
|
104
|
+
fallibleApplied: boolean,
|
|
105
|
+
): TokenMintRecord[] {
|
|
106
|
+
// Accumulate domainSep|kind → amount across the applied transcripts. The
|
|
107
|
+
// guaranteed transcript applied whenever the transaction wasn't a FAILURE
|
|
108
|
+
// (the caller filters those); the fallible transcript only if its segment
|
|
109
|
+
// succeeded.
|
|
110
|
+
const totals = new Map<string, bigint>();
|
|
111
|
+
const transcripts = [
|
|
112
|
+
call.guaranteedTranscript,
|
|
113
|
+
...(fallibleApplied ? [call.fallibleTranscript] : []),
|
|
114
|
+
];
|
|
115
|
+
for (const transcript of transcripts) {
|
|
116
|
+
const effects = transcript?.effects;
|
|
117
|
+
if (!effects) continue;
|
|
118
|
+
for (
|
|
119
|
+
const [kind, mints] of [
|
|
120
|
+
["shielded", effects.shieldedMints],
|
|
121
|
+
["unshielded", effects.unshieldedMints],
|
|
122
|
+
] as const
|
|
123
|
+
) {
|
|
124
|
+
for (const [sepHex, amount] of mints ?? new Map()) {
|
|
125
|
+
const key = `${normalizeHex(String(sepHex))}|${kind}`;
|
|
126
|
+
totals.set(key, (totals.get(key) ?? 0n) + BigInt(amount));
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const records: TokenMintRecord[] = [];
|
|
132
|
+
for (const [key, amount] of totals) {
|
|
133
|
+
const [domainSep, kind] = key.split("|") as [
|
|
134
|
+
string,
|
|
135
|
+
"shielded" | "unshielded",
|
|
136
|
+
];
|
|
137
|
+
records.push({
|
|
138
|
+
contractAddress: normalizeHex(String(call.address)),
|
|
139
|
+
domainSep,
|
|
140
|
+
rawTokenType: normalizeHex(
|
|
141
|
+
String(ledger.rawTokenType(hexToBytes(domainSep), call.address)),
|
|
142
|
+
),
|
|
143
|
+
kind,
|
|
144
|
+
amount: amount.toString(),
|
|
145
|
+
entryPoint: typeof call.entryPoint === "string"
|
|
146
|
+
? call.entryPoint
|
|
147
|
+
: bytesToHex(call.entryPoint),
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
return records;
|
|
151
|
+
}
|
|
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
|
|
|
10
10
|
import { getPage } from "@effectstream/db";
|
|
11
11
|
import { MidnightClient } from "./MidnightClient.ts";
|
|
12
12
|
import { applyDelay } from "../common/utils.ts";
|
|
13
|
+
import { bufferAtCap } from "../common/page-helpers.ts";
|
|
13
14
|
|
|
14
15
|
type LatestBlock = {
|
|
15
16
|
block: {
|
|
@@ -72,6 +73,7 @@ export class MidnightSyncState extends SyncState<
|
|
|
72
73
|
|
|
73
74
|
@bound
|
|
74
75
|
override *stateToInput(): Operation<Input | undefined> {
|
|
76
|
+
if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
|
|
75
77
|
const latestBlockResult = yield* call(() => this.client.fetchLatestBlock());
|
|
76
78
|
const latestHeight = latestBlockResult.block.height;
|
|
77
79
|
|
|
@@ -110,6 +112,15 @@ export class MidnightSyncState extends SyncState<
|
|
|
110
112
|
rootOutput.primitives.push(...primitives);
|
|
111
113
|
}
|
|
112
114
|
|
|
115
|
+
@bound
|
|
116
|
+
override outputToLastPage(data: Output): LastPage<Page, RootPage> {
|
|
117
|
+
return {
|
|
118
|
+
own: { height: data.raw.height, hash: data.raw.hash },
|
|
119
|
+
ownBlockNumber: data.raw.height,
|
|
120
|
+
root: this.toRootPage(data),
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
|
|
113
124
|
@bound
|
|
114
125
|
override getNamespace(): string[] {
|
|
115
126
|
return [this.config.network.name, this.config.syncProtocol.name];
|
|
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
|
|
|
10
10
|
import { getPage } from "@effectstream/db";
|
|
11
11
|
import { NearClient } from "./NearClient.ts";
|
|
12
12
|
import { applyDelay } from "../common/utils.ts";
|
|
13
|
+
import { bufferAtCap } from "../common/page-helpers.ts";
|
|
13
14
|
|
|
14
15
|
export class NearSyncState extends SyncState<
|
|
15
16
|
Input,
|
|
@@ -60,6 +61,7 @@ export class NearSyncState extends SyncState<
|
|
|
60
61
|
|
|
61
62
|
@bound
|
|
62
63
|
override *stateToInput(): Operation<Input | undefined> {
|
|
64
|
+
if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
|
|
63
65
|
const latestHeight = yield* call(() =>
|
|
64
66
|
this.client.getLatestBlockHeight()
|
|
65
67
|
);
|
|
@@ -98,6 +100,15 @@ export class NearSyncState extends SyncState<
|
|
|
98
100
|
rootOutput.primitives.push(...primitives);
|
|
99
101
|
}
|
|
100
102
|
|
|
103
|
+
@bound
|
|
104
|
+
override outputToLastPage(data: Output): LastPage<Page, RootPage> {
|
|
105
|
+
return {
|
|
106
|
+
own: { height: data.height, hash: data.hash },
|
|
107
|
+
ownBlockNumber: data.height,
|
|
108
|
+
root: this.toRootPage(data),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
|
|
101
112
|
@bound
|
|
102
113
|
override getNamespace(): string[] {
|
|
103
114
|
return [this.config.network.name, this.config.syncProtocol.name];
|
|
@@ -7,7 +7,7 @@ import { bound, type NtpBlockNumber } from "@effectstream/utils";
|
|
|
7
7
|
import { type LastPage, SyncState } from "../base/state.ts";
|
|
8
8
|
import type { RootOutput, RootPage } from "../types.ts";
|
|
9
9
|
import type { NtpFetcher } from "./fetcher.ts";
|
|
10
|
-
import { genInputRange } from "../common/page-helpers.ts";
|
|
10
|
+
import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
|
|
11
11
|
import type { Input, Output, Page } from "./types.ts";
|
|
12
12
|
import { toMsTimestamp } from "./types.ts";
|
|
13
13
|
import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
|
|
@@ -48,6 +48,7 @@ export class NtpSyncState extends SyncState<
|
|
|
48
48
|
})),
|
|
49
49
|
blockNumber: Number(data.raw.blockNumber),
|
|
50
50
|
timestamp: this.toRootPage(data),
|
|
51
|
+
resumePages: [],
|
|
51
52
|
primitives: [],
|
|
52
53
|
};
|
|
53
54
|
}
|
|
@@ -59,6 +60,7 @@ export class NtpSyncState extends SyncState<
|
|
|
59
60
|
|
|
60
61
|
@bound
|
|
61
62
|
override *stateToInput(): Operation<Input | undefined> {
|
|
63
|
+
if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
|
|
62
64
|
return yield* genInputRange(
|
|
63
65
|
this as NtpSyncState,
|
|
64
66
|
1, // NTP Pages start from 1, and the fetcher offsets using the start page (date)
|
|
@@ -86,6 +88,16 @@ export class NtpSyncState extends SyncState<
|
|
|
86
88
|
throw new Error("Only parallel chains merge into root");
|
|
87
89
|
}
|
|
88
90
|
|
|
91
|
+
@bound
|
|
92
|
+
override outputToLastPage(data: Output): LastPage<Page, RootPage> {
|
|
93
|
+
const block = Number(data.raw.blockNumber);
|
|
94
|
+
return {
|
|
95
|
+
own: block,
|
|
96
|
+
ownBlockNumber: block,
|
|
97
|
+
root: this.toRootPage(data),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
89
101
|
@bound
|
|
90
102
|
override getNamespace(): string[] {
|
|
91
103
|
return [this.config.network.name, this.config.syncProtocol.name];
|
|
@@ -124,6 +124,11 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
|
|
|
124
124
|
}
|
|
125
125
|
const output = iState.bufferedData.peekAt(0)!;
|
|
126
126
|
const newRoot = iState.toRootOutput(output.output);
|
|
127
|
+
// Resume marker: the single datum this root block is built from (CLAUDE.md #5).
|
|
128
|
+
newRoot.resumePages.push({
|
|
129
|
+
protocol_name: state.name,
|
|
130
|
+
lastPage: iState.outputToLastPage(output.output),
|
|
131
|
+
});
|
|
127
132
|
return {
|
|
128
133
|
updateCache: () => {
|
|
129
134
|
output.cleanup();
|
|
@@ -141,6 +146,11 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
|
|
|
141
146
|
rootInfo.toPage(rootInfo.value),
|
|
142
147
|
) <= 0
|
|
143
148
|
) {
|
|
149
|
+
// Publish the demand before waiting so the cap exempts this chain while the
|
|
150
|
+
// merge is gated on its page (see SyncState.mergeWaitingForPage). No lock
|
|
151
|
+
// needed: the fetch loop re-polls stateToInput every pollingInterval.
|
|
152
|
+
state.mergeWaitingForPage = true;
|
|
153
|
+
state.mergeDemandRoot = rootInfo.toPage(rootInfo.value);
|
|
144
154
|
log.remote(
|
|
145
155
|
ComponentNames.EFFECTSTREAM_SYNC,
|
|
146
156
|
state.getNamespace(),
|
|
@@ -155,6 +165,8 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
|
|
|
155
165
|
(log) => log("Merge unblocked (missing parallel data): got data"),
|
|
156
166
|
);
|
|
157
167
|
}
|
|
168
|
+
state.mergeWaitingForPage = false;
|
|
169
|
+
state.mergeDemandRoot = undefined;
|
|
158
170
|
const cleanups = getFromBuffer(
|
|
159
171
|
{
|
|
160
172
|
data: iState.bufferedData,
|
|
@@ -168,6 +180,17 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
|
|
|
168
180
|
iState.mergeDatum,
|
|
169
181
|
);
|
|
170
182
|
|
|
183
|
+
// Resume marker: the last (highest) datum merged this iteration, still in the
|
|
184
|
+
// Deque at `cleanups.length - 1` (cleanup runs after send). Nothing consumed →
|
|
185
|
+
// no marker. See CLAUDE.md #5.
|
|
186
|
+
if (cleanups.length > 0) {
|
|
187
|
+
const lastConsumed = iState.bufferedData.peekAt(cleanups.length - 1)!;
|
|
188
|
+
rootInfo.value.resumePages.push({
|
|
189
|
+
protocol_name: state.name,
|
|
190
|
+
lastPage: iState.outputToLastPage(lastConsumed.output),
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
|
|
171
194
|
return {
|
|
172
195
|
updateCache: () => {
|
|
173
196
|
state.bufferedData.remove(0, cleanups.length);
|
|
@@ -7,7 +7,7 @@ import { bound, type BlockNumber } from "@effectstream/utils";
|
|
|
7
7
|
import { type LastPage, SyncState } from "../base/state.ts";
|
|
8
8
|
import type { RootOutput, RootPage } from "../types.ts";
|
|
9
9
|
import type { TestFetcher } from "./fetcher.ts";
|
|
10
|
-
import { genInputRange } from "../common/page-helpers.ts";
|
|
10
|
+
import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
|
|
11
11
|
import type { Input, Output, Page } from "./types.ts";
|
|
12
12
|
import { toMsTimestamp } from "./types.ts";
|
|
13
13
|
import type {
|
|
@@ -60,6 +60,7 @@ export class TestSyncState extends SyncState<
|
|
|
60
60
|
})),
|
|
61
61
|
blockNumber: Number(data.raw.blockNumber),
|
|
62
62
|
timestamp: this.toRootPage(data),
|
|
63
|
+
resumePages: [],
|
|
63
64
|
primitives: data.primitives.map((p) => ({
|
|
64
65
|
...p,
|
|
65
66
|
source: this.config.syncProtocol.name,
|
|
@@ -77,6 +78,7 @@ export class TestSyncState extends SyncState<
|
|
|
77
78
|
|
|
78
79
|
@bound
|
|
79
80
|
override *stateToInput(): Operation<Input | undefined> {
|
|
81
|
+
if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
|
|
80
82
|
return yield* genInputRange(
|
|
81
83
|
this as TestSyncState,
|
|
82
84
|
this.config.syncProtocol.startBlockHeight as BlockNumber,
|
|
@@ -114,6 +116,16 @@ export class TestSyncState extends SyncState<
|
|
|
114
116
|
rootOutput.blockInfo.push(...blockInfo);
|
|
115
117
|
}
|
|
116
118
|
|
|
119
|
+
@bound
|
|
120
|
+
override outputToLastPage(data: Output): LastPage<Page, RootPage> {
|
|
121
|
+
const block = Number(data.raw.blockNumber);
|
|
122
|
+
return {
|
|
123
|
+
own: block,
|
|
124
|
+
ownBlockNumber: block as BlockNumber,
|
|
125
|
+
root: this.toRootPage(data),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
117
129
|
@bound
|
|
118
130
|
override getNamespace(): string[] {
|
|
119
131
|
return [this.config.network.name, this.config.syncProtocol.name];
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { applyDelay } from "../common/utils.ts";
|
|
2
|
+
import { bufferAtCap } from "../common/page-helpers.ts";
|
|
2
3
|
import type { Operation } from "effection";
|
|
3
4
|
import { call } from "effection";
|
|
4
5
|
import { getPage } from "@effectstream/db";
|
|
@@ -71,6 +72,7 @@ export class UtxoRpcSyncState extends SyncState<
|
|
|
71
72
|
|
|
72
73
|
@bound
|
|
73
74
|
override *stateToInput(): Operation<Input | undefined> {
|
|
75
|
+
if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
|
|
74
76
|
// Get the value of the latest block height.
|
|
75
77
|
const tipHeight = yield* call(() => this.fetcher.lastHeight());
|
|
76
78
|
if (!tipHeight) {
|
|
@@ -157,6 +159,22 @@ export class UtxoRpcSyncState extends SyncState<
|
|
|
157
159
|
rootOutput.blockInfo.push(...blockInfo);
|
|
158
160
|
}
|
|
159
161
|
|
|
162
|
+
@bound
|
|
163
|
+
override outputToLastPage(data: Output): LastPage<Page, RootPage> {
|
|
164
|
+
// Mirror the fetcher's `own`: slot/height from the block header, and the
|
|
165
|
+
// block hash from `blockHashes` (the fetcher builds it as `[completed.hash]`,
|
|
166
|
+
// the same value it stored in `own.hash`).
|
|
167
|
+
return {
|
|
168
|
+
own: {
|
|
169
|
+
slot: Number(data.raw.header!.slot),
|
|
170
|
+
height: Number(data.raw.header!.height),
|
|
171
|
+
hash: data.blockHashes[0],
|
|
172
|
+
},
|
|
173
|
+
ownBlockNumber: Number(data.raw.header!.height),
|
|
174
|
+
root: this.toRootPage(data),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
|
|
160
178
|
@bound
|
|
161
179
|
override getNamespace(): string[] {
|
|
162
180
|
return [this.config.network.name, this.config.syncProtocol.name];
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { decodeTokenMints } from "../src/sync-protocols/midnight/mint-decoder.ts";
|
|
4
|
+
|
|
5
|
+
// Tolerance contract: the decoder must never throw — an undecodable or
|
|
6
|
+
// inapplicable transaction contributes nothing (the fetcher's per-height
|
|
7
|
+
// error path would otherwise wedge the sync loop).
|
|
8
|
+
describe("decodeTokenMints — tolerance", () => {
|
|
9
|
+
test("no transactionResult (system tx) → []", () => {
|
|
10
|
+
expect(decodeTokenMints("deadbeef", undefined)).toEqual([]);
|
|
11
|
+
expect(decodeTokenMints("deadbeef", null)).toEqual([]);
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
test("status FAILURE → [] (everything rolled back, incl. guaranteed)", () => {
|
|
15
|
+
expect(
|
|
16
|
+
decodeTokenMints("deadbeef", { status: "FAILURE", segments: [] }),
|
|
17
|
+
).toEqual([]);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test("garbage bytes with SUCCESS status → [] without throwing", () => {
|
|
21
|
+
expect(
|
|
22
|
+
decodeTokenMints("not-even-hex", { status: "SUCCESS" }),
|
|
23
|
+
).toEqual([]);
|
|
24
|
+
expect(
|
|
25
|
+
decodeTokenMints("00".repeat(64), { status: "SUCCESS" }),
|
|
26
|
+
).toEqual([]);
|
|
27
|
+
});
|
|
28
|
+
});
|