@effectstream/sync 0.100.22 → 0.100.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CLAUDE.md CHANGED
@@ -35,7 +35,7 @@ config (syncProtocols.main + syncProtocols.parallel)
35
35
  │ PER-CHAIN FETCH LOOPS one spawn per protocol │
36
36
  │ startSync(state): stateToInput → readData → updateState │
37
37
  │ → push Output to bufferedData (Deque) + wake newData/newPageCondVar│
38
- │ → upsertPage(page_number = chunk_end) into sync_protocol_pagination │
38
+ │ → lastPage kept IN-MEMORY only (no DB write; runtime owns resume) │
39
39
  │ → producerChannel.send(...) ← NOTE: nothing consumes this today │
40
40
  └────────────────────────────────────────┬─────────────────────────────┘
41
41
  ▼
@@ -47,8 +47,9 @@ config (syncProtocols.main + syncProtocols.parallel)
47
47
  └────────────────────────────────────────┬─────────────────────────────┘
48
48
  ▼
49
49
  runtime/src/main.ts: for (block of each(finalizedBlockStream))
50
- acquire DB mutex → processFinalizedBlock (STF) → COMMIT
51
- → removePages(< merged block) → fire-and-forget MQTT events → each.next()
50
+ acquire DB mutex → processFinalizedBlock (STF)
51
+ → upsertPage(block.resumePages) + removePages(< committed) [in txn] → COMMIT
52
+ → fire-and-forget MQTT events → each.next()
52
53
  ```
53
54
 
54
55
  ## The three per-chain abstractions (`src/sync-protocols/<chain>/`)
@@ -112,12 +113,34 @@ iteration it walks protocols in order:
112
113
  a parallel chain's timestamp so older blocks merge into more-recent root slots.
113
114
  4. **Mutate-after-commit.** `OutputAndCleanup` keeps data in the Deque until the block is
114
115
  safely produced; the consumer commits to Postgres *before* emitting events.
115
- 5. **Durable, resumable page queue.** `effectstream.sync_protocol_pagination` is keyed
116
- `(protocol_name, page_number)`. Fetch appends a row per chunk (`upsertPage`,
117
- `page_number = chunk_end`); on commit the consumer deletes consumed chunks
118
- (`removePages(page_number < merged_block)`, `runtime/process-blocks.ts:328`); on boot
119
- `restoreState` resumes from `getPage` = `ORDER BY page_number ASC LIMIT 1` (oldest
120
- remaining chunk).
116
+ 5. **Durable, block-accurate resume queue.** *(Single source of truth for how restart/resume
117
+ works — code comments point here; see also Finding #2 for the bug this fixes.)*
118
+
119
+ `effectstream.sync_protocol_pagination` is keyed `(protocol_name, page_number)` and the
120
+ **runtime is the sole writer** of the resume position (Fix D). End to end:
121
+ - **Fetch loop** (`base/state.ts:updateState`) keeps `lastPage` in memory only — it does
122
+ **not** persist (it runs ahead of commits, so its chunk-end page is not a safe resume
123
+ point).
124
+ - **Merge** (`orchestration/merge.ts`) tags every `ChainBlock` with `resumePages`: one
125
+ marker per protocol = `outputToLastPage(lastConsumed)`, the `LastPage`
126
+ (`{ own, ownBlockNumber, root }`) of the **highest datum merged into the block**. Each
127
+ protocol implements `outputToLastPage` (`base/state.ts` + per-chain `state.ts`) because
128
+ `own` is a structured page for object-paged chains and can't be rebuilt from the block
129
+ number. Pending data above the committed timestamp is **not** marked, so it is always
130
+ re-fetched, never skipped.
131
+ - **Commit** (`runtime/process-blocks.ts`, STEP 7) writes each marker **inside the block's
132
+ transaction**: `upsertPage(page_number = ownBlockNumber)` then
133
+ `removePages(page_number < ownBlockNumber)` — leaving exactly one row per protocol = the
134
+ committed watermark, atomic with the block.
135
+ - **Boot** (`restoreState`) reads `getPage` = `ORDER BY page_number ASC LIMIT 1` and
136
+ resumes from `nextInterval(own).from` = the next uncommitted block.
137
+
138
+ **Re-scan on restart is bounded by `stepSize`, not a chain's whole quiet tail.** Every
139
+ fetched chunk's boundary page (`data.to`) is buffered and consumed even when empty, so the
140
+ marker advances to within one `stepSize` of the committed frontier — and reaches the
141
+ fetched tip exactly at full catch-up. So a long-quiet chain re-fetches at most ~`stepSize`
142
+ blocks after a restart, regardless of how long ago its last data was. Trade-off:
143
+ `getSyncAndLastPage` now reports `synced_page == fetched_page`.
121
144
 
122
145
  ---
123
146
 
@@ -144,26 +167,42 @@ Consequences during catch-up:
144
167
  (`mergeIntoRoot`, `merge.ts`), so a slow/stalled chain stalls all block production while
145
168
  the others keep ballooning (head-of-line blocking).
146
169
 
147
- → Fix: **(C) backpressure** — cap `bufferedData` and pause the fetch loop until the merge drains.
148
-
149
- ### 2. Silent data gap on restart (correctness)
150
-
151
- The page queue is **chunk-granular** but commits are **single-block-granular**:
152
- - fetch persists `upsertPage(page_number = chunk_end)` (`base/state.ts:132`; `own = ownBlockNumber = Number(data.to)`),
153
- - commit deletes `removePages(page_number < merged_block_N)` (`process-blocks.ts:328`),
154
- - restart resumes from `getPage` = oldest remaining chunk, at `nextInterval(own).from = chunk_end + 1`.
155
-
156
- During catch-up chunks are full `stepSize`, so at restart the committed watermark `N` sits
157
- **inside** the oldest retained chunk `[start, end]` (`start ≤ N < end ⇒ end ≥ N ⇒ not deleted`).
158
- Resume jumps to `end + 1`, **silently skipping source blocks `(N, end]`** (up to
159
- `stepSize - 1`). The in-memory Deque held them; they're gone on restart; the effectstream
160
- height continues seamlessly with **missing primitives → divergent app state**. There is
161
- **no error** and **no restart/resume test** (the only tests in `test/examples.test.ts` are
162
- export smoke-checks). The existing `getSyncAndLastPage` `synced_page` (`MIN(page_number)`)
163
- is itself chunk-granular and does not reflect the true committed block.
164
-
165
- → Fix: **(D) block-accurate resume** — persist the committed per-protocol watermark and
166
- resume from `watermark + 1`, reconciling stale fetched-ahead page rows.
170
+ → Fix: **(C) backpressure** — ✅ SHIPPED. `bufferAtCap` (`common/page-helpers.ts`) caps
171
+ `bufferedData` and pauses the fetch loop (each chain's `stateToInput` returns `undefined`)
172
+ until the merge drains. See README "Backpressure (`maxBufferedPages`)".
173
+
174
+ > **Merge→apply queue cap (Fix C′) — ✅ SHIPPED.** The Deque cap above bounds *fetch*, but the
175
+ > merge then sends finalized blocks into an effection channel whose subscriber queue buffers
176
+ > unbounded — so the merge (no I/O) could still race ahead of the serial apply loop and grow that
177
+ > queue toward the whole backlog, especially with empty-block coalescing (Fix E) removing the
178
+ > per-block apply brake. The runtime now gates the merge's `send` (`runtime/src/main.ts`): it blocks
179
+ > once `EFFECTSTREAM_FINALIZED_STREAM_CAP` (default 2048) produced blocks are unconsumed, and the
180
+ > apply loop wakes it as it drains. `inFlight = produced − consumed` is surfaced on
181
+ > `/debug/metrics`. Repro/guard: `runtime/test/reproduction/coalesce-memory.test.ts`.
182
+
183
+ > **Merge-demand exemption (safeguard).** A naive cap can deadlock the merge: it drains a
184
+ > parallel chain's buffer only *after* its page passes the root timestamp `τ`, so if the cap
185
+ > pauses the fetcher while its page is still `≤ τ` (a far skip-ahead, or a chain finer-grained
186
+ > than the cap), merge waits for the page ↔ the page waits for a fetch ↔ the fetch waits for
187
+ > the buffer to drain — circular. Guard: `mergeIntoRoot` sets `SyncState.mergeWaitingForPage`
188
+ > while blocked on a chain's page, and `bufferAtCap` returns `false` for that chain while the
189
+ > flag is set — so the fetcher advances past `τ`. Bounded by necessity: only
190
+ > `(lastPage.root, τ]` is buffered above the cap. Repros: `buffering.test.ts` 1c/1d.
191
+
192
+ ### 2. Silent data gap on restart (correctness) — ✅ FIXED (D)
193
+
194
+ **The bug (historical).** The page queue was **chunk-granular** but commits are
195
+ **single-block-granular**: the fetch loop persisted `upsertPage(page_number = chunk_end)`,
196
+ commit deleted `removePages(page_number < merged_block_N)`, and restart resumed from
197
+ `getPage` = oldest remaining chunk at `nextInterval(own).from = chunk_end + 1`. During
198
+ catch-up chunks are full `stepSize`, so the committed watermark `N` sat **inside** the
199
+ oldest retained chunk `[start, end]` (`end ≥ N ⇒ not deleted`); resume jumped to `end + 1`,
200
+ **silently skipping `(N, end]`** → missing primitives → divergent app state, with no error.
201
+
202
+ **The fix (D) — block-accurate resume, runtime is the sole writer.** The runtime persists a
203
+ block-accurate marker per protocol on commit instead of the fetch loop persisting chunk-end
204
+ pages. Full mechanism in **design idea #5** above. Regression test: the
205
+ `consistency.test.ts` "mid-chunk restart" case (Postgres-only).
167
206
 
168
207
  ### 3. Serial replay is the catch-up bottleneck
169
208
 
@@ -182,6 +221,8 @@ block-hash chain.
182
221
  > survives a restart. See `docs/ADDING-A-SYNC-PROTOCOL.md` for how to add a chain
183
222
  > (with `test` as the worked example).
184
223
  >
185
- > Deterministic reproductions of #1/#2/#3 and their fixes land in **follow-up
186
- > PRs**, each test alongside its fix: #1 → fetch backpressure; #2 →
187
- > block-accurate resume; #3 → opt-in empty-block commit batching during catch-up.
224
+ > Deterministic reproductions of #1/#2/#3 and their fixes land alongside each fix:
225
+ > **#2 → block-accurate resume is DONE** (`consistency.test.ts`); **#1 → fetch
226
+ > backpressure is DONE** (`buffering.test.ts`, incl. 1c/1d for the merge-demand
227
+ > exemption). Still in follow-up PRs: #3 → opt-in empty-block commit batching during
228
+ > catch-up.
package/README.md CHANGED
@@ -46,6 +46,68 @@ const protocols = await genSyncProtocols(config);
46
46
  // protocols.parallelEvmRPC_fast.runOne() // poll one block
47
47
  ```
48
48
 
49
+ ## Backpressure (`maxBufferedPages`)
50
+
51
+ During deep catch-up a chain's fetch loop races to its tip far faster than the
52
+ merge can drain (the merge applies one block per DB transaction). Without a bound
53
+ the in-memory buffer (`SyncState.bufferedData`) grows toward the **entire backlog**
54
+ — hundreds of thousands of block objects — which is an OOM risk.
55
+
56
+ **The cap.** Every chain's `stateToInput` calls `bufferAtCap(state, syncProtocol)`
57
+ first (`sync-protocols/common/page-helpers.ts`): when
58
+ `bufferedData.size() >= maxBufferedPages` it returns `undefined`, so the chain stops
59
+ fetching — exactly as if it had caught up to the tip — and the polling loop sleeps
60
+ `pollingInterval` and retries. The merge keeps draining the buffer; once it drops
61
+ below the cap the next poll resumes. Peak in-memory buffering is therefore bounded
62
+ to **≈ `maxBufferedPages + stepSize`** per chain (one in-flight chunk can overshoot
63
+ the cap), instead of the whole backlog.
64
+
65
+ **Config.** `maxBufferedPages` is an optional field on every sync-protocol config
66
+ (declared once on the shared `PollingSyncProtocol` schema). When unset it defaults to
67
+ **`4 × stepSize`** (the `MAX_BUFFER_MULTIPLE` constant in `page-helpers.ts`), always
68
+ clamped to **`≥ stepSize + 1`** so a chain can always fetch at least one chunk to feed
69
+ the merge. ~4 chunks of look-ahead never starves the merge while keeping memory
70
+ bounded; raise it to trade memory for more fetch look-ahead. The cap only bites during
71
+ catch-up — in steady state the buffer sits near zero and the cap is never reached.
72
+
73
+ **Deadlock-safety (merge-demand exemption).** A naive cap can deadlock the merge: it
74
+ produces a root block at timestamp `τ` only once a parallel chain's page passes `τ`, and
75
+ drains that chain's buffer only *after* the page passes `τ`. If the cap pauses the
76
+ fetcher while its page is still `≤ τ` (the data up to `τ` exceeds the cap — e.g. a far
77
+ skip-ahead or a parallel chain finer-grained than the cap), neither side can proceed: a
78
+ circular wait, not a lost wakeup.
79
+
80
+ The safeguard is a **merge-demand exemption**: while the merge is blocked on a chain's
81
+ page (`mergeIntoRoot` sets `SyncState.mergeWaitingForPage`), `bufferAtCap` returns
82
+ `false` for that chain so the fetcher advances until its page crosses `τ`. Memory stays
83
+ **bounded by necessity** — only `(lastPage.root, τ]` is buffered above the cap, exactly
84
+ what the merge must hold to build block `τ`; once the page passes `τ` the normal cap
85
+ re-engages. A chain that is merely tip-limited (caught up, not gated by the merge) is
86
+ *not* exempted, so steady-state and head-of-line bounds are unaffected. Reproductions:
87
+ `buffering.test.ts` 1c (skip-ahead) and 1d (density).
88
+
89
+ **Scope.** The guard runs in **every** chain's `stateToInput`, so all sync chains are
90
+ covered: EVM, NTP, Bitcoin, Avail, Celestia, NEAR, Midnight, Cardano (UTXO-RPC), and
91
+ the synthetic `test` chain. Two notes:
92
+ - **Cardano (UTXO-RPC)** has no `stepSize` (it streams one block per pass), so the cap
93
+ falls back to a default chunk size of 1000 (⇒ default cap 4000); set `maxBufferedPages`
94
+ explicitly to tune it.
95
+ - The cap bounds `SyncState.bufferedData` (the merge-facing Deque the backpressure
96
+ feature protects). The UTXO-RPC fetcher additionally keeps its own internal FollowTip stream
97
+ buffer; pausing `stateToInput` stops draining it into `bufferedData`, but bounding
98
+ that lower-level stream is a separate, fetcher-specific concern.
99
+
100
+ **Observability.** Each `SyncState` tracks, and the runtime's `/debug/metrics`
101
+ endpoint reports per protocol: `cap` (resolved `maxBufferedPages`), `buf` (current
102
+ size), `bufHighWater` (peak since boot — catches spikes between samples), `pausedNow`,
103
+ `pauses` (rising-edge count — **the "backpressure engaged" signal**), `pausedMs`
104
+ (total time paused), and `mergeWaiting`/`mergeDemandRoot` (merge-demand exemption:
105
+ `mergeWaiting=true` means the merge is gated on this chain's page and the cap is lifted).
106
+ `pauses > 0` means the cap actively bounded memory; `0` means it
107
+ was never needed in that run. Steady-state these sit at `0`; during a real deep
108
+ catch-up (or under the perf harness's `PERF_APPLY_DELAY_MS` drain throttle) they climb
109
+ as `buf` pins to `cap`.
110
+
49
111
  ## Key exports
50
112
 
51
113
  - `genSyncProtocols(dbConn, syncInfo)` - Effection generator that instantiates a runtime fetcher + state pair for every protocol in `syncInfo` (from `config.syncProtocols`). Called from the runtime's process-blocks loop.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@effectstream/sync",
3
- "version": "0.100.22",
3
+ "version": "0.100.23",
4
4
  "license": "MIT",
5
5
  "homepage": "https://effectstream.github.io/docs/",
6
6
  "repository": {
@@ -13,10 +13,10 @@
13
13
  },
14
14
  "exports": "./src/mod.ts",
15
15
  "dependencies": {
16
- "@effectstream/config": "0.100.22",
17
- "@effectstream/db": "0.100.22",
18
- "@effectstream/log": "0.100.22",
19
- "@effectstream/utils": "0.100.22",
16
+ "@effectstream/config": "0.100.23",
17
+ "@effectstream/db": "0.100.23",
18
+ "@effectstream/log": "0.100.23",
19
+ "@effectstream/utils": "0.100.23",
20
20
  "@midnight-ntwrk/ledger-v8": "8.0.3",
21
21
  "@midnight-ntwrk/midnight-js-indexer-public-data-provider": "4.0.4",
22
22
  "@midnight-ntwrk/onchain-runtime": "npm:@midnight-ntwrk/onchain-runtime-v3@3.0.0",
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
10
10
  import { getPage } from "@effectstream/db";
11
11
  import { AvailClient } from "./AvailClient.ts";
12
12
  import { applyDelay } from "../common/utils.ts";
13
+ import { bufferAtCap } from "../common/page-helpers.ts";
13
14
 
14
15
  export class AvailSyncState extends SyncState<
15
16
  Input,
@@ -63,6 +64,7 @@ export class AvailSyncState extends SyncState<
63
64
 
64
65
  @bound
65
66
  override *stateToInput(): Operation<Input | undefined> {
67
+ if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
66
68
  const latestHeight = yield* call(() => this.client.getLatestBlockHeight());
67
69
  const startHeight = this.lastPage?.own.height ??
68
70
  this.config.syncProtocol.startBlockHeight - 1;
@@ -99,6 +101,15 @@ export class AvailSyncState extends SyncState<
99
101
  rootOutput.primitives.push(...primitives);
100
102
  }
101
103
 
104
+ @bound
105
+ override outputToLastPage(data: Output): LastPage<Page, RootPage> {
106
+ return {
107
+ own: { height: data.raw.number, hash: data.raw.hash },
108
+ ownBlockNumber: data.raw.number,
109
+ root: this.toRootPage(data),
110
+ };
111
+ }
112
+
102
113
  @bound
103
114
  override getNamespace(): string[] {
104
115
  return [this.config.network.name, this.config.syncProtocol.name];
@@ -1,4 +1,4 @@
1
- import { type Operation, until } from "effection";
1
+ import { type Operation } from "effection";
2
2
  import Deque from "denque";
3
3
  import {
4
4
  type BlockNumber,
@@ -11,7 +11,6 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
11
11
  import type { BaseDataFetcher, DataFetched } from "./fetcher.ts";
12
12
  import type { PageRelation } from "./page.ts";
13
13
  import type { PoolClient } from "pg";
14
- import { acquireDBMutex, releaseDBMutex, upsertPage } from "@effectstream/db";
15
14
 
16
15
  export type LastPage<Page, RootPage> = {
17
16
  own: Page;
@@ -75,6 +74,32 @@ export abstract class SyncState<
75
74
  /** Timestamp of the most recent error, or 0 if no error has occurred. */
76
75
  public lastErrorTimestamp: number = 0;
77
76
 
77
+ // ── Backpressure observability (see common/page-helpers.ts + README) ──
78
+ /** Resolved fetch cap (`maxBufferedPages`); set on each backpressure check, 0 until the first. */
79
+ public bufferCap: number = 0;
80
+ /** Peak `bufferedData.size()` seen since boot — catches spikes the periodic sampler misses. */
81
+ public bufferHighWater: number = 0;
82
+ /** Whether fetching is currently paused because the buffer is at/over the cap. */
83
+ public pausedNow: boolean = false;
84
+ /** How many times the cap engaged (rising edges) — the "backpressure fired" counter. */
85
+ public backpressurePauses: number = 0;
86
+ /** Total time fetching has been paused by the cap, in ms. */
87
+ public backpressurePausedMs: number = 0;
88
+ /** Start of the current pause, for accumulating `backpressurePausedMs`. */
89
+ private pauseStartMs: number = 0;
90
+
91
+ // ── Merge-demand exemption (see common/page-helpers.ts + README) ──
92
+ /**
93
+ * True while the merge is blocked waiting for THIS chain's page to pass the
94
+ * current root timestamp. The cap must not pause the fetcher here: the merge
95
+ * drains this chain's buffer only after its page passes the target, so pausing
96
+ * would be a circular wait. While exempt the fetcher buffers only the data the
97
+ * merge must hold to build that block anyway (bounded by necessity).
98
+ */
99
+ public mergeWaitingForPage: boolean = false;
100
+ /** Root page (ms timestamp) the merge is waiting to exceed — observability only. */
101
+ public mergeDemandRoot: RootPage | undefined = undefined;
102
+
78
103
  constructor(
79
104
  public readonly name: string,
80
105
  lastPage: undefined | LastPage<Page, RootPage>,
@@ -106,6 +131,24 @@ export abstract class SyncState<
106
131
  */
107
132
  abstract mergeDatum(ourOutput: Output, rootOutput: RootOutput): void;
108
133
 
134
+ recordBackpressure(atCap: boolean, cap: number): void {
135
+ this.bufferCap = cap;
136
+ const now = Date.now();
137
+ if (atCap && !this.pausedNow) {
138
+ this.pausedNow = true;
139
+ this.pauseStartMs = now;
140
+ this.backpressurePauses += 1;
141
+ } else if (!atCap && this.pausedNow) {
142
+ this.pausedNow = false;
143
+ this.backpressurePausedMs += now - this.pauseStartMs;
144
+ }
145
+ }
146
+ /**
147
+ * Build the resume marker for a single output: the {@link LastPage} to resume
148
+ * fetching *after* it. See CLAUDE.md design idea #5 (restart/resume).
149
+ */
150
+ abstract outputToLastPage(data: Output): LastPage<Page, RootPage>;
151
+
109
152
  @bound
110
153
  *updateState(
111
154
  _input: Input,
@@ -115,6 +158,8 @@ export abstract class SyncState<
115
158
  for (const datum of data.output) {
116
159
  this.bufferedData.push(datum);
117
160
  }
161
+ // Peak right after a push — the local max before the merge drains it.
162
+ this.bufferHighWater = Math.max(this.bufferHighWater, this.bufferedData.size());
118
163
  this.newDataCondVar.wake();
119
164
  }
120
165
 
@@ -129,13 +174,8 @@ export abstract class SyncState<
129
174
  ),
130
175
  );
131
176
 
132
- yield* acquireDBMutex(`update-state-${this.name}`);
133
- yield* until(upsertPage.run({
134
- protocol_name: this.name,
135
- page_number: data.lastPage.ownBlockNumber,
136
- page: JSON.stringify(data.lastPage),
137
- }, this.dbConn as any)); // Client,
138
- releaseDBMutex(`update-state-${this.name}`);
177
+ // lastPage is in-memory only; the runtime is the sole writer of the
178
+ // persisted resume marker (see CLAUDE.md design idea #5).
139
179
  this.lastPage = data.lastPage;
140
180
  this.newPageCondVar.wake();
141
181
  }
@@ -16,7 +16,7 @@ import {
16
16
  type Output,
17
17
  type Page,
18
18
  } from "./types.ts";
19
- import { genInputRange } from "../common/page-helpers.ts";
19
+ import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
20
20
  import type {
21
21
  ConfigNetworkType,
22
22
  SyncProtocolWithNetwork,
@@ -58,6 +58,7 @@ export class BitcoinSyncState extends SyncState<
58
58
  block_number: Number(data.raw.height),
59
59
  blockHash: hash,
60
60
  })),
61
+ resumePages: [],
61
62
  primitives: data.primitives.map((primitive) => ({
62
63
  ...primitive,
63
64
  source: this.config.syncProtocol.name,
@@ -75,6 +76,7 @@ export class BitcoinSyncState extends SyncState<
75
76
 
76
77
  @bound
77
78
  override *stateToInput(): Operation<Input | undefined> {
79
+ if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
78
80
  return yield* genInputRange(
79
81
  this as BitcoinSyncState,
80
82
  this.config.syncProtocol.startBlockHeight as Page,
@@ -112,6 +114,16 @@ export class BitcoinSyncState extends SyncState<
112
114
  rootOutput.blockInfo.push(...blockInfo);
113
115
  }
114
116
 
117
+ @bound
118
+ override outputToLastPage(data: Output): LastPage<Page, RootPage> {
119
+ const block = Number(data.raw.height);
120
+ return {
121
+ own: block,
122
+ ownBlockNumber: block,
123
+ root: this.toRootPage(data),
124
+ };
125
+ }
126
+
115
127
  @bound
116
128
  override getNamespace(): string[] {
117
129
  return [this.config.network.name, this.config.syncProtocol.name];
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
10
10
  import { getPage } from "@effectstream/db";
11
11
  import { CelestiaClient } from "./CelestiaClient.ts";
12
12
  import { applyDelay } from "../common/utils.ts";
13
+ import { bufferAtCap } from "../common/page-helpers.ts";
13
14
 
14
15
  export class CelestiaSyncState extends SyncState<
15
16
  Input,
@@ -61,6 +62,7 @@ export class CelestiaSyncState extends SyncState<
61
62
 
62
63
  @bound
63
64
  override *stateToInput(): Operation<Input | undefined> {
65
+ if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
64
66
  const latestHeight = yield* call(() =>
65
67
  this.client.getLatestBlockHeight()
66
68
  );
@@ -99,6 +101,15 @@ export class CelestiaSyncState extends SyncState<
99
101
  rootOutput.primitives.push(...primitives);
100
102
  }
101
103
 
104
+ @bound
105
+ override outputToLastPage(data: Output): LastPage<Page, RootPage> {
106
+ return {
107
+ own: { height: data.height, hash: data.hash },
108
+ ownBlockNumber: data.height,
109
+ root: this.toRootPage(data),
110
+ };
111
+ }
112
+
102
113
  @bound
103
114
  override getNamespace(): string[] {
104
115
  return [this.config.network.name, this.config.syncProtocol.name];
@@ -5,6 +5,47 @@ import { ComponentNames, log, SeverityNumber } from "@effectstream/log";
5
5
  import type { PaginatedSyncProtocols } from "../types.ts";
6
6
  import type { PageTypeOf } from "../base/state.ts";
7
7
 
8
+ // Fetch backpressure. See README "Backpressure (`maxBufferedPages`)".
9
+ const MAX_BUFFER_MULTIPLE = 4;
10
+ // Fallback chunk size for chains that fetch without an explicit stepSize
11
+ // (e.g. utxorpc streams one block per pass).
12
+ const DEFAULT_STEP_SIZE = 1000;
13
+
14
+ /** Resolved fetch-backpressure cap for a chain. See README "Backpressure". */
15
+ export function bufferCapFor(
16
+ syncProtocol: { maxBufferedPages?: number; stepSize?: number },
17
+ ): number {
18
+ const step = syncProtocol.stepSize ?? DEFAULT_STEP_SIZE;
19
+ return Math.max(
20
+ syncProtocol.maxBufferedPages ?? MAX_BUFFER_MULTIPLE * step,
21
+ step + 1,
22
+ );
23
+ }
24
+
25
+ /**
26
+ * Whether a chain should pause fetching because its in-memory buffer is at/over
27
+ * the cap. Each chain's `stateToInput` calls this first and returns `undefined`
28
+ * when true, so the fetch loop sleeps + retries while the merge drains. Also
29
+ * records backpressure metrics on `state` (for `/debug/metrics`).
30
+ * See README "Backpressure (`maxBufferedPages`)".
31
+ */
32
+ export function bufferAtCap(
33
+ state: {
34
+ bufferedData: { size(): number };
35
+ recordBackpressure(atCap: boolean, cap: number): void;
36
+ mergeWaitingForPage: boolean;
37
+ },
38
+ syncProtocol: { maxBufferedPages?: number; stepSize?: number },
39
+ ): boolean {
40
+ const cap = bufferCapFor(syncProtocol);
41
+ // Exempt this chain from the cap while the merge is blocked on its page (pausing
42
+ // there would deadlock — see SyncState.mergeWaitingForPage). Reporting
43
+ // atCap=false while exempt is correct: the chain is genuinely not paused.
44
+ const atCap = !state.mergeWaitingForPage && state.bufferedData.size() >= cap;
45
+ state.recordBackpressure(atCap, cap);
46
+ return atCap;
47
+ }
48
+
8
49
  type ConfigSubset<Page> = {
9
50
  name: string;
10
51
  /**
@@ -5,6 +5,7 @@ import type {
5
5
  TimestampMs,
6
6
  } from "@effectstream/utils";
7
7
  import type { PageRelation } from "../base/page.ts";
8
+ import type { LastPage } from "../base/state.ts";
8
9
  import type {
9
10
  ConfigSyncProtocolType,
10
11
  FlattenSyncProtocolIOFor,
@@ -19,6 +20,14 @@ export type ChainBlock = {
19
20
  block_number: BlockNumber;
20
21
  blockHash: BlockHash;
21
22
  }[];
23
+ /**
24
+ * Per-protocol block-accurate resume markers the runtime persists on commit.
25
+ * See `@effectstream/sync` CLAUDE.md, design idea #5 (restart/resume).
26
+ */
27
+ resumePages: {
28
+ protocol_name: string;
29
+ lastPage: LastPage<unknown, unknown>;
30
+ }[];
22
31
  primitives: (
23
32
  & FlattenSyncProtocolIOFor<
24
33
  ConfigSyncProtocolType
@@ -7,7 +7,7 @@ import { bound, type EvmBlockNumber } from "@effectstream/utils";
7
7
  import { type LastPage, SyncState } from "../base/state.ts";
8
8
  import type { RootOutput, RootPage } from "../types.ts";
9
9
  import type { EvmFetcher } from "./fetcher.ts";
10
- import { genInputRange } from "../common/page-helpers.ts";
10
+ import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
11
11
  import type { Input, Output, Page } from "./types.ts";
12
12
  import { toMsTimestamp } from "./types.ts";
13
13
  import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
@@ -53,6 +53,7 @@ export class EvmSyncState extends SyncState<
53
53
 
54
54
  @bound
55
55
  override *stateToInput(): Operation<Input | undefined> {
56
+ if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
56
57
  return yield* genInputRange(
57
58
  this as EvmSyncState,
58
59
  this.config.syncProtocol.startBlockHeight as EvmBlockNumber,
@@ -90,6 +91,16 @@ export class EvmSyncState extends SyncState<
90
91
  rootOutput.blockInfo.push(...blockInfo);
91
92
  }
92
93
 
94
+ @bound
95
+ override outputToLastPage(data: Output): LastPage<Page, RootPage> {
96
+ const block = Number(data.raw.number);
97
+ return {
98
+ own: block,
99
+ ownBlockNumber: block,
100
+ root: this.toRootPage(data),
101
+ };
102
+ }
103
+
93
104
  @bound
94
105
  override getNamespace(): string[] {
95
106
  return [this.config.network.name, this.config.syncProtocol.name];
@@ -40,6 +40,14 @@ export interface MidnightGqlBlockState {
40
40
  // Present only on RegularTransaction (selected via inline fragment): the
41
41
  // coin-commitment Merkle tree root after this transaction.
42
42
  zswapMerkleTreeRoot?: string;
43
+ // Selected only for MidnightTokenMintPrimitive: the serialized
44
+ // transaction bytes (Transaction interface) and the apply result
45
+ // (RegularTransaction only — absent on system transactions).
46
+ raw?: string;
47
+ transactionResult?: {
48
+ status: "SUCCESS" | "PARTIAL_SUCCESS" | "FAILURE";
49
+ segments?: { id: number; success: boolean }[] | null;
50
+ };
43
51
  }[];
44
52
  };
45
53
  };
@@ -55,6 +63,8 @@ export interface BlockFetchOptions {
55
63
  unshieldedCreatedOutputs?: boolean;
56
64
  /** Include `... on RegularTransaction { zswapMerkleTreeRoot }` (needed for MidnightZswapRootPrimitive). Default: false */
57
65
  zswapRoots?: boolean;
66
+ /** Include tx `raw` + `... on RegularTransaction { transactionResult }` (needed for MidnightTokenMintPrimitive). Default: false */
67
+ tokenMints?: boolean;
58
68
  }
59
69
 
60
70
  type PublicDataProvider = ReturnType<typeof indexerPublicDataProvider>;
@@ -182,6 +192,7 @@ export class MidnightClient {
182
192
  unshieldedSpentOutputs = false,
183
193
  unshieldedCreatedOutputs = false,
184
194
  zswapRoots = false,
195
+ tokenMints = false,
185
196
  } = options;
186
197
  const contractActionsField = contractActions
187
198
  ? `contractActions { address state }`
@@ -200,6 +211,14 @@ export class MidnightClient {
200
211
  const zswapRootFragment = zswapRoots
201
212
  ? `... on RegularTransaction { zswapMerkleTreeRoot }`
202
213
  : "";
214
+ // `raw` is on the Transaction interface; `transactionResult` is
215
+ // RegularTransaction-only (system transactions have neither result nor
216
+ // contract calls, so they are skipped downstream). Duplicate inline
217
+ // fragments merge legally in GraphQL.
218
+ const tokenMintFields = tokenMints
219
+ ? `raw
220
+ ... on RegularTransaction { transactionResult { status segments { id success } } }`
221
+ : "";
203
222
  const query = `query {
204
223
  block(offset: { height: ${blockHeight} }) {
205
224
  hash
@@ -216,6 +235,7 @@ export class MidnightClient {
216
235
  ${unshieldedSpentField}
217
236
  ${unshieldedCreatedField}
218
237
  ${zswapRootFragment}
238
+ ${tokenMintFields}
219
239
  }
220
240
  }
221
241
  }`;
@@ -19,6 +19,7 @@ import { bound } from "@effectstream/utils";
19
19
  import { MidnightClient, type BlockFetchOptions, type MidnightGqlBlockState } from "./MidnightClient.ts";
20
20
  import { ContractState, StateValue } from "@midnight-ntwrk/onchain-runtime";
21
21
  import { decodeZswapInputEvent } from "./zswap-decoder.ts";
22
+ import { decodeTokenMints } from "./mint-decoder.ts";
22
23
 
23
24
  export class MidnightFetcher extends BaseDataFetcher<
24
25
  Input,
@@ -86,7 +87,8 @@ export class MidnightFetcher extends BaseDataFetcher<
86
87
  p.primitive.type !== "Midnight:Nullifier" &&
87
88
  p.primitive.type !== "Midnight:UnshieldedSpend" &&
88
89
  p.primitive.type !== "Midnight:UnshieldedCreate" &&
89
- p.primitive.type !== "Midnight:ZswapRoot",
90
+ p.primitive.type !== "Midnight:ZswapRoot" &&
91
+ p.primitive.type !== "Midnight:TokenMint",
90
92
  ),
91
93
  zswapLedgerEvents: this.config.primitives.some(
92
94
  (p) => p.primitive.type === "Midnight:Nullifier",
@@ -100,6 +102,9 @@ export class MidnightFetcher extends BaseDataFetcher<
100
102
  zswapRoots: this.config.primitives.some(
101
103
  (p) => p.primitive.type === "Midnight:ZswapRoot",
102
104
  ),
105
+ tokenMints: this.config.primitives.some(
106
+ (p) => p.primitive.type === "Midnight:TokenMint",
107
+ ),
103
108
  };
104
109
  const self = this;
105
110
  const heights = Array.from(
@@ -192,6 +197,8 @@ export class MidnightFetcher extends BaseDataFetcher<
192
197
  syncResults.push(...this.fetchUnshieldedCreates(height, primitiveEntry, block));
193
198
  } else if (primitiveEntry.primitive.type === "Midnight:ZswapRoot") {
194
199
  syncResults.push(...this.fetchZswapRoots(height, primitiveEntry, block));
200
+ } else if (primitiveEntry.primitive.type === "Midnight:TokenMint") {
201
+ syncResults.push(...this.fetchTokenMints(height, primitiveEntry, block));
195
202
  } else {
196
203
  asyncOps.push(
197
204
  this.fetchContractState(height, client, primitiveEntry, block),
@@ -339,6 +346,42 @@ export class MidnightFetcher extends BaseDataFetcher<
339
346
  ];
340
347
  }
341
348
 
349
+ // Decode custom token mints out of each regular transaction's raw bytes
350
+ // (ledger-v8 deserialize → contract call transcript effects). One input per
351
+ // (tx, call, domainSep, kind) that actually applied on chain — see
352
+ // mint-decoder.ts for the segment semantics. System transactions carry no
353
+ // transactionResult and are skipped.
354
+ @bound
355
+ fetchTokenMints(
356
+ height: number,
357
+ primitiveEntry: PrimitiveEntryType,
358
+ block: MidnightGqlBlockState,
359
+ ): PrimitiveType[] {
360
+ const results: PrimitiveType[] = [];
361
+ for (const tx of block.block.transactions) {
362
+ if (!tx.raw || !tx.transactionResult) continue;
363
+ for (const mint of decodeTokenMints(tx.raw, tx.transactionResult)) {
364
+ results.push({
365
+ syncProtocol: {
366
+ name: primitiveEntry.syncProtocol,
367
+ blockNumber: height,
368
+ transactionHash: tx.hash,
369
+ contractAddress: mint.contractAddress,
370
+ },
371
+ primitive: primitiveEntry.primitive.name,
372
+ output: {
373
+ payloadType: "midnight-token-mint",
374
+ payload: {
375
+ ...mint,
376
+ txHash: tx.hash,
377
+ },
378
+ },
379
+ });
380
+ }
381
+ }
382
+ return results;
383
+ }
384
+
342
385
  @bound
343
386
  *fetchContractState(
344
387
  height: number,
@@ -0,0 +1,151 @@
1
+ import { Buffer } from "node:buffer";
2
+ import * as ledger from "@midnight-ntwrk/ledger-v8";
3
+
4
+ // Decodes custom token mints out of a raw Midnight transaction. A contract
5
+ // call's transcript effects record mints as `domain_sep → amount` maps
6
+ // (`shieldedMints` / `unshieldedMints`); the resulting token type ("color")
7
+ // is `rawTokenType(domain_sep, contract_address)`. Neither the indexer's
8
+ // GraphQL nor the wallet expose this mapping, so the only route is
9
+ // deserializing the transaction bytes with ledger-v8 — same approach as
10
+ // zswap-decoder.ts for nullifier events.
11
+ //
12
+ // Mirrors zswap-decoder.ts tolerance: never throws — an undecodable
13
+ // transaction logs a warning and contributes nothing.
14
+
15
+ export interface TokenMintRecord {
16
+ /** Address of the minting contract, hex */
17
+ contractAddress: string;
18
+ /** 32-byte domain separator pre-image, hex */
19
+ domainSep: string;
20
+ /** Derived token type ("color") = rawTokenType(domainSep, contractAddress), hex */
21
+ rawTokenType: string;
22
+ kind: "shielded" | "unshielded";
23
+ /** Total minted for this (call, domainSep, kind), as a decimal string (u64 exceeds MAX_SAFE_INTEGER) */
24
+ amount: string;
25
+ /** Contract entry point that minted, when printable */
26
+ entryPoint?: string;
27
+ }
28
+
29
+ export interface TxResultLike {
30
+ status: "SUCCESS" | "PARTIAL_SUCCESS" | "FAILURE";
31
+ segments?: { id: number; success: boolean }[] | null;
32
+ }
33
+
34
+ function hexToBytes(hex: string): Uint8Array {
35
+ const clean = hex.startsWith("0x") ? hex.slice(2) : hex;
36
+ return Uint8Array.from(Buffer.from(clean, "hex"));
37
+ }
38
+
39
+ function bytesToHex(bytes: Uint8Array): string {
40
+ return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
41
+ }
42
+
43
+ function normalizeHex(value: string): string {
44
+ return value.replace(/^0x/, "").toLowerCase();
45
+ }
46
+
47
+ /**
48
+ * Extract every applied token mint from a raw serialized transaction.
49
+ *
50
+ * Segment semantics (failed segments roll back on chain):
51
+ * - no `result` (system transactions) or status FAILURE → nothing applied;
52
+ * - SUCCESS → guaranteed + all fallible transcripts applied;
53
+ * - PARTIAL_SUCCESS → guaranteed transcripts applied, fallible only where
54
+ * the intent's segment (the `intents` map key) succeeded.
55
+ */
56
+ export function decodeTokenMints(
57
+ rawHex: string,
58
+ result?: TxResultLike | null,
59
+ ): TokenMintRecord[] {
60
+ if (!result || result.status === "FAILURE") return [];
61
+
62
+ let tx: any;
63
+ try {
64
+ tx = ledger.Transaction.deserialize(
65
+ "signature",
66
+ "proof",
67
+ "binding",
68
+ hexToBytes(rawHex),
69
+ );
70
+ } catch (e) {
71
+ console.warn(
72
+ `[mint-decoder] undecodable transaction, skipping: ${
73
+ (e as Error)?.message ?? e
74
+ }`,
75
+ );
76
+ return [];
77
+ }
78
+
79
+ const records: TokenMintRecord[] = [];
80
+ for (const [segmentId, intent] of tx.intents ?? new Map()) {
81
+ const fallibleApplied = result.status === "SUCCESS" ||
82
+ (result.segments?.some((s) => s.id === Number(segmentId) && s.success) ??
83
+ false);
84
+ for (const action of intent.actions ?? []) {
85
+ // Only contract calls carry transcripts; deploys and maintenance
86
+ // updates cannot mint.
87
+ if (!(action instanceof ledger.ContractCall)) continue;
88
+ try {
89
+ records.push(...mintsOfCall(action, fallibleApplied));
90
+ } catch (e) {
91
+ console.warn(
92
+ `[mint-decoder] failed to read mints of call to ${
93
+ (action as any)?.address ?? "?"
94
+ }: ${(e as Error)?.message ?? e}`,
95
+ );
96
+ }
97
+ }
98
+ }
99
+ return records;
100
+ }
101
+
102
+ function mintsOfCall(
103
+ call: any,
104
+ fallibleApplied: boolean,
105
+ ): TokenMintRecord[] {
106
+ // Accumulate domainSep|kind → amount across the applied transcripts. The
107
+ // guaranteed transcript applied whenever the transaction wasn't a FAILURE
108
+ // (the caller filters those); the fallible transcript only if its segment
109
+ // succeeded.
110
+ const totals = new Map<string, bigint>();
111
+ const transcripts = [
112
+ call.guaranteedTranscript,
113
+ ...(fallibleApplied ? [call.fallibleTranscript] : []),
114
+ ];
115
+ for (const transcript of transcripts) {
116
+ const effects = transcript?.effects;
117
+ if (!effects) continue;
118
+ for (
119
+ const [kind, mints] of [
120
+ ["shielded", effects.shieldedMints],
121
+ ["unshielded", effects.unshieldedMints],
122
+ ] as const
123
+ ) {
124
+ for (const [sepHex, amount] of mints ?? new Map()) {
125
+ const key = `${normalizeHex(String(sepHex))}|${kind}`;
126
+ totals.set(key, (totals.get(key) ?? 0n) + BigInt(amount));
127
+ }
128
+ }
129
+ }
130
+
131
+ const records: TokenMintRecord[] = [];
132
+ for (const [key, amount] of totals) {
133
+ const [domainSep, kind] = key.split("|") as [
134
+ string,
135
+ "shielded" | "unshielded",
136
+ ];
137
+ records.push({
138
+ contractAddress: normalizeHex(String(call.address)),
139
+ domainSep,
140
+ rawTokenType: normalizeHex(
141
+ String(ledger.rawTokenType(hexToBytes(domainSep), call.address)),
142
+ ),
143
+ kind,
144
+ amount: amount.toString(),
145
+ entryPoint: typeof call.entryPoint === "string"
146
+ ? call.entryPoint
147
+ : bytesToHex(call.entryPoint),
148
+ });
149
+ }
150
+ return records;
151
+ }
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
10
10
  import { getPage } from "@effectstream/db";
11
11
  import { MidnightClient } from "./MidnightClient.ts";
12
12
  import { applyDelay } from "../common/utils.ts";
13
+ import { bufferAtCap } from "../common/page-helpers.ts";
13
14
 
14
15
  type LatestBlock = {
15
16
  block: {
@@ -72,6 +73,7 @@ export class MidnightSyncState extends SyncState<
72
73
 
73
74
  @bound
74
75
  override *stateToInput(): Operation<Input | undefined> {
76
+ if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
75
77
  const latestBlockResult = yield* call(() => this.client.fetchLatestBlock());
76
78
  const latestHeight = latestBlockResult.block.height;
77
79
 
@@ -110,6 +112,15 @@ export class MidnightSyncState extends SyncState<
110
112
  rootOutput.primitives.push(...primitives);
111
113
  }
112
114
 
115
+ @bound
116
+ override outputToLastPage(data: Output): LastPage<Page, RootPage> {
117
+ return {
118
+ own: { height: data.raw.height, hash: data.raw.hash },
119
+ ownBlockNumber: data.raw.height,
120
+ root: this.toRootPage(data),
121
+ };
122
+ }
123
+
113
124
  @bound
114
125
  override getNamespace(): string[] {
115
126
  return [this.config.network.name, this.config.syncProtocol.name];
@@ -10,6 +10,7 @@ import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/c
10
10
  import { getPage } from "@effectstream/db";
11
11
  import { NearClient } from "./NearClient.ts";
12
12
  import { applyDelay } from "../common/utils.ts";
13
+ import { bufferAtCap } from "../common/page-helpers.ts";
13
14
 
14
15
  export class NearSyncState extends SyncState<
15
16
  Input,
@@ -60,6 +61,7 @@ export class NearSyncState extends SyncState<
60
61
 
61
62
  @bound
62
63
  override *stateToInput(): Operation<Input | undefined> {
64
+ if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
63
65
  const latestHeight = yield* call(() =>
64
66
  this.client.getLatestBlockHeight()
65
67
  );
@@ -98,6 +100,15 @@ export class NearSyncState extends SyncState<
98
100
  rootOutput.primitives.push(...primitives);
99
101
  }
100
102
 
103
+ @bound
104
+ override outputToLastPage(data: Output): LastPage<Page, RootPage> {
105
+ return {
106
+ own: { height: data.height, hash: data.hash },
107
+ ownBlockNumber: data.height,
108
+ root: this.toRootPage(data),
109
+ };
110
+ }
111
+
101
112
  @bound
102
113
  override getNamespace(): string[] {
103
114
  return [this.config.network.name, this.config.syncProtocol.name];
@@ -7,7 +7,7 @@ import { bound, type NtpBlockNumber } from "@effectstream/utils";
7
7
  import { type LastPage, SyncState } from "../base/state.ts";
8
8
  import type { RootOutput, RootPage } from "../types.ts";
9
9
  import type { NtpFetcher } from "./fetcher.ts";
10
- import { genInputRange } from "../common/page-helpers.ts";
10
+ import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
11
11
  import type { Input, Output, Page } from "./types.ts";
12
12
  import { toMsTimestamp } from "./types.ts";
13
13
  import type { ConfigNetworkType, SyncProtocolWithNetwork } from "@effectstream/config";
@@ -48,6 +48,7 @@ export class NtpSyncState extends SyncState<
48
48
  })),
49
49
  blockNumber: Number(data.raw.blockNumber),
50
50
  timestamp: this.toRootPage(data),
51
+ resumePages: [],
51
52
  primitives: [],
52
53
  };
53
54
  }
@@ -59,6 +60,7 @@ export class NtpSyncState extends SyncState<
59
60
 
60
61
  @bound
61
62
  override *stateToInput(): Operation<Input | undefined> {
63
+ if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
62
64
  return yield* genInputRange(
63
65
  this as NtpSyncState,
64
66
  1, // NTP Pages start from 1, and the fetcher offsets using the start page (date)
@@ -86,6 +88,16 @@ export class NtpSyncState extends SyncState<
86
88
  throw new Error("Only parallel chains merge into root");
87
89
  }
88
90
 
91
+ @bound
92
+ override outputToLastPage(data: Output): LastPage<Page, RootPage> {
93
+ const block = Number(data.raw.blockNumber);
94
+ return {
95
+ own: block,
96
+ ownBlockNumber: block,
97
+ root: this.toRootPage(data),
98
+ };
99
+ }
100
+
89
101
  @bound
90
102
  override getNamespace(): string[] {
91
103
  return [this.config.network.name, this.config.syncProtocol.name];
@@ -124,6 +124,11 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
124
124
  }
125
125
  const output = iState.bufferedData.peekAt(0)!;
126
126
  const newRoot = iState.toRootOutput(output.output);
127
+ // Resume marker: the single datum this root block is built from (CLAUDE.md #5).
128
+ newRoot.resumePages.push({
129
+ protocol_name: state.name,
130
+ lastPage: iState.outputToLastPage(output.output),
131
+ });
127
132
  return {
128
133
  updateCache: () => {
129
134
  output.cleanup();
@@ -141,6 +146,11 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
141
146
  rootInfo.toPage(rootInfo.value),
142
147
  ) <= 0
143
148
  ) {
149
+ // Publish the demand before waiting so the cap exempts this chain while the
150
+ // merge is gated on its page (see SyncState.mergeWaitingForPage). No lock
151
+ // needed: the fetch loop re-polls stateToInput every pollingInterval.
152
+ state.mergeWaitingForPage = true;
153
+ state.mergeDemandRoot = rootInfo.toPage(rootInfo.value);
144
154
  log.remote(
145
155
  ComponentNames.EFFECTSTREAM_SYNC,
146
156
  state.getNamespace(),
@@ -155,6 +165,8 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
155
165
  (log) => log("Merge unblocked (missing parallel data): got data"),
156
166
  );
157
167
  }
168
+ state.mergeWaitingForPage = false;
169
+ state.mergeDemandRoot = undefined;
158
170
  const cleanups = getFromBuffer(
159
171
  {
160
172
  data: iState.bufferedData,
@@ -168,6 +180,17 @@ export function* mergeIntoRoot<SyncProtocol extends AllSyncProtocols>(
168
180
  iState.mergeDatum,
169
181
  );
170
182
 
183
+ // Resume marker: the last (highest) datum merged this iteration, still in the
184
+ // Deque at `cleanups.length - 1` (cleanup runs after send). Nothing consumed →
185
+ // no marker. See CLAUDE.md #5.
186
+ if (cleanups.length > 0) {
187
+ const lastConsumed = iState.bufferedData.peekAt(cleanups.length - 1)!;
188
+ rootInfo.value.resumePages.push({
189
+ protocol_name: state.name,
190
+ lastPage: iState.outputToLastPage(lastConsumed.output),
191
+ });
192
+ }
193
+
171
194
  return {
172
195
  updateCache: () => {
173
196
  state.bufferedData.remove(0, cleanups.length);
@@ -7,7 +7,7 @@ import { bound, type BlockNumber } from "@effectstream/utils";
7
7
  import { type LastPage, SyncState } from "../base/state.ts";
8
8
  import type { RootOutput, RootPage } from "../types.ts";
9
9
  import type { TestFetcher } from "./fetcher.ts";
10
- import { genInputRange } from "../common/page-helpers.ts";
10
+ import { bufferAtCap, genInputRange } from "../common/page-helpers.ts";
11
11
  import type { Input, Output, Page } from "./types.ts";
12
12
  import { toMsTimestamp } from "./types.ts";
13
13
  import type {
@@ -60,6 +60,7 @@ export class TestSyncState extends SyncState<
60
60
  })),
61
61
  blockNumber: Number(data.raw.blockNumber),
62
62
  timestamp: this.toRootPage(data),
63
+ resumePages: [],
63
64
  primitives: data.primitives.map((p) => ({
64
65
  ...p,
65
66
  source: this.config.syncProtocol.name,
@@ -77,6 +78,7 @@ export class TestSyncState extends SyncState<
77
78
 
78
79
  @bound
79
80
  override *stateToInput(): Operation<Input | undefined> {
81
+ if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
80
82
  return yield* genInputRange(
81
83
  this as TestSyncState,
82
84
  this.config.syncProtocol.startBlockHeight as BlockNumber,
@@ -114,6 +116,16 @@ export class TestSyncState extends SyncState<
114
116
  rootOutput.blockInfo.push(...blockInfo);
115
117
  }
116
118
 
119
+ @bound
120
+ override outputToLastPage(data: Output): LastPage<Page, RootPage> {
121
+ const block = Number(data.raw.blockNumber);
122
+ return {
123
+ own: block,
124
+ ownBlockNumber: block as BlockNumber,
125
+ root: this.toRootPage(data),
126
+ };
127
+ }
128
+
117
129
  @bound
118
130
  override getNamespace(): string[] {
119
131
  return [this.config.network.name, this.config.syncProtocol.name];
@@ -1,4 +1,5 @@
1
1
  import { applyDelay } from "../common/utils.ts";
2
+ import { bufferAtCap } from "../common/page-helpers.ts";
2
3
  import type { Operation } from "effection";
3
4
  import { call } from "effection";
4
5
  import { getPage } from "@effectstream/db";
@@ -71,6 +72,7 @@ export class UtxoRpcSyncState extends SyncState<
71
72
 
72
73
  @bound
73
74
  override *stateToInput(): Operation<Input | undefined> {
75
+ if (bufferAtCap(this, this.config.syncProtocol)) return undefined;
74
76
  // Get the value of the latest block height.
75
77
  const tipHeight = yield* call(() => this.fetcher.lastHeight());
76
78
  if (!tipHeight) {
@@ -157,6 +159,22 @@ export class UtxoRpcSyncState extends SyncState<
157
159
  rootOutput.blockInfo.push(...blockInfo);
158
160
  }
159
161
 
162
+ @bound
163
+ override outputToLastPage(data: Output): LastPage<Page, RootPage> {
164
+ // Mirror the fetcher's `own`: slot/height from the block header, and the
165
+ // block hash from `blockHashes` (the fetcher builds it as `[completed.hash]`,
166
+ // the same value it stored in `own.hash`).
167
+ return {
168
+ own: {
169
+ slot: Number(data.raw.header!.slot),
170
+ height: Number(data.raw.header!.height),
171
+ hash: data.blockHashes[0],
172
+ },
173
+ ownBlockNumber: Number(data.raw.header!.height),
174
+ root: this.toRootPage(data),
175
+ };
176
+ }
177
+
160
178
  @bound
161
179
  override getNamespace(): string[] {
162
180
  return [this.config.network.name, this.config.syncProtocol.name];
@@ -0,0 +1,28 @@
1
+ import { describe, expect, test } from "bun:test";
2
+
3
+ import { decodeTokenMints } from "../src/sync-protocols/midnight/mint-decoder.ts";
4
+
5
+ // Tolerance contract: the decoder must never throw — an undecodable or
6
+ // inapplicable transaction contributes nothing (the fetcher's per-height
7
+ // error path would otherwise wedge the sync loop).
8
+ describe("decodeTokenMints — tolerance", () => {
9
+ test("no transactionResult (system tx) → []", () => {
10
+ expect(decodeTokenMints("deadbeef", undefined)).toEqual([]);
11
+ expect(decodeTokenMints("deadbeef", null)).toEqual([]);
12
+ });
13
+
14
+ test("status FAILURE → [] (everything rolled back, incl. guaranteed)", () => {
15
+ expect(
16
+ decodeTokenMints("deadbeef", { status: "FAILURE", segments: [] }),
17
+ ).toEqual([]);
18
+ });
19
+
20
+ test("garbage bytes with SUCCESS status → [] without throwing", () => {
21
+ expect(
22
+ decodeTokenMints("not-even-hex", { status: "SUCCESS" }),
23
+ ).toEqual([]);
24
+ expect(
25
+ decodeTokenMints("00".repeat(64), { status: "SUCCESS" }),
26
+ ).toEqual([]);
27
+ });
28
+ });