envio 3.3.0-alpha.1 → 3.3.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. package/evm.schema.json +10 -0
  2. package/index.d.ts +45 -25
  3. package/package.json +7 -7
  4. package/src/Address.res +5 -2
  5. package/src/Address.res.mjs +3 -1
  6. package/src/Api.res +1 -1
  7. package/src/Api.res.mjs +1 -1
  8. package/src/Batch.res +0 -6
  9. package/src/Batch.res.mjs +1 -12
  10. package/src/BatchProcessing.res +7 -12
  11. package/src/BatchProcessing.res.mjs +8 -7
  12. package/src/Bin.res +3 -0
  13. package/src/Bin.res.mjs +4 -0
  14. package/src/ChainFetching.res +43 -9
  15. package/src/ChainFetching.res.mjs +34 -15
  16. package/src/ChainMetadata.res +1 -11
  17. package/src/ChainMetadata.res.mjs +1 -10
  18. package/src/ChainState.res +605 -151
  19. package/src/ChainState.res.mjs +427 -104
  20. package/src/ChainState.resi +72 -4
  21. package/src/Config.res +61 -33
  22. package/src/Config.res.mjs +50 -10
  23. package/src/ContractRegisterContext.res +2 -12
  24. package/src/ContractRegisterContext.res.mjs +3 -5
  25. package/src/Core.res +14 -3
  26. package/src/CrossChainState.res +113 -89
  27. package/src/CrossChainState.res.mjs +64 -60
  28. package/src/CrossChainState.resi +1 -1
  29. package/src/Ecosystem.res +5 -6
  30. package/src/Ecosystem.res.mjs +3 -3
  31. package/src/Envio.res +17 -11
  32. package/src/EnvioGlobal.res +53 -0
  33. package/src/EnvioGlobal.res.mjs +31 -0
  34. package/src/EventConfigBuilder.res +177 -88
  35. package/src/EventConfigBuilder.res.mjs +99 -42
  36. package/src/EventProcessing.res +74 -28
  37. package/src/EventProcessing.res.mjs +55 -29
  38. package/src/ExitOnCaughtUp.res +10 -2
  39. package/src/ExitOnCaughtUp.res.mjs +10 -1
  40. package/src/FetchState.res +544 -356
  41. package/src/FetchState.res.mjs +412 -471
  42. package/src/HandlerLoader.res +8 -104
  43. package/src/HandlerLoader.res.mjs +2 -81
  44. package/src/HandlerRegister.res +518 -144
  45. package/src/HandlerRegister.res.mjs +285 -133
  46. package/src/HandlerRegister.resi +24 -8
  47. package/src/IndexerState.res +10 -3
  48. package/src/IndexerState.res.mjs +11 -4
  49. package/src/IndexerState.resi +2 -1
  50. package/src/IndexingAddresses.res +108 -0
  51. package/src/IndexingAddresses.res.mjs +101 -0
  52. package/src/IndexingAddresses.resi +34 -0
  53. package/src/Internal.res +155 -46
  54. package/src/Internal.res.mjs +30 -2
  55. package/src/LoadLayer.res +5 -5
  56. package/src/LoadLayer.res.mjs +6 -6
  57. package/src/LogSelection.res +102 -165
  58. package/src/LogSelection.res.mjs +101 -116
  59. package/src/Main.res +71 -236
  60. package/src/Main.res.mjs +60 -145
  61. package/src/Metrics.res +74 -0
  62. package/src/Metrics.res.mjs +75 -0
  63. package/src/PgStorage.res +4 -4
  64. package/src/PgStorage.res.mjs +5 -5
  65. package/src/Prometheus.res +10 -63
  66. package/src/Prometheus.res.mjs +124 -171
  67. package/src/PruneStaleHistory.res +2 -2
  68. package/src/PruneStaleHistory.res.mjs +3 -3
  69. package/src/RawEvent.res +7 -2
  70. package/src/RawEvent.res.mjs +4 -4
  71. package/src/Rollback.res +3 -3
  72. package/src/Rollback.res.mjs +4 -4
  73. package/src/RollbackCommit.res +4 -1
  74. package/src/RollbackCommit.res.mjs +3 -2
  75. package/src/SimulateDeadInputTracker.res +85 -0
  76. package/src/SimulateDeadInputTracker.res.mjs +73 -0
  77. package/src/SimulateDeadInputTracker.resi +12 -0
  78. package/src/SimulateItems.res +91 -23
  79. package/src/SimulateItems.res.mjs +55 -32
  80. package/src/TestIndexer.res +54 -40
  81. package/src/TestIndexer.res.mjs +37 -30
  82. package/src/bindings/Performance.res +7 -0
  83. package/src/bindings/Performance.res.mjs +21 -0
  84. package/src/bindings/Performance.resi +7 -0
  85. package/src/bindings/Viem.res +0 -41
  86. package/src/bindings/Viem.res.mjs +1 -43
  87. package/src/sources/BlockStore.res +46 -0
  88. package/src/sources/BlockStore.res.mjs +24 -0
  89. package/src/sources/EventRouter.res +36 -33
  90. package/src/sources/EventRouter.res.mjs +13 -14
  91. package/src/sources/Evm.res +83 -72
  92. package/src/sources/Evm.res.mjs +53 -69
  93. package/src/sources/EvmChain.res +18 -20
  94. package/src/sources/EvmChain.res.mjs +15 -15
  95. package/src/sources/EvmRpcClient.res +74 -5
  96. package/src/sources/EvmRpcClient.res.mjs +13 -4
  97. package/src/sources/FieldMask.res +39 -0
  98. package/src/sources/FieldMask.res.mjs +42 -0
  99. package/src/sources/Fuel.res +9 -6
  100. package/src/sources/Fuel.res.mjs +5 -10
  101. package/src/sources/HyperFuelSource.res +46 -50
  102. package/src/sources/HyperFuelSource.res.mjs +64 -54
  103. package/src/sources/HyperSync.res +39 -7
  104. package/src/sources/HyperSync.res.mjs +57 -37
  105. package/src/sources/HyperSync.resi +12 -2
  106. package/src/sources/HyperSyncClient.res +22 -111
  107. package/src/sources/HyperSyncClient.res.mjs +2 -30
  108. package/src/sources/HyperSyncHeightStream.res +1 -8
  109. package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
  110. package/src/sources/HyperSyncSource.res +94 -107
  111. package/src/sources/HyperSyncSource.res.mjs +82 -86
  112. package/src/sources/Rpc.res +15 -47
  113. package/src/sources/Rpc.res.mjs +25 -56
  114. package/src/sources/RpcSource.res +320 -467
  115. package/src/sources/RpcSource.res.mjs +268 -385
  116. package/src/sources/RpcWebSocketHeightStream.res +0 -5
  117. package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
  118. package/src/sources/SimulateSource.res +10 -4
  119. package/src/sources/SimulateSource.res.mjs +16 -6
  120. package/src/sources/Source.res +34 -6
  121. package/src/sources/SourceManager.res +77 -19
  122. package/src/sources/SourceManager.res.mjs +63 -21
  123. package/src/sources/SourceManager.resi +10 -0
  124. package/src/sources/Svm.res +31 -16
  125. package/src/sources/Svm.res.mjs +36 -13
  126. package/src/sources/SvmHyperSyncClient.res +5 -32
  127. package/src/sources/SvmHyperSyncSource.res +159 -134
  128. package/src/sources/SvmHyperSyncSource.res.mjs +164 -129
  129. package/src/sources/TransactionStore.res +49 -0
  130. package/src/sources/TransactionStore.res.mjs +30 -0
  131. package/src/tui/Tui.res +13 -16
  132. package/src/tui/Tui.res.mjs +12 -14
  133. package/svm.schema.json +49 -37
  134. package/src/bindings/Hrtime.res +0 -58
  135. package/src/bindings/Hrtime.res.mjs +0 -90
  136. package/src/bindings/Hrtime.resi +0 -30
@@ -57,7 +57,7 @@ let totalReadyCount = (crossChainState: t) => {
57
57
  let total = ref(0)
58
58
  for i in 0 to crossChainState.chainIds->Array.length - 1 {
59
59
  let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
60
- total := total.contents + cs->ChainState.fetchState->FetchState.bufferReadyCount
60
+ total := total.contents + cs->ChainState.bufferReadyCount
61
61
  }
62
62
  total.contents
63
63
  }
@@ -65,9 +65,7 @@ let totalReadyCount = (crossChainState: t) => {
65
65
  // --- Derived (pure). ---
66
66
 
67
67
  let nextItemIsNone = (crossChainState: t): bool =>
68
- !Batch.hasReadyItem(
69
- crossChainState.chainStates->Dict.valuesToArray->Array.map(ChainState.fetchState),
70
- )
68
+ !(crossChainState.chainStates->Dict.valuesToArray->Array.some(ChainState.hasReadyItem))
71
69
 
72
70
  let getSafeCheckpointId = (crossChainState: t) => {
73
71
  let result: ref<option<bigint>> = ref(None)
@@ -79,7 +77,7 @@ let getSafeCheckpointId = (crossChainState: t) => {
79
77
  | Some(safeCheckpointTracking) =>
80
78
  let safeCheckpointId =
81
79
  safeCheckpointTracking->SafeCheckpointTracking.getSafeCheckpointId(
82
- ~sourceBlockNumber=(cs->ChainState.fetchState).knownHeight,
80
+ ~sourceBlockNumber=cs->ChainState.knownHeight,
83
81
  )
84
82
  switch result.contents {
85
83
  | None => result := Some(safeCheckpointId)
@@ -111,16 +109,9 @@ let createBatch = (
111
109
  // in an append-only ClickHouse insert.
112
110
  isRollback ? 1n : 0n,
113
111
  ),
114
- ~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues((
115
- cs
116
- ): Batch.chainBeforeBatch => {
117
- fetchState: cs->ChainState.fetchState,
118
- progressBlockNumber: cs->ChainState.committedProgressBlockNumber,
119
- totalEventsProcessed: cs->ChainState.numEventsProcessed,
120
- sourceBlockNumber: (cs->ChainState.fetchState).knownHeight,
121
- reorgDetection: cs->ChainState.reorgDetection,
122
- chainConfig: cs->ChainState.chainConfig,
123
- }),
112
+ ~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
113
+ ChainState.toChainBeforeBatch,
114
+ ),
124
115
  ~batchSizeTarget,
125
116
  )
126
117
  }
@@ -144,13 +135,13 @@ let enterReorgThreshold = (crossChainState: t) => {
144
135
  // whole indexer. A chain is marked caught up only once EVERY chain is caught up
145
136
  // (reached endblock or fetched/processed to head) with no processable events
146
137
  // left — so no chain flips to ready while another is still backfilling.
147
- let applyBatchProgress = (crossChainState: t, ~batch: Batch.t) => {
138
+ let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampName: string) => {
148
139
  let chainIds = crossChainState.chainIds
149
140
 
150
141
  let everyChainCaughtUp = ref(true)
151
142
  for i in 0 to chainIds->Array.length - 1 {
152
143
  let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
153
- cs->ChainState.applyBatchProgress(~batch)
144
+ cs->ChainState.applyBatchProgress(~batch, ~blockTimestampName)
154
145
  if !(cs->ChainState.hasProcessedToEndblock || cs->ChainState.isProgressAtHead) {
155
146
  everyChainCaughtUp := false
156
147
  }
@@ -184,10 +175,7 @@ let priorityOrder = (crossChainState: t) =>
184
175
  crossChainState.chainStates
185
176
  ->Dict.valuesToArray
186
177
  ->Array.toSorted((a, b) =>
187
- Float.compare(
188
- a->ChainState.fetchState->FetchState.getProgressPercentage,
189
- b->ChainState.fetchState->FetchState.getProgressPercentage,
190
- )
178
+ Float.compare(a->ChainState.getProgressPercentage, b->ChainState.getProgressPercentage)
191
179
  )
192
180
 
193
181
  // In-flight estimated items across every chain — the live draw against
@@ -202,86 +190,122 @@ let totalReservedSize = (crossChainState: t) => {
202
190
  total.contents
203
191
  }
204
192
 
205
- // Furthest-behind first: order candidate queries by the chain progress % at
206
- // their fromBlock, so the most behind ranges across all chains are admitted
207
- // before the rest.
208
- let compareByProgress = (a: FetchState.query, b: FetchState.query) =>
209
- Float.compare(a.progress, b.progress)
210
-
211
193
  // Dispatch a fetch tick across the whole indexer from one shared pool of
212
- // ~targetBufferSize ready events. Every chain proposes its candidate queries
213
- // (each carrying an estimated response size) against the full free budget; the
214
- // candidates are then pooled, ordered by chain progress (furthest-behind first),
215
- // and admitted until the budget is consumed. So the budget is split per query
216
- // across chains rather than per chain a chain that can only use a little
217
- // leaves the rest for the others automatically.
194
+ // ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
195
+ // first, hand each the budget remaining at that point (plus its own
196
+ // already-reserved share, since a chain's pending queries aren't "spent" —
197
+ // they're this chain's), let it turn that into queries sized against its own
198
+ // chain-density-derived target block, then subtract what it actually used
199
+ // before moving to the next chain. So a chain that can only use a little
200
+ // (density too low, or already caught up) leaves the rest for the others
201
+ // automatically. A chain visited after the budget ran out simply doesn't
202
+ // query this round — the leader's reservations release as its responses land,
203
+ // so the next tick redistributes. Chains that do get budget are additionally
204
+ // capped at the most-behind chain's target progress mapped onto their own
205
+ // range, so no chain runs further ahead than the chain the pool is
206
+ // prioritizing.
218
207
  let checkAndFetch = async (
219
208
  crossChainState: t,
220
209
  ~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
221
210
  ) => {
222
- let remaining = Pervasives.max(
223
- 0,
224
- crossChainState.targetBufferSize -
225
- crossChainState->totalReadyCount -
226
- crossChainState->totalReservedSize->Float.toInt,
211
+ let remaining = ref(
212
+ Pervasives.max(
213
+ 0.,
214
+ crossChainState.targetBufferSize->Int.toFloat -.
215
+ crossChainState->totalReadyCount->Int.toFloat -.
216
+ crossChainState->totalReservedSize,
217
+ ),
227
218
  )
228
219
 
229
- let chainIds = crossChainState.chainIds
220
+ // A chain with no density signal probes blind, so it only gets a bounded
221
+ // slice of the pool — one unknown chain shouldn't hold the whole budget
222
+ // while it takes its first measurements.
223
+ let coldChainBudget = Pervasives.min(5_000., crossChainState.targetBufferSize->Int.toFloat)
224
+
225
+ // Chunk reservations get headroom over the density estimate so a
226
+ // denser-than-expected range doesn't truncate at the server cap; realtime
227
+ // gets more since a forced catch-up query there costs a head-poll roundtrip.
228
+ let chunkItemsMultiplier = crossChainState.isRealtime ? 3. : 1.5
229
+
230
230
  let actionByChain = Dict.make()
231
- // Candidate queries from every chain. Each query carries its chain id and the
232
- // chain progress % at its fromBlock (the admission sort key), set here so the
233
- // pool can be ordered without a side tuple per query.
234
- let candidates = []
235
- for i in 0 to chainIds->Array.length - 1 {
236
- let chainId = chainIds->Array.getUnsafe(i)
237
- let cs = crossChainState->getChainState(chainId)
238
- let fetchState = cs->ChainState.fetchState
239
- switch fetchState->FetchState.getNextQuery(
240
- ~budget=remaining,
241
- ~chainPendingBudget=cs->ChainState.pendingBudget,
242
- ) {
243
- | (WaitingForNewBlock | NothingToQuery) as action =>
244
- actionByChain->Utils.Dict.setByInt(chainId, action)
245
- | Ready(queries) =>
246
- // Default to NothingToQuery; replaced below if any candidate is admitted.
247
- actionByChain->Utils.Dict.setByInt(chainId, FetchState.NothingToQuery)
248
- queries->Array.forEach(query => {
249
- query.chainId = chainId
250
- query.progress =
251
- fetchState->FetchState.getProgressPercentageAt(~blockNumber=query.fromBlock)
252
- candidates->Array.push(query)
253
- })
231
+ let maxProgress = ref(None)
232
+ crossChainState
233
+ ->priorityOrder
234
+ ->Array.forEach(cs => {
235
+ let chainId = (cs->ChainState.chainConfig).id
236
+ if cs->ChainState.knownHeight == 0 {
237
+ // No height yet — nothing to size a budget or an alignment line
238
+ // against. Skip without consuming budget or claiming leadership, so a
239
+ // chain whose source hasn't reported doesn't unconstrain everyone else.
240
+ actionByChain->Utils.Dict.setByInt(chainId, FetchState.WaitingForNewBlock)
241
+ } else {
242
+ let isCold = cs->ChainState.effectiveDensity === None
243
+ let chainTargetItems =
244
+ (isCold ? Pervasives.min(remaining.contents, coldChainBudget) : remaining.contents) +.
245
+ cs->ChainState.pendingBudget
246
+ let maxTargetBlock = switch maxProgress.contents {
247
+ | None if isCold =>
248
+ // A cold chain's target is a guess — it doesn't set the alignment
249
+ // line; the first density-bearing chain after it does.
250
+ None
251
+ | None =>
252
+ // The most-behind chain sets the alignment line for everyone after it.
253
+ maxProgress :=
254
+ Some(
255
+ cs->ChainState.progressAtBlock(
256
+ ~blockNumber=cs->ChainState.targetBlock(~chainTargetItems),
257
+ ),
258
+ )
259
+ None
260
+ // 5% margin past the leader's line: chains whose progress tracks the
261
+ // leader closely would otherwise flap in and out of the clamp as
262
+ // leadership alternates, stalling their pipeline every other tick.
263
+ | Some(progress) => Some(cs->ChainState.blockAtProgress(~progress=progress +. 0.05))
264
+ }
265
+ switch cs->ChainState.getNextQuery(
266
+ ~chainTargetItems,
267
+ ~chunkItemsMultiplier,
268
+ ~maxTargetBlock?,
269
+ ) {
270
+ | (WaitingForNewBlock | NothingToQuery) as action =>
271
+ actionByChain->Utils.Dict.setByInt(chainId, action)
272
+ | Ready(queries) => {
273
+ let consumed =
274
+ queries->Array.reduce(0., (acc, query: FetchState.query) =>
275
+ acc +. query.itemsEst->Int.toFloat
276
+ )
277
+
278
+ let partitions = Dict.make()
279
+ queries->Array.forEach((query: FetchState.query) =>
280
+ partitions->Dict.set(
281
+ query.partitionId,
282
+ {
283
+ "fromBlock": query.fromBlock,
284
+ "targetBlock": query.toBlock,
285
+ "targetEvents": query.itemsTarget,
286
+ },
287
+ )
288
+ )
289
+ Logging.trace({
290
+ "msg": "Started querying",
291
+ "chainId": chainId,
292
+ "partitions": partitions,
293
+ })
294
+
295
+ actionByChain->Utils.Dict.setByInt(chainId, FetchState.Ready(queries))
296
+ // Mark the queries in flight and reserve their size against the
297
+ // shared budget; released as each response lands in
298
+ // handleQueryResult.
299
+ cs->ChainState.startFetchingQueries(~queries)
300
+ remaining := Pervasives.max(0., remaining.contents -. consumed)
301
+ }
302
+ }
254
303
  }
255
- }
256
-
257
- candidates->Array.sort(compareByProgress)
258
-
259
- // Admit furthest-behind first until the budget runs out. The condition is
260
- // checked before each query, so as long as there's any budget left we admit
261
- // the next one even when its estimate alone exceeds the remainder — otherwise a
262
- // chain whose only query is bigger than the budget would never make progress.
263
- let admittedByChain = Dict.make()
264
- let running = ref(0.)
265
- let remainingF = remaining->Int.toFloat
266
- let idx = ref(0)
267
- while running.contents < remainingF && idx.contents < candidates->Array.length {
268
- let query = candidates->Array.getUnsafe(idx.contents)
269
- admittedByChain->Utils.Dict.push(query.chainId->Int.toString, query)
270
- running := running.contents +. query.estResponseSize
271
- idx := idx.contents + 1
272
- }
273
- admittedByChain->Dict.forEachWithKey((queries, chainId) => {
274
- actionByChain->Dict.set(chainId, FetchState.Ready(queries))
275
- // Mark the admitted queries in flight and reserve their size against the
276
- // shared budget; released as each response lands in handleQueryResult.
277
- crossChainState
278
- ->getChainState(chainId->Int.fromString->Option.getUnsafe)
279
- ->ChainState.startFetchingQueries(~queries)
280
304
  })
281
305
 
282
306
  let promises = []
283
- for i in 0 to chainIds->Array.length - 1 {
284
- let chainId = chainIds->Array.getUnsafe(i)
307
+ for i in 0 to crossChainState.chainIds->Array.length - 1 {
308
+ let chainId = crossChainState.chainIds->Array.getUnsafe(i)
285
309
  switch actionByChain->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
286
310
  | Some(NothingToQuery)
287
311
  | None => ()
@@ -6,11 +6,8 @@ import * as Utils from "./Utils.res.mjs";
6
6
  import * as Logging from "./Logging.res.mjs";
7
7
  import * as ChainMap from "./ChainMap.res.mjs";
8
8
  import * as ChainState from "./ChainState.res.mjs";
9
- import * as FetchState from "./FetchState.res.mjs";
10
9
  import * as Prometheus from "./Prometheus.res.mjs";
11
- import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
12
- import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
13
- import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
10
+ import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
14
11
  import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
15
12
  import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
16
13
 
@@ -53,13 +50,13 @@ function totalReadyCount(crossChainState) {
53
50
  let total = 0;
54
51
  for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
55
52
  let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
56
- total = total + FetchState.bufferReadyCount(ChainState.fetchState(cs)) | 0;
53
+ total = total + ChainState.bufferReadyCount(cs) | 0;
57
54
  }
58
55
  return total;
59
56
  }
60
57
 
61
58
  function nextItemIsNone(crossChainState) {
62
- return !Batch.hasReadyItem(Object.values(crossChainState.chainStates).map(ChainState.fetchState));
59
+ return !Object.values(crossChainState.chainStates).some(ChainState.hasReadyItem);
63
60
  }
64
61
 
65
62
  function getSafeCheckpointId(crossChainState) {
@@ -68,7 +65,7 @@ function getSafeCheckpointId(crossChainState) {
68
65
  let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
69
66
  let safeCheckpointTracking = ChainState.safeCheckpointTracking(cs);
70
67
  if (safeCheckpointTracking !== undefined) {
71
- let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.fetchState(cs).knownHeight);
68
+ let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.knownHeight(cs));
72
69
  let current = result;
73
70
  if (!(current !== undefined && safeCheckpointId >= current)) {
74
71
  result = safeCheckpointId;
@@ -84,14 +81,7 @@ function getSafeCheckpointId(crossChainState) {
84
81
  function createBatch(crossChainState, processedCheckpointId, batchSizeTarget, isRollback) {
85
82
  return Batch.make(processedCheckpointId + (
86
83
  isRollback ? 1n : 0n
87
- ), Utils.Dict.mapValues(crossChainState.chainStates, cs => ({
88
- fetchState: ChainState.fetchState(cs),
89
- reorgDetection: ChainState.reorgDetection(cs),
90
- progressBlockNumber: ChainState.committedProgressBlockNumber(cs),
91
- sourceBlockNumber: ChainState.fetchState(cs).knownHeight,
92
- totalEventsProcessed: ChainState.numEventsProcessed(cs),
93
- chainConfig: ChainState.chainConfig(cs)
94
- })), batchSizeTarget, crossChainState.isInReorgThreshold);
84
+ ), Utils.Dict.mapValues(crossChainState.chainStates, ChainState.toChainBeforeBatch), batchSizeTarget, crossChainState.isInReorgThreshold);
95
85
  }
96
86
 
97
87
  function enterReorgThreshold(crossChainState) {
@@ -103,12 +93,12 @@ function enterReorgThreshold(crossChainState) {
103
93
  crossChainState.isInReorgThreshold = true;
104
94
  }
105
95
 
106
- function applyBatchProgress(crossChainState, batch) {
96
+ function applyBatchProgress(crossChainState, batch, blockTimestampName) {
107
97
  let chainIds = crossChainState.chainIds;
108
98
  let everyChainCaughtUp = true;
109
99
  for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
110
100
  let cs = crossChainState.chainStates[chainIds[i]];
111
- ChainState.applyBatchProgress(cs, batch);
101
+ ChainState.applyBatchProgress(cs, batch, blockTimestampName);
112
102
  if (!(ChainState.hasProcessedToEndblock(cs) || ChainState.isProgressAtHead(cs))) {
113
103
  everyChainCaughtUp = false;
114
104
  }
@@ -131,7 +121,7 @@ function applyBatchProgress(crossChainState, batch) {
131
121
  }
132
122
 
133
123
  function priorityOrder(crossChainState) {
134
- return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(FetchState.getProgressPercentage(ChainState.fetchState(a)), FetchState.getProgressPercentage(ChainState.fetchState(b))));
124
+ return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(ChainState.getProgressPercentage(a), ChainState.getProgressPercentage(b)));
135
125
  }
136
126
 
137
127
  function totalReservedSize(crossChainState) {
@@ -143,61 +133,75 @@ function totalReservedSize(crossChainState) {
143
133
  return total;
144
134
  }
145
135
 
146
- function compareByProgress(a, b) {
147
- return Primitive_float.compare(a.progress, b.progress);
148
- }
149
-
150
136
  async function checkAndFetch(crossChainState, dispatchChain) {
151
- let remaining = Primitive_int.max(0, (crossChainState.targetBufferSize - totalReadyCount(crossChainState) | 0) - (totalReservedSize(crossChainState) | 0) | 0);
152
- let chainIds = crossChainState.chainIds;
137
+ let remaining = {
138
+ contents: Primitive_float.max(0, crossChainState.targetBufferSize - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
139
+ };
140
+ let coldChainBudget = Primitive_float.min(5000, crossChainState.targetBufferSize);
141
+ let chunkItemsMultiplier = crossChainState.isRealtime ? 3 : 1.5;
153
142
  let actionByChain = {};
154
- let candidates = [];
155
- for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
156
- let chainId = chainIds[i];
157
- let cs = crossChainState.chainStates[chainId];
158
- let fetchState = ChainState.fetchState(cs);
159
- let action = FetchState.getNextQuery(fetchState, remaining, ChainState.pendingBudget(cs));
143
+ let maxProgress = {
144
+ contents: undefined
145
+ };
146
+ priorityOrder(crossChainState).forEach(cs => {
147
+ let chainId = ChainState.chainConfig(cs).id;
148
+ if (ChainState.knownHeight(cs) === 0) {
149
+ actionByChain[chainId] = "WaitingForNewBlock";
150
+ return;
151
+ }
152
+ let isCold = ChainState.effectiveDensity(cs) === undefined;
153
+ let chainTargetItems = (
154
+ isCold ? Primitive_float.min(remaining.contents, coldChainBudget) : remaining.contents
155
+ ) + ChainState.pendingBudget(cs);
156
+ let progress = maxProgress.contents;
157
+ let maxTargetBlock = progress !== undefined ? ChainState.blockAtProgress(cs, progress + 0.05) : (
158
+ isCold ? undefined : (maxProgress.contents = ChainState.progressAtBlock(cs, ChainState.targetBlock(cs, chainTargetItems)), undefined)
159
+ );
160
+ let action = ChainState.getNextQuery(cs, chainTargetItems, chunkItemsMultiplier, maxTargetBlock);
160
161
  if (typeof action !== "object") {
162
+ if (action === "WaitingForNewBlock") {
163
+ actionByChain[chainId] = action;
164
+ return;
165
+ }
161
166
  actionByChain[chainId] = action;
167
+ return;
162
168
  } else {
163
- actionByChain[chainId] = "NothingToQuery";
164
- action._0.forEach(query => {
165
- query.chainId = chainId;
166
- query.progress = FetchState.getProgressPercentageAt(fetchState, query.fromBlock);
167
- candidates.push(query);
169
+ let queries = action._0;
170
+ let consumed = Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsEst);
171
+ let partitions = {};
172
+ queries.forEach(query => {
173
+ partitions[query.partitionId] = {
174
+ fromBlock: query.fromBlock,
175
+ targetBlock: query.toBlock,
176
+ targetEvents: query.itemsTarget
177
+ };
178
+ });
179
+ Logging.trace({
180
+ msg: "Started querying",
181
+ chainId: chainId,
182
+ partitions: partitions
168
183
  });
184
+ actionByChain[chainId] = {
185
+ TAG: "Ready",
186
+ _0: queries
187
+ };
188
+ ChainState.startFetchingQueries(cs, queries);
189
+ remaining.contents = Primitive_float.max(0, remaining.contents - consumed);
190
+ return;
169
191
  }
170
- }
171
- candidates.sort(compareByProgress);
172
- let admittedByChain = {};
173
- let running = 0;
174
- let idx = 0;
175
- while (running < remaining && idx < candidates.length) {
176
- let query = candidates[idx];
177
- Utils.Dict.push(admittedByChain, query.chainId.toString(), query);
178
- running = running + query.estResponseSize;
179
- idx = idx + 1 | 0;
180
- };
181
- Stdlib_Dict.forEachWithKey(admittedByChain, (queries, chainId) => {
182
- actionByChain[chainId] = {
183
- TAG: "Ready",
184
- _0: queries
185
- };
186
- let chainId$1 = Stdlib_Int.fromString(chainId, undefined);
187
- ChainState.startFetchingQueries(crossChainState.chainStates[chainId$1], queries);
188
192
  });
189
193
  let promises = [];
190
- for (let i$1 = 0, i_finish$1 = chainIds.length; i$1 < i_finish$1; ++i$1) {
191
- let chainId$1 = chainIds[i$1];
192
- let action$1 = actionByChain[chainId$1];
193
- if (action$1 !== undefined) {
194
+ for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
195
+ let chainId = crossChainState.chainIds[i];
196
+ let action = actionByChain[chainId];
197
+ if (action !== undefined) {
194
198
  let exit = 0;
195
- if (typeof action$1 === "object" || action$1 === "WaitingForNewBlock") {
199
+ if (typeof action === "object" || action === "WaitingForNewBlock") {
196
200
  exit = 1;
197
201
  }
198
202
  if (exit === 1) {
199
- let chain = ChainMap.Chain.makeUnsafe(chainId$1);
200
- promises.push(dispatchChain(chain, action$1));
203
+ let chain = ChainMap.Chain.makeUnsafe(chainId);
204
+ promises.push(dispatchChain(chain, action));
201
205
  }
202
206
  }
203
207
  }
@@ -29,7 +29,7 @@ let createBatch: (
29
29
  ~isRollback: bool,
30
30
  ) => Batch.t
31
31
  let enterReorgThreshold: t => unit
32
- let applyBatchProgress: (t, ~batch: Batch.t) => unit
32
+ let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
33
33
 
34
34
  // Fetch control.
35
35
  let priorityOrder: t => array<ChainState.t>
package/src/Ecosystem.res CHANGED
@@ -2,8 +2,6 @@ type name = | @as("evm") Evm | @as("fuel") Fuel | @as("svm") Svm
2
2
 
3
3
  type t = {
4
4
  name: name,
5
- blockFields: array<string>,
6
- transactionFields: array<string>,
7
5
  blockNumberName: string,
8
6
  blockTimestampName: string,
9
7
  blockHashName: string,
@@ -33,7 +31,8 @@ type t = {
33
31
  loggers (see `getItemLogger`). */
34
32
  logger: Pino.t,
35
33
  /** Materialise the user-facing event handed to handlers and contract
36
- registration from an item's opaque payload. */
34
+ registration from an item's opaque payload. `event.transaction` is written
35
+ onto the payload at batch prep (HyperSync) or inline (RPC/simulate). */
37
36
  toEvent: Internal.eventItem => Internal.event,
38
37
  /** Build the per-item child logger for an event item, with
39
38
  ecosystem-specific log fields (EVM/Fuel: contract/event/address; SVM:
@@ -70,12 +69,12 @@ let getItemLogger = {
70
69
  | None =>
71
70
  let logger = switch item {
72
71
  | Internal.Event(_) => ecosystem.toEventLogger(item->Internal.castUnsafeEventItem)
73
- | Block({blockNumber, onBlockConfig}) =>
72
+ | Block({blockNumber, onBlockRegistration}) =>
74
73
  Logging.createChildFrom(
75
74
  ~logger=ecosystem.logger,
76
75
  ~params={
77
- "onBlock": onBlockConfig.name,
78
- "chainId": onBlockConfig.chainId,
76
+ "onBlock": onBlockRegistration.name,
77
+ "chainId": onBlockRegistration.chainId,
79
78
  "block": blockNumber,
80
79
  },
81
80
  )
@@ -26,10 +26,10 @@ function getItemLogger(item, ecosystem) {
26
26
  if (item.kind === 0) {
27
27
  logger$1 = ecosystem.toEventLogger(item);
28
28
  } else {
29
- let onBlockConfig = item.onBlockConfig;
29
+ let onBlockRegistration = item.onBlockRegistration;
30
30
  logger$1 = Logging.createChildFrom(ecosystem.logger, {
31
- onBlock: onBlockConfig.name,
32
- chainId: onBlockConfig.chainId,
31
+ onBlock: onBlockRegistration.name,
32
+ chainId: onBlockRegistration.chainId,
33
33
  block: item.blockNumber
34
34
  });
35
35
  }
package/src/Envio.res CHANGED
@@ -48,6 +48,7 @@ type svmTokenBalance = {
48
48
  }
49
49
 
50
50
  type svmTransaction = {
51
+ transactionIndex?: int,
51
52
  signatures: array<string>,
52
53
  feePayer?: SvmTypes.Pubkey.t,
53
54
  success?: bool,
@@ -65,18 +66,21 @@ type svmLog = {
65
66
  message: string,
66
67
  }
67
68
 
68
- /** Block context for a matched instruction. `time`/`hash` follow the
69
- EVM/Fuel field names so the shared `Ecosystem.t` getters in `Svm.res` read
70
- them uniformly. */
69
+ /** Block context for a matched instruction. `slot`/`hash` are always present;
70
+ `time` can still be absent HyperSync/Solana may not report a block time for
71
+ every slot. Every other field is opt-in via `field_selection.block_fields`,
72
+ materialised from the per-chain block store at batch prep. */
71
73
  type svmInstructionBlock = {
72
74
  /** Slot this instruction's block was matched in. */
73
75
  slot: int,
74
- /** Unix block time (seconds). `0` when HyperSync didn't return a block
75
- for this instruction's slot. */
76
- time: int,
77
- /** Block hash. Currently always empty — populated by the future
78
- reorg-guard `queryBlockHash(slot)` route. */
76
+ /** Unix block time (seconds). */
77
+ time?: int,
79
78
  hash: string,
79
+ /** Block height (distinct from slot). Absent when not selected via
80
+ `field_selection.block_fields`. */
81
+ height?: int,
82
+ parentSlot?: int,
83
+ parentHash?: string,
80
84
  }
81
85
 
82
86
  /** The per-instruction payload handlers receive as their `instruction`
@@ -104,13 +108,15 @@ type svmInstruction = {
104
108
  d8?: string,
105
109
  /** Borsh-decoded params view. See [[svmInstructionParams]]. */
106
110
  params?: svmInstructionParams,
107
- /** Parent transaction. Absent when the per-instruction
108
- `include_transaction` flag is `false`. */
111
+ /** Parent transaction. Carries only the fields selected via
112
+ `field_selection.transaction_fields`; absent when none are selected. */
109
113
  transaction?: svmTransaction,
110
114
  /** Program log entries scoped to this instruction. Absent when the
111
115
  per-instruction `include_logs` flag is `false`. */
112
116
  logs?: array<svmLog>,
113
- block: svmInstructionBlock,
117
+ // Omitted on construction; materialised onto the payload at batch prep,
118
+ // before a handler ever reads it.
119
+ block?: svmInstructionBlock,
114
120
  }
115
121
 
116
122
  /** Arguments passed to handlers registered via `indexer.onInstruction`. */
@@ -0,0 +1,53 @@
1
+ // The single process-wide mutable state record, stashed on `globalThis` so a
2
+ // duplicate envio module instance — e.g. when the CLI's `bin.mjs` resolves
3
+ // envio from one path but the user's handlers resolve it from
4
+ // `node_modules/envio` — shares one state. Without this, each copy keeps its
5
+ // own module-level state: registration would read an empty registry and the
6
+ // `indexer.chains` getters would silently fall back to static config values.
7
+ //
8
+ // Slots are opaque (`unknown`) so this module stays at the bottom of the
9
+ // dependency graph; each owning module casts its slot to the real type once
10
+ // (`HandlerRegister` for the registration slots, `Main` for the runtime
11
+ // slots, `RollbackCommit` for its callbacks).
12
+ //
13
+ // Version-gated: the slot shapes can evolve between envio versions, so the
14
+ // guard uses strict full-version equality. On mismatch we throw with a
15
+ // deduplication hint instead of silently mixing shapes across builds.
16
+ type t = {
17
+ version: string,
18
+ eventRegistrations: dict<unknown>,
19
+ mutable activeRegistration: option<unknown>,
20
+ preRegistered: array<unknown>,
21
+ rollbackCommitCallbacks: array<unknown>,
22
+ mutable indexerState: option<unknown>,
23
+ mutable persistence: option<unknown>,
24
+ }
25
+
26
+ // Record type with `mutable` so assignment typechecks; ReScript keeps the
27
+ // field name verbatim in the generated JS so the globalThis slot is
28
+ // `__envioGlobal`.
29
+ type globalThis = {mutable __envioGlobal: Nullable.t<t>}
30
+ @val external globalThis: globalThis = "globalThis"
31
+
32
+ let value: t = {
33
+ let version = Utils.EnvioPackage.value.version
34
+ switch globalThis.__envioGlobal->Nullable.toOption {
35
+ | Some(existing) if existing.version === version => existing
36
+ | Some(existing) =>
37
+ JsError.throwWithMessage(
38
+ `Multiple incompatible envio versions loaded in the same process: ${existing.version} and ${version}. Deduplicate the 'envio' dependency in your project.`,
39
+ )
40
+ | None =>
41
+ let fresh = {
42
+ version,
43
+ eventRegistrations: Dict.make(),
44
+ activeRegistration: None,
45
+ preRegistered: [],
46
+ rollbackCommitCallbacks: [],
47
+ indexerState: None,
48
+ persistence: None,
49
+ }
50
+ globalThis.__envioGlobal = Nullable.make(fresh)
51
+ fresh
52
+ }
53
+ }