envio 3.3.0-alpha.1 → 3.3.0-alpha.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +10 -0
- package/index.d.ts +45 -25
- package/package.json +7 -7
- package/src/Address.res +5 -2
- package/src/Address.res.mjs +3 -1
- package/src/Api.res +1 -1
- package/src/Api.res.mjs +1 -1
- package/src/Batch.res +0 -6
- package/src/Batch.res.mjs +1 -12
- package/src/BatchProcessing.res +7 -12
- package/src/BatchProcessing.res.mjs +8 -7
- package/src/Bin.res +3 -0
- package/src/Bin.res.mjs +4 -0
- package/src/ChainFetching.res +43 -9
- package/src/ChainFetching.res.mjs +34 -15
- package/src/ChainMetadata.res +1 -11
- package/src/ChainMetadata.res.mjs +1 -10
- package/src/ChainState.res +605 -151
- package/src/ChainState.res.mjs +427 -104
- package/src/ChainState.resi +72 -4
- package/src/Config.res +61 -33
- package/src/Config.res.mjs +50 -10
- package/src/ContractRegisterContext.res +2 -12
- package/src/ContractRegisterContext.res.mjs +3 -5
- package/src/Core.res +14 -3
- package/src/CrossChainState.res +113 -89
- package/src/CrossChainState.res.mjs +64 -60
- package/src/CrossChainState.resi +1 -1
- package/src/Ecosystem.res +5 -6
- package/src/Ecosystem.res.mjs +3 -3
- package/src/Envio.res +17 -11
- package/src/EnvioGlobal.res +53 -0
- package/src/EnvioGlobal.res.mjs +31 -0
- package/src/EventConfigBuilder.res +177 -88
- package/src/EventConfigBuilder.res.mjs +99 -42
- package/src/EventProcessing.res +74 -28
- package/src/EventProcessing.res.mjs +55 -29
- package/src/ExitOnCaughtUp.res +10 -2
- package/src/ExitOnCaughtUp.res.mjs +10 -1
- package/src/FetchState.res +544 -356
- package/src/FetchState.res.mjs +412 -471
- package/src/HandlerLoader.res +8 -104
- package/src/HandlerLoader.res.mjs +2 -81
- package/src/HandlerRegister.res +518 -144
- package/src/HandlerRegister.res.mjs +285 -133
- package/src/HandlerRegister.resi +24 -8
- package/src/IndexerState.res +10 -3
- package/src/IndexerState.res.mjs +11 -4
- package/src/IndexerState.resi +2 -1
- package/src/IndexingAddresses.res +108 -0
- package/src/IndexingAddresses.res.mjs +101 -0
- package/src/IndexingAddresses.resi +34 -0
- package/src/Internal.res +155 -46
- package/src/Internal.res.mjs +30 -2
- package/src/LoadLayer.res +5 -5
- package/src/LoadLayer.res.mjs +6 -6
- package/src/LogSelection.res +102 -165
- package/src/LogSelection.res.mjs +101 -116
- package/src/Main.res +71 -236
- package/src/Main.res.mjs +60 -145
- package/src/Metrics.res +74 -0
- package/src/Metrics.res.mjs +75 -0
- package/src/PgStorage.res +4 -4
- package/src/PgStorage.res.mjs +5 -5
- package/src/Prometheus.res +10 -63
- package/src/Prometheus.res.mjs +124 -171
- package/src/PruneStaleHistory.res +2 -2
- package/src/PruneStaleHistory.res.mjs +3 -3
- package/src/RawEvent.res +7 -2
- package/src/RawEvent.res.mjs +4 -4
- package/src/Rollback.res +3 -3
- package/src/Rollback.res.mjs +4 -4
- package/src/RollbackCommit.res +4 -1
- package/src/RollbackCommit.res.mjs +3 -2
- package/src/SimulateDeadInputTracker.res +85 -0
- package/src/SimulateDeadInputTracker.res.mjs +73 -0
- package/src/SimulateDeadInputTracker.resi +12 -0
- package/src/SimulateItems.res +91 -23
- package/src/SimulateItems.res.mjs +55 -32
- package/src/TestIndexer.res +54 -40
- package/src/TestIndexer.res.mjs +37 -30
- package/src/bindings/Performance.res +7 -0
- package/src/bindings/Performance.res.mjs +21 -0
- package/src/bindings/Performance.resi +7 -0
- package/src/bindings/Viem.res +0 -41
- package/src/bindings/Viem.res.mjs +1 -43
- package/src/sources/BlockStore.res +46 -0
- package/src/sources/BlockStore.res.mjs +24 -0
- package/src/sources/EventRouter.res +36 -33
- package/src/sources/EventRouter.res.mjs +13 -14
- package/src/sources/Evm.res +83 -72
- package/src/sources/Evm.res.mjs +53 -69
- package/src/sources/EvmChain.res +18 -20
- package/src/sources/EvmChain.res.mjs +15 -15
- package/src/sources/EvmRpcClient.res +74 -5
- package/src/sources/EvmRpcClient.res.mjs +13 -4
- package/src/sources/FieldMask.res +39 -0
- package/src/sources/FieldMask.res.mjs +42 -0
- package/src/sources/Fuel.res +9 -6
- package/src/sources/Fuel.res.mjs +5 -10
- package/src/sources/HyperFuelSource.res +46 -50
- package/src/sources/HyperFuelSource.res.mjs +64 -54
- package/src/sources/HyperSync.res +39 -7
- package/src/sources/HyperSync.res.mjs +57 -37
- package/src/sources/HyperSync.resi +12 -2
- package/src/sources/HyperSyncClient.res +22 -111
- package/src/sources/HyperSyncClient.res.mjs +2 -30
- package/src/sources/HyperSyncHeightStream.res +1 -8
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
- package/src/sources/HyperSyncSource.res +94 -107
- package/src/sources/HyperSyncSource.res.mjs +82 -86
- package/src/sources/Rpc.res +15 -47
- package/src/sources/Rpc.res.mjs +25 -56
- package/src/sources/RpcSource.res +320 -467
- package/src/sources/RpcSource.res.mjs +268 -385
- package/src/sources/RpcWebSocketHeightStream.res +0 -5
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
- package/src/sources/SimulateSource.res +10 -4
- package/src/sources/SimulateSource.res.mjs +16 -6
- package/src/sources/Source.res +34 -6
- package/src/sources/SourceManager.res +77 -19
- package/src/sources/SourceManager.res.mjs +63 -21
- package/src/sources/SourceManager.resi +10 -0
- package/src/sources/Svm.res +31 -16
- package/src/sources/Svm.res.mjs +36 -13
- package/src/sources/SvmHyperSyncClient.res +5 -32
- package/src/sources/SvmHyperSyncSource.res +159 -134
- package/src/sources/SvmHyperSyncSource.res.mjs +164 -129
- package/src/sources/TransactionStore.res +49 -0
- package/src/sources/TransactionStore.res.mjs +30 -0
- package/src/tui/Tui.res +13 -16
- package/src/tui/Tui.res.mjs +12 -14
- package/svm.schema.json +49 -37
- package/src/bindings/Hrtime.res +0 -58
- package/src/bindings/Hrtime.res.mjs +0 -90
- package/src/bindings/Hrtime.resi +0 -30
package/src/CrossChainState.res
CHANGED
|
@@ -57,7 +57,7 @@ let totalReadyCount = (crossChainState: t) => {
|
|
|
57
57
|
let total = ref(0)
|
|
58
58
|
for i in 0 to crossChainState.chainIds->Array.length - 1 {
|
|
59
59
|
let cs = crossChainState->getChainState(crossChainState.chainIds->Array.getUnsafe(i))
|
|
60
|
-
total := total.contents + cs->ChainState.
|
|
60
|
+
total := total.contents + cs->ChainState.bufferReadyCount
|
|
61
61
|
}
|
|
62
62
|
total.contents
|
|
63
63
|
}
|
|
@@ -65,9 +65,7 @@ let totalReadyCount = (crossChainState: t) => {
|
|
|
65
65
|
// --- Derived (pure). ---
|
|
66
66
|
|
|
67
67
|
let nextItemIsNone = (crossChainState: t): bool =>
|
|
68
|
-
!
|
|
69
|
-
crossChainState.chainStates->Dict.valuesToArray->Array.map(ChainState.fetchState),
|
|
70
|
-
)
|
|
68
|
+
!(crossChainState.chainStates->Dict.valuesToArray->Array.some(ChainState.hasReadyItem))
|
|
71
69
|
|
|
72
70
|
let getSafeCheckpointId = (crossChainState: t) => {
|
|
73
71
|
let result: ref<option<bigint>> = ref(None)
|
|
@@ -79,7 +77,7 @@ let getSafeCheckpointId = (crossChainState: t) => {
|
|
|
79
77
|
| Some(safeCheckpointTracking) =>
|
|
80
78
|
let safeCheckpointId =
|
|
81
79
|
safeCheckpointTracking->SafeCheckpointTracking.getSafeCheckpointId(
|
|
82
|
-
~sourceBlockNumber=
|
|
80
|
+
~sourceBlockNumber=cs->ChainState.knownHeight,
|
|
83
81
|
)
|
|
84
82
|
switch result.contents {
|
|
85
83
|
| None => result := Some(safeCheckpointId)
|
|
@@ -111,16 +109,9 @@ let createBatch = (
|
|
|
111
109
|
// in an append-only ClickHouse insert.
|
|
112
110
|
isRollback ? 1n : 0n,
|
|
113
111
|
),
|
|
114
|
-
~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
|
|
115
|
-
|
|
116
|
-
)
|
|
117
|
-
fetchState: cs->ChainState.fetchState,
|
|
118
|
-
progressBlockNumber: cs->ChainState.committedProgressBlockNumber,
|
|
119
|
-
totalEventsProcessed: cs->ChainState.numEventsProcessed,
|
|
120
|
-
sourceBlockNumber: (cs->ChainState.fetchState).knownHeight,
|
|
121
|
-
reorgDetection: cs->ChainState.reorgDetection,
|
|
122
|
-
chainConfig: cs->ChainState.chainConfig,
|
|
123
|
-
}),
|
|
112
|
+
~chainsBeforeBatch=crossChainState.chainStates->Utils.Dict.mapValues(
|
|
113
|
+
ChainState.toChainBeforeBatch,
|
|
114
|
+
),
|
|
124
115
|
~batchSizeTarget,
|
|
125
116
|
)
|
|
126
117
|
}
|
|
@@ -144,13 +135,13 @@ let enterReorgThreshold = (crossChainState: t) => {
|
|
|
144
135
|
// whole indexer. A chain is marked caught up only once EVERY chain is caught up
|
|
145
136
|
// (reached endblock or fetched/processed to head) with no processable events
|
|
146
137
|
// left — so no chain flips to ready while another is still backfilling.
|
|
147
|
-
let applyBatchProgress = (crossChainState: t, ~batch: Batch.t) => {
|
|
138
|
+
let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampName: string) => {
|
|
148
139
|
let chainIds = crossChainState.chainIds
|
|
149
140
|
|
|
150
141
|
let everyChainCaughtUp = ref(true)
|
|
151
142
|
for i in 0 to chainIds->Array.length - 1 {
|
|
152
143
|
let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
|
|
153
|
-
cs->ChainState.applyBatchProgress(~batch)
|
|
144
|
+
cs->ChainState.applyBatchProgress(~batch, ~blockTimestampName)
|
|
154
145
|
if !(cs->ChainState.hasProcessedToEndblock || cs->ChainState.isProgressAtHead) {
|
|
155
146
|
everyChainCaughtUp := false
|
|
156
147
|
}
|
|
@@ -184,10 +175,7 @@ let priorityOrder = (crossChainState: t) =>
|
|
|
184
175
|
crossChainState.chainStates
|
|
185
176
|
->Dict.valuesToArray
|
|
186
177
|
->Array.toSorted((a, b) =>
|
|
187
|
-
Float.compare(
|
|
188
|
-
a->ChainState.fetchState->FetchState.getProgressPercentage,
|
|
189
|
-
b->ChainState.fetchState->FetchState.getProgressPercentage,
|
|
190
|
-
)
|
|
178
|
+
Float.compare(a->ChainState.getProgressPercentage, b->ChainState.getProgressPercentage)
|
|
191
179
|
)
|
|
192
180
|
|
|
193
181
|
// In-flight estimated items across every chain — the live draw against
|
|
@@ -202,86 +190,122 @@ let totalReservedSize = (crossChainState: t) => {
|
|
|
202
190
|
total.contents
|
|
203
191
|
}
|
|
204
192
|
|
|
205
|
-
// Furthest-behind first: order candidate queries by the chain progress % at
|
|
206
|
-
// their fromBlock, so the most behind ranges across all chains are admitted
|
|
207
|
-
// before the rest.
|
|
208
|
-
let compareByProgress = (a: FetchState.query, b: FetchState.query) =>
|
|
209
|
-
Float.compare(a.progress, b.progress)
|
|
210
|
-
|
|
211
193
|
// Dispatch a fetch tick across the whole indexer from one shared pool of
|
|
212
|
-
// ~targetBufferSize ready events
|
|
213
|
-
//
|
|
214
|
-
//
|
|
215
|
-
//
|
|
216
|
-
//
|
|
217
|
-
//
|
|
194
|
+
// ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
|
|
195
|
+
// first, hand each the budget remaining at that point (plus its own
|
|
196
|
+
// already-reserved share, since a chain's pending queries aren't "spent" —
|
|
197
|
+
// they're this chain's), let it turn that into queries sized against its own
|
|
198
|
+
// chain-density-derived target block, then subtract what it actually used
|
|
199
|
+
// before moving to the next chain. So a chain that can only use a little
|
|
200
|
+
// (density too low, or already caught up) leaves the rest for the others
|
|
201
|
+
// automatically. A chain visited after the budget ran out simply doesn't
|
|
202
|
+
// query this round — the leader's reservations release as its responses land,
|
|
203
|
+
// so the next tick redistributes. Chains that do get budget are additionally
|
|
204
|
+
// capped at the most-behind chain's target progress mapped onto their own
|
|
205
|
+
// range, so no chain runs further ahead than the chain the pool is
|
|
206
|
+
// prioritizing.
|
|
218
207
|
let checkAndFetch = async (
|
|
219
208
|
crossChainState: t,
|
|
220
209
|
~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
|
|
221
210
|
) => {
|
|
222
|
-
let remaining =
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
211
|
+
let remaining = ref(
|
|
212
|
+
Pervasives.max(
|
|
213
|
+
0.,
|
|
214
|
+
crossChainState.targetBufferSize->Int.toFloat -.
|
|
215
|
+
crossChainState->totalReadyCount->Int.toFloat -.
|
|
216
|
+
crossChainState->totalReservedSize,
|
|
217
|
+
),
|
|
227
218
|
)
|
|
228
219
|
|
|
229
|
-
|
|
220
|
+
// A chain with no density signal probes blind, so it only gets a bounded
|
|
221
|
+
// slice of the pool — one unknown chain shouldn't hold the whole budget
|
|
222
|
+
// while it takes its first measurements.
|
|
223
|
+
let coldChainBudget = Pervasives.min(5_000., crossChainState.targetBufferSize->Int.toFloat)
|
|
224
|
+
|
|
225
|
+
// Chunk reservations get headroom over the density estimate so a
|
|
226
|
+
// denser-than-expected range doesn't truncate at the server cap; realtime
|
|
227
|
+
// gets more since a forced catch-up query there costs a head-poll roundtrip.
|
|
228
|
+
let chunkItemsMultiplier = crossChainState.isRealtime ? 3. : 1.5
|
|
229
|
+
|
|
230
230
|
let actionByChain = Dict.make()
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
231
|
+
let maxProgress = ref(None)
|
|
232
|
+
crossChainState
|
|
233
|
+
->priorityOrder
|
|
234
|
+
->Array.forEach(cs => {
|
|
235
|
+
let chainId = (cs->ChainState.chainConfig).id
|
|
236
|
+
if cs->ChainState.knownHeight == 0 {
|
|
237
|
+
// No height yet — nothing to size a budget or an alignment line
|
|
238
|
+
// against. Skip without consuming budget or claiming leadership, so a
|
|
239
|
+
// chain whose source hasn't reported doesn't unconstrain everyone else.
|
|
240
|
+
actionByChain->Utils.Dict.setByInt(chainId, FetchState.WaitingForNewBlock)
|
|
241
|
+
} else {
|
|
242
|
+
let isCold = cs->ChainState.effectiveDensity === None
|
|
243
|
+
let chainTargetItems =
|
|
244
|
+
(isCold ? Pervasives.min(remaining.contents, coldChainBudget) : remaining.contents) +.
|
|
245
|
+
cs->ChainState.pendingBudget
|
|
246
|
+
let maxTargetBlock = switch maxProgress.contents {
|
|
247
|
+
| None if isCold =>
|
|
248
|
+
// A cold chain's target is a guess — it doesn't set the alignment
|
|
249
|
+
// line; the first density-bearing chain after it does.
|
|
250
|
+
None
|
|
251
|
+
| None =>
|
|
252
|
+
// The most-behind chain sets the alignment line for everyone after it.
|
|
253
|
+
maxProgress :=
|
|
254
|
+
Some(
|
|
255
|
+
cs->ChainState.progressAtBlock(
|
|
256
|
+
~blockNumber=cs->ChainState.targetBlock(~chainTargetItems),
|
|
257
|
+
),
|
|
258
|
+
)
|
|
259
|
+
None
|
|
260
|
+
// 5% margin past the leader's line: chains whose progress tracks the
|
|
261
|
+
// leader closely would otherwise flap in and out of the clamp as
|
|
262
|
+
// leadership alternates, stalling their pipeline every other tick.
|
|
263
|
+
| Some(progress) => Some(cs->ChainState.blockAtProgress(~progress=progress +. 0.05))
|
|
264
|
+
}
|
|
265
|
+
switch cs->ChainState.getNextQuery(
|
|
266
|
+
~chainTargetItems,
|
|
267
|
+
~chunkItemsMultiplier,
|
|
268
|
+
~maxTargetBlock?,
|
|
269
|
+
) {
|
|
270
|
+
| (WaitingForNewBlock | NothingToQuery) as action =>
|
|
271
|
+
actionByChain->Utils.Dict.setByInt(chainId, action)
|
|
272
|
+
| Ready(queries) => {
|
|
273
|
+
let consumed =
|
|
274
|
+
queries->Array.reduce(0., (acc, query: FetchState.query) =>
|
|
275
|
+
acc +. query.itemsEst->Int.toFloat
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
let partitions = Dict.make()
|
|
279
|
+
queries->Array.forEach((query: FetchState.query) =>
|
|
280
|
+
partitions->Dict.set(
|
|
281
|
+
query.partitionId,
|
|
282
|
+
{
|
|
283
|
+
"fromBlock": query.fromBlock,
|
|
284
|
+
"targetBlock": query.toBlock,
|
|
285
|
+
"targetEvents": query.itemsTarget,
|
|
286
|
+
},
|
|
287
|
+
)
|
|
288
|
+
)
|
|
289
|
+
Logging.trace({
|
|
290
|
+
"msg": "Started querying",
|
|
291
|
+
"chainId": chainId,
|
|
292
|
+
"partitions": partitions,
|
|
293
|
+
})
|
|
294
|
+
|
|
295
|
+
actionByChain->Utils.Dict.setByInt(chainId, FetchState.Ready(queries))
|
|
296
|
+
// Mark the queries in flight and reserve their size against the
|
|
297
|
+
// shared budget; released as each response lands in
|
|
298
|
+
// handleQueryResult.
|
|
299
|
+
cs->ChainState.startFetchingQueries(~queries)
|
|
300
|
+
remaining := Pervasives.max(0., remaining.contents -. consumed)
|
|
301
|
+
}
|
|
302
|
+
}
|
|
254
303
|
}
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
candidates->Array.sort(compareByProgress)
|
|
258
|
-
|
|
259
|
-
// Admit furthest-behind first until the budget runs out. The condition is
|
|
260
|
-
// checked before each query, so as long as there's any budget left we admit
|
|
261
|
-
// the next one even when its estimate alone exceeds the remainder — otherwise a
|
|
262
|
-
// chain whose only query is bigger than the budget would never make progress.
|
|
263
|
-
let admittedByChain = Dict.make()
|
|
264
|
-
let running = ref(0.)
|
|
265
|
-
let remainingF = remaining->Int.toFloat
|
|
266
|
-
let idx = ref(0)
|
|
267
|
-
while running.contents < remainingF && idx.contents < candidates->Array.length {
|
|
268
|
-
let query = candidates->Array.getUnsafe(idx.contents)
|
|
269
|
-
admittedByChain->Utils.Dict.push(query.chainId->Int.toString, query)
|
|
270
|
-
running := running.contents +. query.estResponseSize
|
|
271
|
-
idx := idx.contents + 1
|
|
272
|
-
}
|
|
273
|
-
admittedByChain->Dict.forEachWithKey((queries, chainId) => {
|
|
274
|
-
actionByChain->Dict.set(chainId, FetchState.Ready(queries))
|
|
275
|
-
// Mark the admitted queries in flight and reserve their size against the
|
|
276
|
-
// shared budget; released as each response lands in handleQueryResult.
|
|
277
|
-
crossChainState
|
|
278
|
-
->getChainState(chainId->Int.fromString->Option.getUnsafe)
|
|
279
|
-
->ChainState.startFetchingQueries(~queries)
|
|
280
304
|
})
|
|
281
305
|
|
|
282
306
|
let promises = []
|
|
283
|
-
for i in 0 to chainIds->Array.length - 1 {
|
|
284
|
-
let chainId = chainIds->Array.getUnsafe(i)
|
|
307
|
+
for i in 0 to crossChainState.chainIds->Array.length - 1 {
|
|
308
|
+
let chainId = crossChainState.chainIds->Array.getUnsafe(i)
|
|
285
309
|
switch actionByChain->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
|
|
286
310
|
| Some(NothingToQuery)
|
|
287
311
|
| None => ()
|
|
@@ -6,11 +6,8 @@ import * as Utils from "./Utils.res.mjs";
|
|
|
6
6
|
import * as Logging from "./Logging.res.mjs";
|
|
7
7
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
8
8
|
import * as ChainState from "./ChainState.res.mjs";
|
|
9
|
-
import * as FetchState from "./FetchState.res.mjs";
|
|
10
9
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
11
|
-
import * as
|
|
12
|
-
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
13
|
-
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
10
|
+
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
14
11
|
import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
|
|
15
12
|
import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
|
|
16
13
|
|
|
@@ -53,13 +50,13 @@ function totalReadyCount(crossChainState) {
|
|
|
53
50
|
let total = 0;
|
|
54
51
|
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
55
52
|
let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
|
|
56
|
-
total = total +
|
|
53
|
+
total = total + ChainState.bufferReadyCount(cs) | 0;
|
|
57
54
|
}
|
|
58
55
|
return total;
|
|
59
56
|
}
|
|
60
57
|
|
|
61
58
|
function nextItemIsNone(crossChainState) {
|
|
62
|
-
return !
|
|
59
|
+
return !Object.values(crossChainState.chainStates).some(ChainState.hasReadyItem);
|
|
63
60
|
}
|
|
64
61
|
|
|
65
62
|
function getSafeCheckpointId(crossChainState) {
|
|
@@ -68,7 +65,7 @@ function getSafeCheckpointId(crossChainState) {
|
|
|
68
65
|
let cs = crossChainState.chainStates[crossChainState.chainIds[i]];
|
|
69
66
|
let safeCheckpointTracking = ChainState.safeCheckpointTracking(cs);
|
|
70
67
|
if (safeCheckpointTracking !== undefined) {
|
|
71
|
-
let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.
|
|
68
|
+
let safeCheckpointId = SafeCheckpointTracking.getSafeCheckpointId(safeCheckpointTracking, ChainState.knownHeight(cs));
|
|
72
69
|
let current = result;
|
|
73
70
|
if (!(current !== undefined && safeCheckpointId >= current)) {
|
|
74
71
|
result = safeCheckpointId;
|
|
@@ -84,14 +81,7 @@ function getSafeCheckpointId(crossChainState) {
|
|
|
84
81
|
function createBatch(crossChainState, processedCheckpointId, batchSizeTarget, isRollback) {
|
|
85
82
|
return Batch.make(processedCheckpointId + (
|
|
86
83
|
isRollback ? 1n : 0n
|
|
87
|
-
), Utils.Dict.mapValues(crossChainState.chainStates,
|
|
88
|
-
fetchState: ChainState.fetchState(cs),
|
|
89
|
-
reorgDetection: ChainState.reorgDetection(cs),
|
|
90
|
-
progressBlockNumber: ChainState.committedProgressBlockNumber(cs),
|
|
91
|
-
sourceBlockNumber: ChainState.fetchState(cs).knownHeight,
|
|
92
|
-
totalEventsProcessed: ChainState.numEventsProcessed(cs),
|
|
93
|
-
chainConfig: ChainState.chainConfig(cs)
|
|
94
|
-
})), batchSizeTarget, crossChainState.isInReorgThreshold);
|
|
84
|
+
), Utils.Dict.mapValues(crossChainState.chainStates, ChainState.toChainBeforeBatch), batchSizeTarget, crossChainState.isInReorgThreshold);
|
|
95
85
|
}
|
|
96
86
|
|
|
97
87
|
function enterReorgThreshold(crossChainState) {
|
|
@@ -103,12 +93,12 @@ function enterReorgThreshold(crossChainState) {
|
|
|
103
93
|
crossChainState.isInReorgThreshold = true;
|
|
104
94
|
}
|
|
105
95
|
|
|
106
|
-
function applyBatchProgress(crossChainState, batch) {
|
|
96
|
+
function applyBatchProgress(crossChainState, batch, blockTimestampName) {
|
|
107
97
|
let chainIds = crossChainState.chainIds;
|
|
108
98
|
let everyChainCaughtUp = true;
|
|
109
99
|
for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
|
|
110
100
|
let cs = crossChainState.chainStates[chainIds[i]];
|
|
111
|
-
ChainState.applyBatchProgress(cs, batch);
|
|
101
|
+
ChainState.applyBatchProgress(cs, batch, blockTimestampName);
|
|
112
102
|
if (!(ChainState.hasProcessedToEndblock(cs) || ChainState.isProgressAtHead(cs))) {
|
|
113
103
|
everyChainCaughtUp = false;
|
|
114
104
|
}
|
|
@@ -131,7 +121,7 @@ function applyBatchProgress(crossChainState, batch) {
|
|
|
131
121
|
}
|
|
132
122
|
|
|
133
123
|
function priorityOrder(crossChainState) {
|
|
134
|
-
return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(
|
|
124
|
+
return Object.values(crossChainState.chainStates).toSorted((a, b) => Primitive_float.compare(ChainState.getProgressPercentage(a), ChainState.getProgressPercentage(b)));
|
|
135
125
|
}
|
|
136
126
|
|
|
137
127
|
function totalReservedSize(crossChainState) {
|
|
@@ -143,61 +133,75 @@ function totalReservedSize(crossChainState) {
|
|
|
143
133
|
return total;
|
|
144
134
|
}
|
|
145
135
|
|
|
146
|
-
function compareByProgress(a, b) {
|
|
147
|
-
return Primitive_float.compare(a.progress, b.progress);
|
|
148
|
-
}
|
|
149
|
-
|
|
150
136
|
async function checkAndFetch(crossChainState, dispatchChain) {
|
|
151
|
-
let remaining =
|
|
152
|
-
|
|
137
|
+
let remaining = {
|
|
138
|
+
contents: Primitive_float.max(0, crossChainState.targetBufferSize - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
|
|
139
|
+
};
|
|
140
|
+
let coldChainBudget = Primitive_float.min(5000, crossChainState.targetBufferSize);
|
|
141
|
+
let chunkItemsMultiplier = crossChainState.isRealtime ? 3 : 1.5;
|
|
153
142
|
let actionByChain = {};
|
|
154
|
-
let
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
let
|
|
159
|
-
|
|
143
|
+
let maxProgress = {
|
|
144
|
+
contents: undefined
|
|
145
|
+
};
|
|
146
|
+
priorityOrder(crossChainState).forEach(cs => {
|
|
147
|
+
let chainId = ChainState.chainConfig(cs).id;
|
|
148
|
+
if (ChainState.knownHeight(cs) === 0) {
|
|
149
|
+
actionByChain[chainId] = "WaitingForNewBlock";
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
let isCold = ChainState.effectiveDensity(cs) === undefined;
|
|
153
|
+
let chainTargetItems = (
|
|
154
|
+
isCold ? Primitive_float.min(remaining.contents, coldChainBudget) : remaining.contents
|
|
155
|
+
) + ChainState.pendingBudget(cs);
|
|
156
|
+
let progress = maxProgress.contents;
|
|
157
|
+
let maxTargetBlock = progress !== undefined ? ChainState.blockAtProgress(cs, progress + 0.05) : (
|
|
158
|
+
isCold ? undefined : (maxProgress.contents = ChainState.progressAtBlock(cs, ChainState.targetBlock(cs, chainTargetItems)), undefined)
|
|
159
|
+
);
|
|
160
|
+
let action = ChainState.getNextQuery(cs, chainTargetItems, chunkItemsMultiplier, maxTargetBlock);
|
|
160
161
|
if (typeof action !== "object") {
|
|
162
|
+
if (action === "WaitingForNewBlock") {
|
|
163
|
+
actionByChain[chainId] = action;
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
161
166
|
actionByChain[chainId] = action;
|
|
167
|
+
return;
|
|
162
168
|
} else {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
169
|
+
let queries = action._0;
|
|
170
|
+
let consumed = Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsEst);
|
|
171
|
+
let partitions = {};
|
|
172
|
+
queries.forEach(query => {
|
|
173
|
+
partitions[query.partitionId] = {
|
|
174
|
+
fromBlock: query.fromBlock,
|
|
175
|
+
targetBlock: query.toBlock,
|
|
176
|
+
targetEvents: query.itemsTarget
|
|
177
|
+
};
|
|
178
|
+
});
|
|
179
|
+
Logging.trace({
|
|
180
|
+
msg: "Started querying",
|
|
181
|
+
chainId: chainId,
|
|
182
|
+
partitions: partitions
|
|
168
183
|
});
|
|
184
|
+
actionByChain[chainId] = {
|
|
185
|
+
TAG: "Ready",
|
|
186
|
+
_0: queries
|
|
187
|
+
};
|
|
188
|
+
ChainState.startFetchingQueries(cs, queries);
|
|
189
|
+
remaining.contents = Primitive_float.max(0, remaining.contents - consumed);
|
|
190
|
+
return;
|
|
169
191
|
}
|
|
170
|
-
}
|
|
171
|
-
candidates.sort(compareByProgress);
|
|
172
|
-
let admittedByChain = {};
|
|
173
|
-
let running = 0;
|
|
174
|
-
let idx = 0;
|
|
175
|
-
while (running < remaining && idx < candidates.length) {
|
|
176
|
-
let query = candidates[idx];
|
|
177
|
-
Utils.Dict.push(admittedByChain, query.chainId.toString(), query);
|
|
178
|
-
running = running + query.estResponseSize;
|
|
179
|
-
idx = idx + 1 | 0;
|
|
180
|
-
};
|
|
181
|
-
Stdlib_Dict.forEachWithKey(admittedByChain, (queries, chainId) => {
|
|
182
|
-
actionByChain[chainId] = {
|
|
183
|
-
TAG: "Ready",
|
|
184
|
-
_0: queries
|
|
185
|
-
};
|
|
186
|
-
let chainId$1 = Stdlib_Int.fromString(chainId, undefined);
|
|
187
|
-
ChainState.startFetchingQueries(crossChainState.chainStates[chainId$1], queries);
|
|
188
192
|
});
|
|
189
193
|
let promises = [];
|
|
190
|
-
for (let i
|
|
191
|
-
let chainId
|
|
192
|
-
let action
|
|
193
|
-
if (action
|
|
194
|
+
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
195
|
+
let chainId = crossChainState.chainIds[i];
|
|
196
|
+
let action = actionByChain[chainId];
|
|
197
|
+
if (action !== undefined) {
|
|
194
198
|
let exit = 0;
|
|
195
|
-
if (typeof action
|
|
199
|
+
if (typeof action === "object" || action === "WaitingForNewBlock") {
|
|
196
200
|
exit = 1;
|
|
197
201
|
}
|
|
198
202
|
if (exit === 1) {
|
|
199
|
-
let chain = ChainMap.Chain.makeUnsafe(chainId
|
|
200
|
-
promises.push(dispatchChain(chain, action
|
|
203
|
+
let chain = ChainMap.Chain.makeUnsafe(chainId);
|
|
204
|
+
promises.push(dispatchChain(chain, action));
|
|
201
205
|
}
|
|
202
206
|
}
|
|
203
207
|
}
|
package/src/CrossChainState.resi
CHANGED
|
@@ -29,7 +29,7 @@ let createBatch: (
|
|
|
29
29
|
~isRollback: bool,
|
|
30
30
|
) => Batch.t
|
|
31
31
|
let enterReorgThreshold: t => unit
|
|
32
|
-
let applyBatchProgress: (t, ~batch: Batch.t) => unit
|
|
32
|
+
let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
|
|
33
33
|
|
|
34
34
|
// Fetch control.
|
|
35
35
|
let priorityOrder: t => array<ChainState.t>
|
package/src/Ecosystem.res
CHANGED
|
@@ -2,8 +2,6 @@ type name = | @as("evm") Evm | @as("fuel") Fuel | @as("svm") Svm
|
|
|
2
2
|
|
|
3
3
|
type t = {
|
|
4
4
|
name: name,
|
|
5
|
-
blockFields: array<string>,
|
|
6
|
-
transactionFields: array<string>,
|
|
7
5
|
blockNumberName: string,
|
|
8
6
|
blockTimestampName: string,
|
|
9
7
|
blockHashName: string,
|
|
@@ -33,7 +31,8 @@ type t = {
|
|
|
33
31
|
loggers (see `getItemLogger`). */
|
|
34
32
|
logger: Pino.t,
|
|
35
33
|
/** Materialise the user-facing event handed to handlers and contract
|
|
36
|
-
registration from an item's opaque payload.
|
|
34
|
+
registration from an item's opaque payload. `event.transaction` is written
|
|
35
|
+
onto the payload at batch prep (HyperSync) or inline (RPC/simulate). */
|
|
37
36
|
toEvent: Internal.eventItem => Internal.event,
|
|
38
37
|
/** Build the per-item child logger for an event item, with
|
|
39
38
|
ecosystem-specific log fields (EVM/Fuel: contract/event/address; SVM:
|
|
@@ -70,12 +69,12 @@ let getItemLogger = {
|
|
|
70
69
|
| None =>
|
|
71
70
|
let logger = switch item {
|
|
72
71
|
| Internal.Event(_) => ecosystem.toEventLogger(item->Internal.castUnsafeEventItem)
|
|
73
|
-
| Block({blockNumber,
|
|
72
|
+
| Block({blockNumber, onBlockRegistration}) =>
|
|
74
73
|
Logging.createChildFrom(
|
|
75
74
|
~logger=ecosystem.logger,
|
|
76
75
|
~params={
|
|
77
|
-
"onBlock":
|
|
78
|
-
"chainId":
|
|
76
|
+
"onBlock": onBlockRegistration.name,
|
|
77
|
+
"chainId": onBlockRegistration.chainId,
|
|
79
78
|
"block": blockNumber,
|
|
80
79
|
},
|
|
81
80
|
)
|
package/src/Ecosystem.res.mjs
CHANGED
|
@@ -26,10 +26,10 @@ function getItemLogger(item, ecosystem) {
|
|
|
26
26
|
if (item.kind === 0) {
|
|
27
27
|
logger$1 = ecosystem.toEventLogger(item);
|
|
28
28
|
} else {
|
|
29
|
-
let
|
|
29
|
+
let onBlockRegistration = item.onBlockRegistration;
|
|
30
30
|
logger$1 = Logging.createChildFrom(ecosystem.logger, {
|
|
31
|
-
onBlock:
|
|
32
|
-
chainId:
|
|
31
|
+
onBlock: onBlockRegistration.name,
|
|
32
|
+
chainId: onBlockRegistration.chainId,
|
|
33
33
|
block: item.blockNumber
|
|
34
34
|
});
|
|
35
35
|
}
|
package/src/Envio.res
CHANGED
|
@@ -48,6 +48,7 @@ type svmTokenBalance = {
|
|
|
48
48
|
}
|
|
49
49
|
|
|
50
50
|
type svmTransaction = {
|
|
51
|
+
transactionIndex?: int,
|
|
51
52
|
signatures: array<string>,
|
|
52
53
|
feePayer?: SvmTypes.Pubkey.t,
|
|
53
54
|
success?: bool,
|
|
@@ -65,18 +66,21 @@ type svmLog = {
|
|
|
65
66
|
message: string,
|
|
66
67
|
}
|
|
67
68
|
|
|
68
|
-
/** Block context for a matched instruction. `
|
|
69
|
-
|
|
70
|
-
|
|
69
|
+
/** Block context for a matched instruction. `slot`/`hash` are always present;
|
|
70
|
+
`time` can still be absent — HyperSync/Solana may not report a block time for
|
|
71
|
+
every slot. Every other field is opt-in via `field_selection.block_fields`,
|
|
72
|
+
materialised from the per-chain block store at batch prep. */
|
|
71
73
|
type svmInstructionBlock = {
|
|
72
74
|
/** Slot this instruction's block was matched in. */
|
|
73
75
|
slot: int,
|
|
74
|
-
/** Unix block time (seconds).
|
|
75
|
-
|
|
76
|
-
time: int,
|
|
77
|
-
/** Block hash. Currently always empty — populated by the future
|
|
78
|
-
reorg-guard `queryBlockHash(slot)` route. */
|
|
76
|
+
/** Unix block time (seconds). */
|
|
77
|
+
time?: int,
|
|
79
78
|
hash: string,
|
|
79
|
+
/** Block height (distinct from slot). Absent when not selected via
|
|
80
|
+
`field_selection.block_fields`. */
|
|
81
|
+
height?: int,
|
|
82
|
+
parentSlot?: int,
|
|
83
|
+
parentHash?: string,
|
|
80
84
|
}
|
|
81
85
|
|
|
82
86
|
/** The per-instruction payload handlers receive as their `instruction`
|
|
@@ -104,13 +108,15 @@ type svmInstruction = {
|
|
|
104
108
|
d8?: string,
|
|
105
109
|
/** Borsh-decoded params view. See [[svmInstructionParams]]. */
|
|
106
110
|
params?: svmInstructionParams,
|
|
107
|
-
/** Parent transaction.
|
|
108
|
-
`
|
|
111
|
+
/** Parent transaction. Carries only the fields selected via
|
|
112
|
+
`field_selection.transaction_fields`; absent when none are selected. */
|
|
109
113
|
transaction?: svmTransaction,
|
|
110
114
|
/** Program log entries scoped to this instruction. Absent when the
|
|
111
115
|
per-instruction `include_logs` flag is `false`. */
|
|
112
116
|
logs?: array<svmLog>,
|
|
113
|
-
|
|
117
|
+
// Omitted on construction; materialised onto the payload at batch prep,
|
|
118
|
+
// before a handler ever reads it.
|
|
119
|
+
block?: svmInstructionBlock,
|
|
114
120
|
}
|
|
115
121
|
|
|
116
122
|
/** Arguments passed to handlers registered via `indexer.onInstruction`. */
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// The single process-wide mutable state record, stashed on `globalThis` so a
|
|
2
|
+
// duplicate envio module instance — e.g. when the CLI's `bin.mjs` resolves
|
|
3
|
+
// envio from one path but the user's handlers resolve it from
|
|
4
|
+
// `node_modules/envio` — shares one state. Without this, each copy keeps its
|
|
5
|
+
// own module-level state: registration would read an empty registry and the
|
|
6
|
+
// `indexer.chains` getters would silently fall back to static config values.
|
|
7
|
+
//
|
|
8
|
+
// Slots are opaque (`unknown`) so this module stays at the bottom of the
|
|
9
|
+
// dependency graph; each owning module casts its slot to the real type once
|
|
10
|
+
// (`HandlerRegister` for the registration slots, `Main` for the runtime
|
|
11
|
+
// slots, `RollbackCommit` for its callbacks).
|
|
12
|
+
//
|
|
13
|
+
// Version-gated: the slot shapes can evolve between envio versions, so the
|
|
14
|
+
// guard uses strict full-version equality. On mismatch we throw with a
|
|
15
|
+
// deduplication hint instead of silently mixing shapes across builds.
|
|
16
|
+
type t = {
|
|
17
|
+
version: string,
|
|
18
|
+
eventRegistrations: dict<unknown>,
|
|
19
|
+
mutable activeRegistration: option<unknown>,
|
|
20
|
+
preRegistered: array<unknown>,
|
|
21
|
+
rollbackCommitCallbacks: array<unknown>,
|
|
22
|
+
mutable indexerState: option<unknown>,
|
|
23
|
+
mutable persistence: option<unknown>,
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Record type with `mutable` so assignment typechecks; ReScript keeps the
|
|
27
|
+
// field name verbatim in the generated JS so the globalThis slot is
|
|
28
|
+
// `__envioGlobal`.
|
|
29
|
+
type globalThis = {mutable __envioGlobal: Nullable.t<t>}
|
|
30
|
+
@val external globalThis: globalThis = "globalThis"
|
|
31
|
+
|
|
32
|
+
let value: t = {
|
|
33
|
+
let version = Utils.EnvioPackage.value.version
|
|
34
|
+
switch globalThis.__envioGlobal->Nullable.toOption {
|
|
35
|
+
| Some(existing) if existing.version === version => existing
|
|
36
|
+
| Some(existing) =>
|
|
37
|
+
JsError.throwWithMessage(
|
|
38
|
+
`Multiple incompatible envio versions loaded in the same process: ${existing.version} and ${version}. Deduplicate the 'envio' dependency in your project.`,
|
|
39
|
+
)
|
|
40
|
+
| None =>
|
|
41
|
+
let fresh = {
|
|
42
|
+
version,
|
|
43
|
+
eventRegistrations: Dict.make(),
|
|
44
|
+
activeRegistration: None,
|
|
45
|
+
preRegistered: [],
|
|
46
|
+
rollbackCommitCallbacks: [],
|
|
47
|
+
indexerState: None,
|
|
48
|
+
persistence: None,
|
|
49
|
+
}
|
|
50
|
+
globalThis.__envioGlobal = Nullable.make(fresh)
|
|
51
|
+
fresh
|
|
52
|
+
}
|
|
53
|
+
}
|