envio 3.3.0-alpha.8 → 3.3.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/ChainState.res +104 -37
- package/src/ChainState.res.mjs +80 -19
- package/src/ChainState.resi +4 -1
- package/src/CrossChainState.res +56 -29
- package/src/CrossChainState.res.mjs +11 -2
- package/src/EnvioGlobal.res +53 -0
- package/src/EnvioGlobal.res.mjs +31 -0
- package/src/EventConfigBuilder.res +16 -17
- package/src/EventConfigBuilder.res.mjs +7 -6
- package/src/FetchState.res +151 -130
- package/src/FetchState.res.mjs +105 -88
- package/src/HandlerLoader.res +1 -1
- package/src/HandlerLoader.res.mjs +1 -1
- package/src/HandlerRegister.res +476 -305
- package/src/HandlerRegister.res.mjs +276 -209
- package/src/HandlerRegister.resi +11 -6
- package/src/Internal.res +23 -4
- package/src/LogSelection.res +102 -165
- package/src/LogSelection.res.mjs +101 -116
- package/src/Main.res +29 -143
- package/src/Main.res.mjs +25 -88
- package/src/RollbackCommit.res +4 -1
- package/src/RollbackCommit.res.mjs +3 -2
- package/src/SimulateItems.res +1 -1
- package/src/sources/Evm.res +4 -1
- package/src/sources/Evm.res.mjs +1 -1
- package/src/sources/Fuel.res +3 -1
- package/src/sources/Fuel.res.mjs +1 -1
- package/src/sources/HyperSyncSource.res +22 -33
- package/src/sources/HyperSyncSource.res.mjs +17 -26
- package/src/sources/RpcSource.res +28 -39
- package/src/sources/RpcSource.res.mjs +18 -31
- package/src/sources/SourceManager.res +1 -7
- package/src/sources/SourceManager.res.mjs +1 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "3.3.0-alpha.
|
|
3
|
+
"version": "3.3.0-alpha.9",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.mjs",
|
|
@@ -69,10 +69,10 @@
|
|
|
69
69
|
"tsx": "4.21.0"
|
|
70
70
|
},
|
|
71
71
|
"optionalDependencies": {
|
|
72
|
-
"envio-linux-x64": "3.3.0-alpha.
|
|
73
|
-
"envio-linux-x64-musl": "3.3.0-alpha.
|
|
74
|
-
"envio-linux-arm64": "3.3.0-alpha.
|
|
75
|
-
"envio-darwin-x64": "3.3.0-alpha.
|
|
76
|
-
"envio-darwin-arm64": "3.3.0-alpha.
|
|
72
|
+
"envio-linux-x64": "3.3.0-alpha.9",
|
|
73
|
+
"envio-linux-x64-musl": "3.3.0-alpha.9",
|
|
74
|
+
"envio-linux-arm64": "3.3.0-alpha.9",
|
|
75
|
+
"envio-darwin-x64": "3.3.0-alpha.9",
|
|
76
|
+
"envio-darwin-arm64": "3.3.0-alpha.9"
|
|
77
77
|
}
|
|
78
78
|
}
|
package/src/ChainState.res
CHANGED
|
@@ -136,25 +136,6 @@ let makeInternal = (
|
|
|
136
136
|
}
|
|
137
137
|
})
|
|
138
138
|
|
|
139
|
-
// TODO: Move it to the HandlerRegister module
|
|
140
|
-
// so the error is thrown with better stack trace
|
|
141
|
-
onBlockRegistrations->Array.forEach(onBlockRegistration => {
|
|
142
|
-
if onBlockRegistration.startBlock->Option.getOr(startBlock) < startBlock {
|
|
143
|
-
JsError.throwWithMessage(
|
|
144
|
-
`The start block for onBlock handler "${onBlockRegistration.name}" is less than the chain start block (${startBlock->Int.toString}). This is not supported yet.`,
|
|
145
|
-
)
|
|
146
|
-
}
|
|
147
|
-
switch endBlock {
|
|
148
|
-
| Some(chainEndBlock) =>
|
|
149
|
-
if onBlockRegistration.endBlock->Option.getOr(chainEndBlock) > chainEndBlock {
|
|
150
|
-
JsError.throwWithMessage(
|
|
151
|
-
`The end block for onBlock handler "${onBlockRegistration.name}" is greater than the chain end block (${chainEndBlock->Int.toString}). This is not supported yet.`,
|
|
152
|
-
)
|
|
153
|
-
}
|
|
154
|
-
| None => ()
|
|
155
|
-
}
|
|
156
|
-
})
|
|
157
|
-
|
|
158
139
|
let contractConfigs = IndexingAddresses.makeContractConfigs(~onEventRegistrations)
|
|
159
140
|
let indexingAddressIndex = IndexingAddresses.make(~contractConfigs, ~addresses=indexingAddresses)
|
|
160
141
|
|
|
@@ -385,7 +366,8 @@ let isReadyToEnterReorgThreshold = (cs: t) => cs.fetchState->FetchState.isReadyT
|
|
|
385
366
|
let startFetchingQueries = (cs: t, ~queries: array<FetchState.query>) => {
|
|
386
367
|
cs.fetchState->FetchState.startFetchingQueries(~queries)
|
|
387
368
|
cs.pendingBudget =
|
|
388
|
-
cs.pendingBudget +.
|
|
369
|
+
cs.pendingBudget +.
|
|
370
|
+
queries->Array.reduce(0., (acc, query) => acc +. query.itemsTarget->Int.toFloat)
|
|
389
371
|
}
|
|
390
372
|
|
|
391
373
|
// Drop every in-flight query and release their reservations together, keeping
|
|
@@ -395,24 +377,108 @@ let resetPendingQueries = (cs: t) => {
|
|
|
395
377
|
cs.pendingBudget = 0.
|
|
396
378
|
}
|
|
397
379
|
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
//
|
|
401
|
-
//
|
|
402
|
-
//
|
|
403
|
-
let
|
|
380
|
+
let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
|
|
381
|
+
|
|
382
|
+
// Block span over which a batch fully replaces the chain density estimate;
|
|
383
|
+
// smaller batches blend in proportionally, so a few sparse/dense blocks only
|
|
384
|
+
// nudge it.
|
|
385
|
+
let densityBlendWindow = 100.
|
|
386
|
+
|
|
387
|
+
// The last block this chain can fetch right now: the head, or endBlock when
|
|
388
|
+
// it's below the head.
|
|
389
|
+
let fetchCeiling = (cs: t) => {
|
|
390
|
+
let fetchState = cs.fetchState
|
|
391
|
+
switch fetchState.endBlock {
|
|
392
|
+
| Some(endBlock) => Pervasives.min(endBlock, fetchState.knownHeight)
|
|
393
|
+
| None => fetchState.knownHeight
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// This chain's share of the indexer-wide buffer budget turned into a soft
|
|
398
|
+
// target block: via chain density, or the fetch ceiling when density isn't
|
|
399
|
+
// known yet.
|
|
400
|
+
let targetBlock = (cs: t, ~chainTargetItems: float) => {
|
|
404
401
|
let fetchState = cs.fetchState
|
|
405
|
-
let
|
|
402
|
+
let fetchCeiling = cs->fetchCeiling
|
|
406
403
|
let bufferBlockNumber = fetchState->FetchState.bufferBlockNumber
|
|
407
|
-
|
|
404
|
+
switch cs.chainDensity {
|
|
408
405
|
| Some(density) if density > 0. =>
|
|
409
406
|
Pervasives.min(
|
|
410
|
-
|
|
407
|
+
fetchCeiling,
|
|
411
408
|
bufferBlockNumber + Math.ceil(chainTargetItems /. density)->Float.toInt,
|
|
412
409
|
)
|
|
413
|
-
| _ =>
|
|
410
|
+
| _ => fetchCeiling
|
|
414
411
|
}
|
|
415
|
-
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
// Block range that cross-chain progress alignment maps fractions over: from
|
|
415
|
+
// the first block that can hold this chain's events to the last block it will
|
|
416
|
+
// fetch.
|
|
417
|
+
let progressRange = (cs: t) => {
|
|
418
|
+
let fetchState = cs.fetchState
|
|
419
|
+
let lower = fetchState.firstEventBlock->Option.getOr(fetchState.startBlock)
|
|
420
|
+
let upper = switch fetchState.endBlock {
|
|
421
|
+
| Some(endBlock) => endBlock
|
|
422
|
+
| None => fetchState.knownHeight
|
|
423
|
+
}
|
|
424
|
+
(lower, upper)
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
// A degenerate range (chain already at or past its last block) maps to 1 so it
|
|
428
|
+
// never constrains the other chains. Clamped at 0 for the initial -1 fetch
|
|
429
|
+
// frontier — the only possible blockNumber below the range's lower bound.
|
|
430
|
+
let progressAtBlock = (cs: t, ~blockNumber) => {
|
|
431
|
+
let (lower, upper) = cs->progressRange
|
|
432
|
+
upper <= lower
|
|
433
|
+
? 1.
|
|
434
|
+
: Pervasives.max(
|
|
435
|
+
0.,
|
|
436
|
+
Pervasives.min(1., (blockNumber - lower)->Int.toFloat /. (upper - lower)->Int.toFloat),
|
|
437
|
+
)
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
let blockAtProgress = (cs: t, ~progress) => {
|
|
441
|
+
let (lower, upper) = cs->progressRange
|
|
442
|
+
lower + Math.ceil(progress *. (upper - lower)->Int.toFloat)->Float.toInt
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
// Propose queries sized against this chain's target block. Called by
|
|
446
|
+
// CrossChainState's waterfall, furthest-behind chain first, with
|
|
447
|
+
// chainTargetItems set to whatever budget remains at that point and
|
|
448
|
+
// maxTargetBlock set to the most-behind chain's progress mapped onto this
|
|
449
|
+
// chain, so a chain with budget can't run further ahead than the chain the
|
|
450
|
+
// whole pool is prioritizing.
|
|
451
|
+
let getNextQuery = (cs: t, ~chainTargetItems: float, ~maxTargetBlock=?) => {
|
|
452
|
+
let chainTargetBlock = cs->targetBlock(~chainTargetItems)
|
|
453
|
+
let chainTargetBlock = switch maxTargetBlock {
|
|
454
|
+
| Some(maxTargetBlock) => Pervasives.min(chainTargetBlock, maxTargetBlock)
|
|
455
|
+
| None => chainTargetBlock
|
|
456
|
+
}
|
|
457
|
+
// When the target block is clamped (head/endBlock/cross-chain alignment) a
|
|
458
|
+
// known-density chain can't use the whole handed budget — cap the fresh part
|
|
459
|
+
// at what the clamped range actually costs (in-flight reservations stay on
|
|
460
|
+
// top: they're already accounted and shouldn't crowd out new partitions), so
|
|
461
|
+
// the waterfall's leftover flows to the next chain in the same tick instead
|
|
462
|
+
// of being held by an oversized probe.
|
|
463
|
+
let chainTargetItems = switch cs.chainDensity {
|
|
464
|
+
| Some(density) if density > 0. =>
|
|
465
|
+
let rangeCost =
|
|
466
|
+
density *. (chainTargetBlock - cs.fetchState->FetchState.bufferBlockNumber)->Int.toFloat
|
|
467
|
+
// 3x headroom only for a chain already caught up once and polling the
|
|
468
|
+
// head: there a single query covers the whole remaining range, and a
|
|
469
|
+
// slightly denser-than-expected range would otherwise truncate at the
|
|
470
|
+
// server cap and force an immediate catch-up query for the last blocks.
|
|
471
|
+
// During backfill the range never fits one query anyway, so headroom
|
|
472
|
+
// would just hold budget away from other chains.
|
|
473
|
+
let rangeCost =
|
|
474
|
+
chainTargetBlock >= cs->fetchCeiling && cs->isReady ? rangeCost *. 3. : rangeCost
|
|
475
|
+
Pervasives.min(chainTargetItems, Math.ceil(rangeCost) +. cs.pendingBudget)
|
|
476
|
+
| _ =>
|
|
477
|
+
// No density signal yet: bound the probe so one unknown chain doesn't
|
|
478
|
+
// hold the whole pool while it takes its first measurements.
|
|
479
|
+
Pervasives.min(chainTargetItems, 5_000. +. cs.pendingBudget)
|
|
480
|
+
}
|
|
481
|
+
cs.fetchState->FetchState.getNextQuery(~chainTargetBlock, ~chainTargetItems)
|
|
416
482
|
}
|
|
417
483
|
|
|
418
484
|
// Run a fetch tick for this chain against its sources, feeding the owned fetch
|
|
@@ -451,8 +517,6 @@ let getHighestBlockBelowThreshold = (cs: t): int => {
|
|
|
451
517
|
|
|
452
518
|
let isActivelyIndexing = (cs: t) => cs.fetchState->FetchState.isActivelyIndexing
|
|
453
519
|
|
|
454
|
-
let isReady = (cs: t) => cs.timestampCaughtUpToHeadOrEndblock !== None
|
|
455
|
-
|
|
456
520
|
// True once the fetch frontier has reached the head/endBlock for this chain.
|
|
457
521
|
let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
|
|
458
522
|
|
|
@@ -689,7 +753,7 @@ let handleQueryResult = (
|
|
|
689
753
|
->FetchState.updateKnownHeight(~knownHeight)
|
|
690
754
|
|
|
691
755
|
// The query is no longer in flight, so release its reservation.
|
|
692
|
-
cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsTarget)
|
|
756
|
+
cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsTarget->Int.toFloat)
|
|
693
757
|
}
|
|
694
758
|
|
|
695
759
|
// Run reorg detection against a fetch response and commit the updated guard.
|
|
@@ -852,8 +916,9 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
|
|
|
852
916
|
}
|
|
853
917
|
|
|
854
918
|
// Chain-wide density update: seed with the batch's own events/block on
|
|
855
|
-
// the first update, then
|
|
856
|
-
// sparse/dense blocks
|
|
919
|
+
// the first update, then blend weighted by the batch's block span — a
|
|
920
|
+
// few sparse/dense blocks barely nudge the estimate, while a
|
|
921
|
+
// window-sized batch replaces it.
|
|
857
922
|
let deltaBlocks = chainAfterBatch.progressBlockNumber - cs.committedProgressBlockNumber
|
|
858
923
|
if deltaBlocks > 0 {
|
|
859
924
|
let deltaEvents = chainAfterBatch.totalEventsProcessed -. cs.numEventsProcessed
|
|
@@ -866,7 +931,9 @@ let applyBatchProgress = (cs: t, ~batch: Batch.t, ~blockTimestampName: string) =
|
|
|
866
931
|
cs.chainDensity = Some(
|
|
867
932
|
switch cs.chainDensity {
|
|
868
933
|
| None => batchDensity
|
|
869
|
-
| Some(oldDensity) =>
|
|
934
|
+
| Some(oldDensity) =>
|
|
935
|
+
let alpha = Pervasives.min(1., deltaBlocks->Int.toFloat /. densityBlendWindow)
|
|
936
|
+
oldDensity *. (1. -. alpha) +. batchDensity *. alpha
|
|
870
937
|
},
|
|
871
938
|
)
|
|
872
939
|
}
|
package/src/ChainState.res.mjs
CHANGED
|
@@ -74,7 +74,6 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
74
74
|
onEventRegistrations: [],
|
|
75
75
|
onBlockRegistrations: []
|
|
76
76
|
});
|
|
77
|
-
let onBlockRegistrations = match.onBlockRegistrations;
|
|
78
77
|
let onEventRegistrations = match.onEventRegistrations;
|
|
79
78
|
chainConfig.contracts.forEach(contract => {
|
|
80
79
|
let startBlock = contract.startBlock;
|
|
@@ -82,17 +81,9 @@ function makeInternal(chainConfig, indexingAddresses, startBlock, endBlock, firs
|
|
|
82
81
|
return Stdlib_JsError.throwWithMessage(`The start block for contract "` + contract.name + `" is less than the chain start block. This is not supported yet.`);
|
|
83
82
|
}
|
|
84
83
|
});
|
|
85
|
-
onBlockRegistrations.forEach(onBlockRegistration => {
|
|
86
|
-
if (Stdlib_Option.getOr(onBlockRegistration.startBlock, startBlock) < startBlock) {
|
|
87
|
-
Stdlib_JsError.throwWithMessage(`The start block for onBlock handler "` + onBlockRegistration.name + `" is less than the chain start block (` + startBlock.toString() + `). This is not supported yet.`);
|
|
88
|
-
}
|
|
89
|
-
if (endBlock !== undefined && Stdlib_Option.getOr(onBlockRegistration.endBlock, endBlock) > endBlock) {
|
|
90
|
-
return Stdlib_JsError.throwWithMessage(`The end block for onBlock handler "` + onBlockRegistration.name + `" is greater than the chain end block (` + endBlock.toString() + `). This is not supported yet.`);
|
|
91
|
-
}
|
|
92
|
-
});
|
|
93
84
|
let contractConfigs = IndexingAddresses.makeContractConfigs(onEventRegistrations);
|
|
94
85
|
let indexingAddressIndex = IndexingAddresses.make(contractConfigs, indexingAddresses);
|
|
95
|
-
let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
|
|
86
|
+
let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, contractConfigs, indexingAddresses, config.maxAddrInPartition, chainConfig.id, (config.batchSize << 1), knownHeight, progressBlockNumber, match.onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock));
|
|
96
87
|
let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
|
|
97
88
|
if (reorgCheckpoint.chain_id === chainConfig.id) {
|
|
98
89
|
return reorgCheckpoint;
|
|
@@ -245,13 +236,77 @@ function startFetchingQueries(cs, queries) {
|
|
|
245
236
|
cs.pendingBudget = cs.pendingBudget + Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsTarget);
|
|
246
237
|
}
|
|
247
238
|
|
|
248
|
-
function
|
|
239
|
+
function isReady(cs) {
|
|
240
|
+
return cs.timestampCaughtUpToHeadOrEndblock !== undefined;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function fetchCeiling(cs) {
|
|
249
244
|
let fetchState = cs.fetchState;
|
|
250
|
-
let
|
|
245
|
+
let endBlock = fetchState.endBlock;
|
|
246
|
+
if (endBlock !== undefined) {
|
|
247
|
+
return Primitive_int.min(endBlock, fetchState.knownHeight);
|
|
248
|
+
} else {
|
|
249
|
+
return fetchState.knownHeight;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function targetBlock(cs, chainTargetItems) {
|
|
254
|
+
let fetchState = cs.fetchState;
|
|
255
|
+
let fetchCeiling$1 = fetchCeiling(cs);
|
|
251
256
|
let bufferBlockNumber = FetchState.bufferBlockNumber(fetchState);
|
|
252
257
|
let density = cs.chainDensity;
|
|
253
|
-
|
|
254
|
-
|
|
258
|
+
if (density !== undefined && density > 0) {
|
|
259
|
+
return Primitive_int.min(fetchCeiling$1, bufferBlockNumber + (Math.ceil(chainTargetItems / density) | 0) | 0);
|
|
260
|
+
} else {
|
|
261
|
+
return fetchCeiling$1;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
function progressRange(cs) {
|
|
266
|
+
let fetchState = cs.fetchState;
|
|
267
|
+
let lower = Stdlib_Option.getOr(fetchState.firstEventBlock, fetchState.startBlock);
|
|
268
|
+
let endBlock = fetchState.endBlock;
|
|
269
|
+
let upper = endBlock !== undefined ? endBlock : fetchState.knownHeight;
|
|
270
|
+
return [
|
|
271
|
+
lower,
|
|
272
|
+
upper
|
|
273
|
+
];
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
function progressAtBlock(cs, blockNumber) {
|
|
277
|
+
let match = progressRange(cs);
|
|
278
|
+
let upper = match[1];
|
|
279
|
+
let lower = match[0];
|
|
280
|
+
if (upper <= lower) {
|
|
281
|
+
return 1;
|
|
282
|
+
} else {
|
|
283
|
+
return Primitive_float.max(0, Primitive_float.min(1, (blockNumber - lower | 0) / (upper - lower | 0)));
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function blockAtProgress(cs, progress) {
|
|
288
|
+
let match = progressRange(cs);
|
|
289
|
+
let lower = match[0];
|
|
290
|
+
return lower + (Math.ceil(progress * (match[1] - lower | 0)) | 0) | 0;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function getNextQuery(cs, chainTargetItems, maxTargetBlock) {
|
|
294
|
+
let chainTargetBlock = targetBlock(cs, chainTargetItems);
|
|
295
|
+
let chainTargetBlock$1 = maxTargetBlock !== undefined ? Primitive_int.min(chainTargetBlock, maxTargetBlock) : chainTargetBlock;
|
|
296
|
+
let density = cs.chainDensity;
|
|
297
|
+
let chainTargetItems$1;
|
|
298
|
+
if (density !== undefined) {
|
|
299
|
+
if (density > 0) {
|
|
300
|
+
let rangeCost = density * (chainTargetBlock$1 - FetchState.bufferBlockNumber(cs.fetchState) | 0);
|
|
301
|
+
let rangeCost$1 = chainTargetBlock$1 >= fetchCeiling(cs) && cs.timestampCaughtUpToHeadOrEndblock !== undefined ? rangeCost * 3 : rangeCost;
|
|
302
|
+
chainTargetItems$1 = Primitive_float.min(chainTargetItems, Math.ceil(rangeCost$1) + cs.pendingBudget);
|
|
303
|
+
} else {
|
|
304
|
+
chainTargetItems$1 = Primitive_float.min(chainTargetItems, 5000 + cs.pendingBudget);
|
|
305
|
+
}
|
|
306
|
+
} else {
|
|
307
|
+
chainTargetItems$1 = Primitive_float.min(chainTargetItems, 5000 + cs.pendingBudget);
|
|
308
|
+
}
|
|
309
|
+
return FetchState.getNextQuery(cs.fetchState, chainTargetBlock$1, chainTargetItems$1);
|
|
255
310
|
}
|
|
256
311
|
|
|
257
312
|
function dispatch(cs, executeQuery, waitForNewBlock, onNewBlock, action, stateId) {
|
|
@@ -282,10 +337,6 @@ function isActivelyIndexing(cs) {
|
|
|
282
337
|
return FetchState.isActivelyIndexing(cs.fetchState);
|
|
283
338
|
}
|
|
284
339
|
|
|
285
|
-
function isReady(cs) {
|
|
286
|
-
return cs.timestampCaughtUpToHeadOrEndblock !== undefined;
|
|
287
|
-
}
|
|
288
|
-
|
|
289
340
|
function isFetchingAtHead(cs) {
|
|
290
341
|
return FetchState.isFetchingAtHead(cs.fetchState);
|
|
291
342
|
}
|
|
@@ -603,7 +654,14 @@ function applyBatchProgress(cs, batch, blockTimestampName) {
|
|
|
603
654
|
if (exit === 1) {
|
|
604
655
|
let batchDensity = deltaEvents / deltaBlocks;
|
|
605
656
|
let oldDensity = cs.chainDensity;
|
|
606
|
-
|
|
657
|
+
let tmp;
|
|
658
|
+
if (oldDensity !== undefined) {
|
|
659
|
+
let alpha = Primitive_float.min(1, deltaBlocks / 100);
|
|
660
|
+
tmp = oldDensity * (1 - alpha) + batchDensity * alpha;
|
|
661
|
+
} else {
|
|
662
|
+
tmp = batchDensity;
|
|
663
|
+
}
|
|
664
|
+
cs.chainDensity = tmp;
|
|
607
665
|
}
|
|
608
666
|
}
|
|
609
667
|
cs.committedProgressBlockNumber = chainAfterBatch.progressBlockNumber;
|
|
@@ -684,6 +742,9 @@ export {
|
|
|
684
742
|
chainDensity,
|
|
685
743
|
hasReadyItem,
|
|
686
744
|
isReadyToEnterReorgThreshold,
|
|
745
|
+
targetBlock,
|
|
746
|
+
progressAtBlock,
|
|
747
|
+
blockAtProgress,
|
|
687
748
|
getNextQuery,
|
|
688
749
|
dispatch,
|
|
689
750
|
toChainData,
|
package/src/ChainState.resi
CHANGED
|
@@ -64,7 +64,10 @@ let hasReadyItem: t => bool
|
|
|
64
64
|
let isReadyToEnterReorgThreshold: t => bool
|
|
65
65
|
|
|
66
66
|
// Fetch control.
|
|
67
|
-
let
|
|
67
|
+
let targetBlock: (t, ~chainTargetItems: float) => int
|
|
68
|
+
let progressAtBlock: (t, ~blockNumber: int) => float
|
|
69
|
+
let blockAtProgress: (t, ~progress: float) => int
|
|
70
|
+
let getNextQuery: (t, ~chainTargetItems: float, ~maxTargetBlock: int=?) => FetchState.nextQuery
|
|
68
71
|
let dispatch: (
|
|
69
72
|
t,
|
|
70
73
|
~executeQuery: FetchState.query => promise<unit>,
|
package/src/CrossChainState.res
CHANGED
|
@@ -198,8 +198,12 @@ let totalReservedSize = (crossChainState: t) => {
|
|
|
198
198
|
// chain-density-derived target block, then subtract what it actually used
|
|
199
199
|
// before moving to the next chain. So a chain that can only use a little
|
|
200
200
|
// (density too low, or already caught up) leaves the rest for the others
|
|
201
|
-
// automatically
|
|
202
|
-
//
|
|
201
|
+
// automatically. A chain visited after the budget ran out simply doesn't
|
|
202
|
+
// query this round — the leader's reservations release as its responses land,
|
|
203
|
+
// so the next tick redistributes. Chains that do get budget are additionally
|
|
204
|
+
// capped at the most-behind chain's target progress mapped onto their own
|
|
205
|
+
// range, so no chain runs further ahead than the chain the pool is
|
|
206
|
+
// prioritizing.
|
|
203
207
|
let checkAndFetch = async (
|
|
204
208
|
crossChainState: t,
|
|
205
209
|
~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
|
|
@@ -214,40 +218,63 @@ let checkAndFetch = async (
|
|
|
214
218
|
)
|
|
215
219
|
|
|
216
220
|
let actionByChain = Dict.make()
|
|
221
|
+
let maxProgress = ref(None)
|
|
217
222
|
crossChainState
|
|
218
223
|
->priorityOrder
|
|
219
224
|
->Array.forEach(cs => {
|
|
220
225
|
let chainId = (cs->ChainState.chainConfig).id
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
226
|
+
if cs->ChainState.knownHeight == 0 {
|
|
227
|
+
// No height yet — nothing to size a budget or an alignment line
|
|
228
|
+
// against. Skip without consuming budget or claiming leadership, so a
|
|
229
|
+
// chain whose source hasn't reported doesn't unconstrain everyone else.
|
|
230
|
+
actionByChain->Utils.Dict.setByInt(chainId, FetchState.WaitingForNewBlock)
|
|
231
|
+
} else {
|
|
232
|
+
let chainTargetItems = remaining.contents +. cs->ChainState.pendingBudget
|
|
233
|
+
let maxTargetBlock = switch maxProgress.contents {
|
|
234
|
+
| None =>
|
|
235
|
+
// The most-behind chain sets the alignment line for everyone after it.
|
|
236
|
+
maxProgress :=
|
|
237
|
+
Some(
|
|
238
|
+
cs->ChainState.progressAtBlock(
|
|
239
|
+
~blockNumber=cs->ChainState.targetBlock(~chainTargetItems),
|
|
240
|
+
),
|
|
241
|
+
)
|
|
242
|
+
None
|
|
243
|
+
| Some(progress) => Some(cs->ChainState.blockAtProgress(~progress))
|
|
244
|
+
}
|
|
245
|
+
switch cs->ChainState.getNextQuery(~chainTargetItems, ~maxTargetBlock?) {
|
|
246
|
+
| (WaitingForNewBlock | NothingToQuery) as action =>
|
|
247
|
+
actionByChain->Utils.Dict.setByInt(chainId, action)
|
|
248
|
+
| Ready(queries) => {
|
|
249
|
+
let consumed =
|
|
250
|
+
queries->Array.reduce(0., (acc, query: FetchState.query) =>
|
|
251
|
+
acc +. query.itemsTarget->Int.toFloat
|
|
252
|
+
)
|
|
228
253
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
254
|
+
let partitions = Dict.make()
|
|
255
|
+
queries->Array.forEach((query: FetchState.query) =>
|
|
256
|
+
partitions->Dict.set(
|
|
257
|
+
query.partitionId,
|
|
258
|
+
{
|
|
259
|
+
"fromBlock": query.fromBlock,
|
|
260
|
+
"targetBlock": query.toBlock,
|
|
261
|
+
"targetEvents": query.itemsTarget,
|
|
262
|
+
},
|
|
263
|
+
)
|
|
238
264
|
)
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
})
|
|
265
|
+
Logging.trace({
|
|
266
|
+
"msg": "Started querying",
|
|
267
|
+
"chainId": chainId,
|
|
268
|
+
"partitions": partitions,
|
|
269
|
+
})
|
|
245
270
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
271
|
+
actionByChain->Utils.Dict.setByInt(chainId, FetchState.Ready(queries))
|
|
272
|
+
// Mark the queries in flight and reserve their size against the
|
|
273
|
+
// shared budget; released as each response lands in
|
|
274
|
+
// handleQueryResult.
|
|
275
|
+
cs->ChainState.startFetchingQueries(~queries)
|
|
276
|
+
remaining := Pervasives.max(0., remaining.contents -. consumed)
|
|
277
|
+
}
|
|
251
278
|
}
|
|
252
279
|
}
|
|
253
280
|
})
|
|
@@ -138,10 +138,19 @@ async function checkAndFetch(crossChainState, dispatchChain) {
|
|
|
138
138
|
contents: Primitive_float.max(0, crossChainState.targetBufferSize - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
|
|
139
139
|
};
|
|
140
140
|
let actionByChain = {};
|
|
141
|
+
let maxProgress = {
|
|
142
|
+
contents: undefined
|
|
143
|
+
};
|
|
141
144
|
priorityOrder(crossChainState).forEach(cs => {
|
|
142
145
|
let chainId = ChainState.chainConfig(cs).id;
|
|
146
|
+
if (ChainState.knownHeight(cs) === 0) {
|
|
147
|
+
actionByChain[chainId] = "WaitingForNewBlock";
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
143
150
|
let chainTargetItems = remaining.contents + ChainState.pendingBudget(cs);
|
|
144
|
-
let
|
|
151
|
+
let progress = maxProgress.contents;
|
|
152
|
+
let maxTargetBlock = progress !== undefined ? ChainState.blockAtProgress(cs, progress) : (maxProgress.contents = ChainState.progressAtBlock(cs, ChainState.targetBlock(cs, chainTargetItems)), undefined);
|
|
153
|
+
let action = ChainState.getNextQuery(cs, chainTargetItems, maxTargetBlock);
|
|
145
154
|
if (typeof action !== "object") {
|
|
146
155
|
if (action === "WaitingForNewBlock") {
|
|
147
156
|
actionByChain[chainId] = action;
|
|
@@ -157,7 +166,7 @@ async function checkAndFetch(crossChainState, dispatchChain) {
|
|
|
157
166
|
partitions[query.partitionId] = {
|
|
158
167
|
fromBlock: query.fromBlock,
|
|
159
168
|
targetBlock: query.toBlock,
|
|
160
|
-
targetEvents:
|
|
169
|
+
targetEvents: query.itemsTarget
|
|
161
170
|
};
|
|
162
171
|
});
|
|
163
172
|
Logging.trace({
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// The single process-wide mutable state record, stashed on `globalThis` so a
|
|
2
|
+
// duplicate envio module instance — e.g. when the CLI's `bin.mjs` resolves
|
|
3
|
+
// envio from one path but the user's handlers resolve it from
|
|
4
|
+
// `node_modules/envio` — shares one state. Without this, each copy keeps its
|
|
5
|
+
// own module-level state: registration would read an empty registry and the
|
|
6
|
+
// `indexer.chains` getters would silently fall back to static config values.
|
|
7
|
+
//
|
|
8
|
+
// Slots are opaque (`unknown`) so this module stays at the bottom of the
|
|
9
|
+
// dependency graph; each owning module casts its slot to the real type once
|
|
10
|
+
// (`HandlerRegister` for the registration slots, `Main` for the runtime
|
|
11
|
+
// slots, `RollbackCommit` for its callbacks).
|
|
12
|
+
//
|
|
13
|
+
// Version-gated: the slot shapes can evolve between envio versions, so the
|
|
14
|
+
// guard uses strict full-version equality. On mismatch we throw with a
|
|
15
|
+
// deduplication hint instead of silently mixing shapes across builds.
|
|
16
|
+
type t = {
|
|
17
|
+
version: string,
|
|
18
|
+
eventRegistrations: dict<unknown>,
|
|
19
|
+
mutable activeRegistration: option<unknown>,
|
|
20
|
+
preRegistered: array<unknown>,
|
|
21
|
+
rollbackCommitCallbacks: array<unknown>,
|
|
22
|
+
mutable indexerState: option<unknown>,
|
|
23
|
+
mutable persistence: option<unknown>,
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// Record type with `mutable` so assignment typechecks; ReScript keeps the
|
|
27
|
+
// field name verbatim in the generated JS so the globalThis slot is
|
|
28
|
+
// `__envioGlobal`.
|
|
29
|
+
type globalThis = {mutable __envioGlobal: Nullable.t<t>}
|
|
30
|
+
@val external globalThis: globalThis = "globalThis"
|
|
31
|
+
|
|
32
|
+
let value: t = {
|
|
33
|
+
let version = Utils.EnvioPackage.value.version
|
|
34
|
+
switch globalThis.__envioGlobal->Nullable.toOption {
|
|
35
|
+
| Some(existing) if existing.version === version => existing
|
|
36
|
+
| Some(existing) =>
|
|
37
|
+
JsError.throwWithMessage(
|
|
38
|
+
`Multiple incompatible envio versions loaded in the same process: ${existing.version} and ${version}. Deduplicate the 'envio' dependency in your project.`,
|
|
39
|
+
)
|
|
40
|
+
| None =>
|
|
41
|
+
let fresh = {
|
|
42
|
+
version,
|
|
43
|
+
eventRegistrations: Dict.make(),
|
|
44
|
+
activeRegistration: None,
|
|
45
|
+
preRegistered: [],
|
|
46
|
+
rollbackCommitCallbacks: [],
|
|
47
|
+
indexerState: None,
|
|
48
|
+
persistence: None,
|
|
49
|
+
}
|
|
50
|
+
globalThis.__envioGlobal = Nullable.make(fresh)
|
|
51
|
+
fresh
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Utils from "./Utils.res.mjs";
|
|
4
|
+
import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
|
|
5
|
+
|
|
6
|
+
let version = Utils.EnvioPackage.value.version;
|
|
7
|
+
|
|
8
|
+
let existing = globalThis.__envioGlobal;
|
|
9
|
+
|
|
10
|
+
let value;
|
|
11
|
+
|
|
12
|
+
if (existing == null) {
|
|
13
|
+
let fresh = {
|
|
14
|
+
version: version,
|
|
15
|
+
eventRegistrations: {},
|
|
16
|
+
activeRegistration: undefined,
|
|
17
|
+
preRegistered: [],
|
|
18
|
+
rollbackCommitCallbacks: [],
|
|
19
|
+
indexerState: undefined,
|
|
20
|
+
persistence: undefined
|
|
21
|
+
};
|
|
22
|
+
globalThis.__envioGlobal = fresh;
|
|
23
|
+
value = fresh;
|
|
24
|
+
} else {
|
|
25
|
+
value = existing.version === version ? existing : Stdlib_JsError.throwWithMessage(`Multiple incompatible envio versions loaded in the same process: ` + existing.version + ` and ` + version + `. Deduplicate the 'envio' dependency in your project.`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export {
|
|
29
|
+
value,
|
|
30
|
+
}
|
|
31
|
+
/* existing Not a pure module */
|