envio 3.3.0-alpha.7 → 3.3.0-alpha.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +23 -5
- package/package.json +6 -6
- package/src/Api.res +1 -1
- package/src/Api.res.mjs +1 -1
- package/src/ChainFetching.res +21 -10
- package/src/ChainFetching.res.mjs +11 -10
- package/src/ChainState.res +401 -171
- package/src/ChainState.res.mjs +270 -104
- package/src/ChainState.resi +19 -7
- package/src/Config.res +11 -27
- package/src/Config.res.mjs +8 -7
- package/src/Core.res +7 -0
- package/src/CrossChainState.res +82 -82
- package/src/CrossChainState.res.mjs +52 -59
- package/src/CrossChainState.resi +1 -1
- package/src/Ecosystem.res +3 -3
- package/src/Ecosystem.res.mjs +3 -3
- package/src/Envio.res +14 -9
- package/src/EnvioGlobal.res +53 -0
- package/src/EnvioGlobal.res.mjs +31 -0
- package/src/EventConfigBuilder.res +117 -69
- package/src/EventConfigBuilder.res.mjs +71 -34
- package/src/EventProcessing.res +19 -15
- package/src/EventProcessing.res.mjs +13 -12
- package/src/FetchState.res +410 -185
- package/src/FetchState.res.mjs +268 -253
- package/src/HandlerLoader.res +8 -113
- package/src/HandlerLoader.res.mjs +2 -88
- package/src/HandlerRegister.res +518 -144
- package/src/HandlerRegister.res.mjs +285 -133
- package/src/HandlerRegister.resi +24 -8
- package/src/IndexerState.res +6 -3
- package/src/IndexerState.res.mjs +3 -3
- package/src/IndexerState.resi +1 -1
- package/src/IndexingAddresses.res +10 -7
- package/src/IndexingAddresses.res.mjs +8 -7
- package/src/IndexingAddresses.resi +3 -1
- package/src/Internal.res +104 -39
- package/src/Internal.res.mjs +11 -1
- package/src/LogSelection.res +102 -165
- package/src/LogSelection.res.mjs +101 -116
- package/src/Main.res +49 -164
- package/src/Main.res.mjs +39 -104
- package/src/Metrics.res +43 -2
- package/src/Metrics.res.mjs +39 -3
- package/src/Prometheus.res +0 -35
- package/src/Prometheus.res.mjs +33 -68
- package/src/RawEvent.res +7 -2
- package/src/RawEvent.res.mjs +4 -4
- package/src/RollbackCommit.res +4 -1
- package/src/RollbackCommit.res.mjs +3 -2
- package/src/SimulateItems.res +20 -7
- package/src/SimulateItems.res.mjs +7 -11
- package/src/TestIndexer.res +1 -1
- package/src/TestIndexer.res.mjs +1 -1
- package/src/sources/BlockStore.res +46 -0
- package/src/sources/BlockStore.res.mjs +24 -0
- package/src/sources/EventRouter.res +19 -12
- package/src/sources/EventRouter.res.mjs +7 -5
- package/src/sources/Evm.res +58 -5
- package/src/sources/Evm.res.mjs +44 -5
- package/src/sources/EvmChain.res +14 -18
- package/src/sources/EvmChain.res.mjs +12 -13
- package/src/sources/FieldMask.res +39 -0
- package/src/sources/FieldMask.res.mjs +42 -0
- package/src/sources/Fuel.res +8 -3
- package/src/sources/Fuel.res.mjs +5 -4
- package/src/sources/HyperFuelSource.res +31 -40
- package/src/sources/HyperFuelSource.res.mjs +52 -44
- package/src/sources/HyperSync.res +31 -9
- package/src/sources/HyperSync.res.mjs +47 -31
- package/src/sources/HyperSync.resi +10 -3
- package/src/sources/HyperSyncClient.res +15 -4
- package/src/sources/HyperSyncHeightStream.res +1 -8
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
- package/src/sources/HyperSyncSource.res +58 -90
- package/src/sources/HyperSyncSource.res.mjs +57 -66
- package/src/sources/RpcSource.res +118 -120
- package/src/sources/RpcSource.res.mjs +92 -74
- package/src/sources/RpcWebSocketHeightStream.res +0 -5
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
- package/src/sources/SimulateSource.res +5 -3
- package/src/sources/SimulateSource.res.mjs +12 -4
- package/src/sources/Source.res +19 -5
- package/src/sources/SourceManager.res +60 -10
- package/src/sources/SourceManager.res.mjs +54 -8
- package/src/sources/SourceManager.resi +10 -0
- package/src/sources/Svm.res +14 -10
- package/src/sources/Svm.res.mjs +23 -6
- package/src/sources/SvmHyperSyncClient.res +5 -6
- package/src/sources/SvmHyperSyncSource.res +82 -41
- package/src/sources/SvmHyperSyncSource.res.mjs +86 -41
- package/src/sources/TransactionStore.res +6 -107
- package/src/sources/TransactionStore.res.mjs +5 -86
- package/svm.schema.json +19 -0
package/src/Config.res
CHANGED
|
@@ -198,6 +198,7 @@ let svmEventDescriptorSchema = S.schema(s =>
|
|
|
198
198
|
"discriminator": s.matches(S.option(S.string)),
|
|
199
199
|
"discriminatorByteLen": s.matches(S.int),
|
|
200
200
|
"transactionFields": s.matches(S.array(Internal.svmTransactionFieldSchema)),
|
|
201
|
+
"blockFields": s.matches(S.option(S.array(Internal.svmBlockFieldSchema))),
|
|
201
202
|
"includeLogs": s.matches(S.bool),
|
|
202
203
|
"accountFilters": s.matches(
|
|
203
204
|
S.option(
|
|
@@ -654,7 +655,6 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
654
655
|
~events: option<array<_>>,
|
|
655
656
|
~abi,
|
|
656
657
|
~chainId: int,
|
|
657
|
-
~startBlock: option<int>,
|
|
658
658
|
~addresses: array<string>,
|
|
659
659
|
~svmDefinedTypes: JSON.t=JSON.Null,
|
|
660
660
|
) => {
|
|
@@ -677,10 +677,6 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
677
677
|
~kind=fuelKind,
|
|
678
678
|
~sighash,
|
|
679
679
|
~rawAbi=abi->(Utils.magic: EvmTypes.Abi.t => JSON.t),
|
|
680
|
-
~isWildcard=false,
|
|
681
|
-
~handler=None,
|
|
682
|
-
~contractRegister=None,
|
|
683
|
-
~startBlock?,
|
|
684
680
|
) :> Internal.eventConfig)
|
|
685
681
|
| None =>
|
|
686
682
|
JsError.throwWithMessage(
|
|
@@ -706,6 +702,7 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
706
702
|
"discriminator": option<string>,
|
|
707
703
|
"discriminatorByteLen": int,
|
|
708
704
|
"transactionFields": array<Internal.svmTransactionField>,
|
|
705
|
+
"blockFields": option<array<Internal.svmBlockField>>,
|
|
709
706
|
"includeLogs": bool,
|
|
710
707
|
"accountFilters": option<
|
|
711
708
|
array<array<{"position": int, "values": array<string>}>>,
|
|
@@ -741,16 +738,13 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
741
738
|
~discriminator=svm["discriminator"],
|
|
742
739
|
~discriminatorByteLen=svm["discriminatorByteLen"],
|
|
743
740
|
~transactionFields=svm["transactionFields"],
|
|
741
|
+
~blockFields=?svm["blockFields"],
|
|
744
742
|
~includeLogs=svm["includeLogs"],
|
|
745
743
|
~accountFilters,
|
|
746
744
|
~isInner=svm["isInner"],
|
|
747
|
-
~isWildcard=false,
|
|
748
|
-
~handler=None,
|
|
749
|
-
~contractRegister=None,
|
|
750
745
|
~accounts=svm["accounts"]->Option.getOr([]),
|
|
751
746
|
~args=svm["args"]->Option.getOr(JSON.Null),
|
|
752
747
|
~definedTypes=svmDefinedTypes,
|
|
753
|
-
~startBlock?,
|
|
754
748
|
) :> Internal.eventConfig)
|
|
755
749
|
| _ =>
|
|
756
750
|
(EventConfigBuilder.buildEvmEventConfig(
|
|
@@ -758,15 +752,8 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
758
752
|
~eventName,
|
|
759
753
|
~sighash,
|
|
760
754
|
~params,
|
|
761
|
-
~isWildcard=false,
|
|
762
|
-
~handler=None,
|
|
763
|
-
~contractRegister=None,
|
|
764
|
-
~eventFilters=None,
|
|
765
|
-
~probeChainId=chainId,
|
|
766
|
-
~onEventBlockFilterSchema=ecosystem.onEventBlockFilterSchema,
|
|
767
755
|
~blockFields=?eventItem["blockFields"],
|
|
768
756
|
~transactionFields=?eventItem["transactionFields"],
|
|
769
|
-
~startBlock?,
|
|
770
757
|
~globalBlockFieldsSet,
|
|
771
758
|
~globalTransactionFieldsSet,
|
|
772
759
|
) :> Internal.eventConfig)
|
|
@@ -818,17 +805,14 @@ let fromPublic = (publicConfigJson: JSON.t) => {
|
|
|
818
805
|
let addresses = rawAddresses->Array.map(parseAddress)
|
|
819
806
|
let startBlock = chainContract->Option.flatMap(cc => cc["startBlock"])
|
|
820
807
|
|
|
821
|
-
// Build event
|
|
822
|
-
//
|
|
823
|
-
//
|
|
824
|
-
// that branch on `chain.id` are taken through the same path
|
|
825
|
-
// they will follow at runtime.
|
|
808
|
+
// Build event definitions from JSON (field selections resolved
|
|
809
|
+
// inline). Handlers and per-chain `where` filters are layered on
|
|
810
|
+
// later as `onEventRegistration`s in `ChainState.makeInternal`.
|
|
826
811
|
let events = buildContractEvents(
|
|
827
812
|
~contractName=capitalizedName,
|
|
828
813
|
~events=contractData["events"],
|
|
829
814
|
~abi=contractData["abi"],
|
|
830
815
|
~chainId,
|
|
831
|
-
~startBlock,
|
|
832
816
|
~addresses=rawAddresses,
|
|
833
817
|
~svmDefinedTypes=contractData["svmAbi"]
|
|
834
818
|
->Option.map(a => a["definedTypes"])
|
|
@@ -1303,11 +1287,11 @@ let throwIfIncompatible = (
|
|
|
1303
1287
|
}
|
|
1304
1288
|
}
|
|
1305
1289
|
|
|
1306
|
-
// The returned value is a pure function of the JSON
|
|
1307
|
-
//
|
|
1308
|
-
// `
|
|
1309
|
-
// memoize without invalidation.
|
|
1310
|
-
let
|
|
1290
|
+
// The returned value is a pure function of the JSON: it holds only event
|
|
1291
|
+
// definitions, never handler/contractRegister/where state (that's layered on
|
|
1292
|
+
// separately as `HandlerRegister.registrationsByChainId`). That purity is
|
|
1293
|
+
// what lets this memoize without invalidation.
|
|
1294
|
+
let load = () =>
|
|
1311
1295
|
switch cached.contents {
|
|
1312
1296
|
| Some(c) => c
|
|
1313
1297
|
| None => {
|
package/src/Config.res.mjs
CHANGED
|
@@ -113,6 +113,7 @@ let svmEventDescriptorSchema = S$RescriptSchema.schema(s => ({
|
|
|
113
113
|
discriminator: s.m(S$RescriptSchema.option(S$RescriptSchema.string)),
|
|
114
114
|
discriminatorByteLen: s.m(S$RescriptSchema.int),
|
|
115
115
|
transactionFields: s.m(S$RescriptSchema.array(Internal.svmTransactionFieldSchema)),
|
|
116
|
+
blockFields: s.m(S$RescriptSchema.option(S$RescriptSchema.array(Internal.svmBlockFieldSchema))),
|
|
116
117
|
includeLogs: s.m(S$RescriptSchema.bool),
|
|
117
118
|
accountFilters: s.m(S$RescriptSchema.option(S$RescriptSchema.array(S$RescriptSchema.schema(s => ({
|
|
118
119
|
position: s.m(S$RescriptSchema.int),
|
|
@@ -490,7 +491,7 @@ function fromPublic(publicConfigJson) {
|
|
|
490
491
|
};
|
|
491
492
|
});
|
|
492
493
|
}
|
|
493
|
-
let buildContractEvents = (contractName, events, abi, chainId,
|
|
494
|
+
let buildContractEvents = (contractName, events, abi, chainId, addresses, svmDefinedTypesOpt) => {
|
|
494
495
|
let svmDefinedTypes = svmDefinedTypesOpt !== undefined ? svmDefinedTypesOpt : null;
|
|
495
496
|
if (events !== undefined) {
|
|
496
497
|
return events.map(eventItem => {
|
|
@@ -500,10 +501,10 @@ function fromPublic(publicConfigJson) {
|
|
|
500
501
|
let kind = eventItem.kind;
|
|
501
502
|
switch (ecosystemName) {
|
|
502
503
|
case "evm" :
|
|
503
|
-
return EventConfigBuilder.buildEvmEventConfig(contractName, eventName, sighash, params,
|
|
504
|
+
return EventConfigBuilder.buildEvmEventConfig(contractName, eventName, sighash, params, eventItem.blockFields, eventItem.transactionFields, Primitive_option.some(globalBlockFieldsSet), Primitive_option.some(globalTransactionFieldsSet));
|
|
504
505
|
case "fuel" :
|
|
505
506
|
if (kind !== undefined) {
|
|
506
|
-
return EventConfigBuilder.buildFuelEventConfig(contractName, eventName, kind, sighash, abi
|
|
507
|
+
return EventConfigBuilder.buildFuelEventConfig(contractName, eventName, kind, sighash, abi);
|
|
507
508
|
} else {
|
|
508
509
|
return Stdlib_JsError.throwWithMessage(`Fuel event ` + contractName + `.` + eventName + ` is missing "kind" in internal config`);
|
|
509
510
|
}
|
|
@@ -518,7 +519,7 @@ function fromPublic(publicConfigJson) {
|
|
|
518
519
|
position: af.position,
|
|
519
520
|
values: af.values
|
|
520
521
|
})));
|
|
521
|
-
return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeLogs, svm.transactionFields, accountFilters, svm.isInner,
|
|
522
|
+
return EventConfigBuilder.buildSvmInstructionEventConfig(contractName, eventName, programId, svm.discriminator, svm.discriminatorByteLen, svm.includeLogs, svm.transactionFields, svm.blockFields, accountFilters, svm.isInner, Stdlib_Option.getOr(svm.accounts, []), Stdlib_Option.getOr(svm.args, null), svmDefinedTypes);
|
|
522
523
|
}
|
|
523
524
|
});
|
|
524
525
|
} else {
|
|
@@ -559,7 +560,7 @@ function fromPublic(publicConfigJson) {
|
|
|
559
560
|
let rawAddresses = Stdlib_Option.getOr(Stdlib_Option.flatMap(chainContract, cc => cc.addresses), []);
|
|
560
561
|
let addresses = rawAddresses.map(parseAddress);
|
|
561
562
|
let startBlock = Stdlib_Option.flatMap(chainContract, cc => cc.startBlock);
|
|
562
|
-
let events = buildContractEvents(capitalizedName, contractData.events, contractData.abi, chainId,
|
|
563
|
+
let events = buildContractEvents(capitalizedName, contractData.events, contractData.abi, chainId, rawAddresses, Stdlib_Option.getOr(Stdlib_Option.map(contractData.svmAbi, a => a.definedTypes), null));
|
|
563
564
|
return {
|
|
564
565
|
name: capitalizedName,
|
|
565
566
|
abi: contractData.abi,
|
|
@@ -1011,7 +1012,7 @@ function throwIfIncompatible(changedPaths, resetCommand, runCommand, hasClickhou
|
|
|
1011
1012
|
Stdlib_JsError.throwWithMessage(`The following config changes are incompatible with the existing indexer data:\n\n` + bullets + `\n\nPick one:\n 1. ` + padTo(option1, col) + `# resume indexing where it left off\n 2. ` + padTo(resetCommand, col) + `# delete all indexed data and start over` + option3);
|
|
1012
1013
|
}
|
|
1013
1014
|
|
|
1014
|
-
function
|
|
1015
|
+
function load() {
|
|
1015
1016
|
let c = cached.contents;
|
|
1016
1017
|
if (c !== undefined) {
|
|
1017
1018
|
return c;
|
|
@@ -1060,7 +1061,7 @@ export {
|
|
|
1060
1061
|
canonicalJson,
|
|
1061
1062
|
diffPaths,
|
|
1062
1063
|
throwIfIncompatible,
|
|
1063
|
-
|
|
1064
|
+
load,
|
|
1064
1065
|
getPgUserEntities,
|
|
1065
1066
|
}
|
|
1066
1067
|
/* schema Not a pure module */
|
package/src/Core.res
CHANGED
|
@@ -9,6 +9,7 @@ type evmRpcClientCtor
|
|
|
9
9
|
type svmHypersyncClientCtor
|
|
10
10
|
type hyperfuelClientCtor
|
|
11
11
|
type transactionStoreCtor
|
|
12
|
+
type blockStoreCtor
|
|
12
13
|
|
|
13
14
|
type addon = {
|
|
14
15
|
getConfigJson: (~configPath: Null.t<string>, ~directory: Null.t<string>) => string,
|
|
@@ -23,10 +24,16 @@ type addon = {
|
|
|
23
24
|
hyperfuelClient: hyperfuelClientCtor,
|
|
24
25
|
@as("TransactionStore")
|
|
25
26
|
transactionStore: transactionStoreCtor,
|
|
27
|
+
@as("BlockStore")
|
|
28
|
+
blockStore: blockStoreCtor,
|
|
26
29
|
// Ordered transaction-field names exposed for the field-code contract test
|
|
27
30
|
// (the ReScript `transactionFields` arrays must match the Rust ordinals).
|
|
28
31
|
evmTransactionFieldNames: unit => array<string>,
|
|
29
32
|
svmTransactionFieldNames: unit => array<string>,
|
|
33
|
+
// Ordered block-field names for the same contract test (`blockFields` arrays
|
|
34
|
+
// must match the Rust ordinals).
|
|
35
|
+
evmBlockFieldNames: unit => array<string>,
|
|
36
|
+
svmBlockFieldNames: unit => array<string>,
|
|
30
37
|
}
|
|
31
38
|
|
|
32
39
|
@module("node:module") external createRequire: string => {..} = "createRequire"
|
package/src/CrossChainState.res
CHANGED
|
@@ -135,13 +135,13 @@ let enterReorgThreshold = (crossChainState: t) => {
|
|
|
135
135
|
// whole indexer. A chain is marked caught up only once EVERY chain is caught up
|
|
136
136
|
// (reached endblock or fetched/processed to head) with no processable events
|
|
137
137
|
// left — so no chain flips to ready while another is still backfilling.
|
|
138
|
-
let applyBatchProgress = (crossChainState: t, ~batch: Batch.t) => {
|
|
138
|
+
let applyBatchProgress = (crossChainState: t, ~batch: Batch.t, ~blockTimestampName: string) => {
|
|
139
139
|
let chainIds = crossChainState.chainIds
|
|
140
140
|
|
|
141
141
|
let everyChainCaughtUp = ref(true)
|
|
142
142
|
for i in 0 to chainIds->Array.length - 1 {
|
|
143
143
|
let cs = crossChainState->getChainState(chainIds->Array.getUnsafe(i))
|
|
144
|
-
cs->ChainState.applyBatchProgress(~batch)
|
|
144
|
+
cs->ChainState.applyBatchProgress(~batch, ~blockTimestampName)
|
|
145
145
|
if !(cs->ChainState.hasProcessedToEndblock || cs->ChainState.isProgressAtHead) {
|
|
146
146
|
everyChainCaughtUp := false
|
|
147
147
|
}
|
|
@@ -190,98 +190,98 @@ let totalReservedSize = (crossChainState: t) => {
|
|
|
190
190
|
total.contents
|
|
191
191
|
}
|
|
192
192
|
|
|
193
|
-
// Furthest-behind first: order candidate queries by the chain progress % at
|
|
194
|
-
// their fromBlock, so the most behind ranges across all chains are admitted
|
|
195
|
-
// before the rest.
|
|
196
|
-
let compareByProgress = (a: FetchState.query, b: FetchState.query) =>
|
|
197
|
-
Float.compare(a.progress, b.progress)
|
|
198
|
-
|
|
199
193
|
// Dispatch a fetch tick across the whole indexer from one shared pool of
|
|
200
|
-
// ~targetBufferSize ready events
|
|
201
|
-
//
|
|
202
|
-
//
|
|
203
|
-
//
|
|
204
|
-
//
|
|
205
|
-
//
|
|
206
|
-
// leaves the rest for the others
|
|
194
|
+
// ~targetBufferSize ready events, as a waterfall: visit chains furthest-behind
|
|
195
|
+
// first, hand each the budget remaining at that point (plus its own
|
|
196
|
+
// already-reserved share, since a chain's pending queries aren't "spent" —
|
|
197
|
+
// they're this chain's), let it turn that into queries sized against its own
|
|
198
|
+
// chain-density-derived target block, then subtract what it actually used
|
|
199
|
+
// before moving to the next chain. So a chain that can only use a little
|
|
200
|
+
// (density too low, or already caught up) leaves the rest for the others
|
|
201
|
+
// automatically. A chain visited after the budget ran out simply doesn't
|
|
202
|
+
// query this round — the leader's reservations release as its responses land,
|
|
203
|
+
// so the next tick redistributes. Chains that do get budget are additionally
|
|
204
|
+
// capped at the most-behind chain's target progress mapped onto their own
|
|
205
|
+
// range, so no chain runs further ahead than the chain the pool is
|
|
206
|
+
// prioritizing.
|
|
207
207
|
let checkAndFetch = async (
|
|
208
208
|
crossChainState: t,
|
|
209
209
|
~dispatchChain: (~chain: ChainMap.Chain.t, ~action: FetchState.nextQuery) => promise<unit>,
|
|
210
210
|
) => {
|
|
211
|
-
let remaining =
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
211
|
+
let remaining = ref(
|
|
212
|
+
Pervasives.max(
|
|
213
|
+
0.,
|
|
214
|
+
crossChainState.targetBufferSize->Int.toFloat -.
|
|
215
|
+
crossChainState->totalReadyCount->Int.toFloat -.
|
|
216
|
+
crossChainState->totalReservedSize,
|
|
217
|
+
),
|
|
216
218
|
)
|
|
217
219
|
|
|
218
|
-
let chainIds = crossChainState.chainIds
|
|
219
220
|
let actionByChain = Dict.make()
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
actionByChain->Utils.Dict.setByInt(chainId,
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
221
|
+
let maxProgress = ref(None)
|
|
222
|
+
crossChainState
|
|
223
|
+
->priorityOrder
|
|
224
|
+
->Array.forEach(cs => {
|
|
225
|
+
let chainId = (cs->ChainState.chainConfig).id
|
|
226
|
+
if cs->ChainState.knownHeight == 0 {
|
|
227
|
+
// No height yet — nothing to size a budget or an alignment line
|
|
228
|
+
// against. Skip without consuming budget or claiming leadership, so a
|
|
229
|
+
// chain whose source hasn't reported doesn't unconstrain everyone else.
|
|
230
|
+
actionByChain->Utils.Dict.setByInt(chainId, FetchState.WaitingForNewBlock)
|
|
231
|
+
} else {
|
|
232
|
+
let chainTargetItems = remaining.contents +. cs->ChainState.pendingBudget
|
|
233
|
+
let maxTargetBlock = switch maxProgress.contents {
|
|
234
|
+
| None =>
|
|
235
|
+
// The most-behind chain sets the alignment line for everyone after it.
|
|
236
|
+
maxProgress :=
|
|
237
|
+
Some(
|
|
238
|
+
cs->ChainState.progressAtBlock(
|
|
239
|
+
~blockNumber=cs->ChainState.targetBlock(~chainTargetItems),
|
|
240
|
+
),
|
|
241
|
+
)
|
|
242
|
+
None
|
|
243
|
+
| Some(progress) => Some(cs->ChainState.blockAtProgress(~progress))
|
|
244
|
+
}
|
|
245
|
+
switch cs->ChainState.getNextQuery(~chainTargetItems, ~maxTargetBlock?) {
|
|
246
|
+
| (WaitingForNewBlock | NothingToQuery) as action =>
|
|
247
|
+
actionByChain->Utils.Dict.setByInt(chainId, action)
|
|
248
|
+
| Ready(queries) => {
|
|
249
|
+
let consumed =
|
|
250
|
+
queries->Array.reduce(0., (acc, query: FetchState.query) =>
|
|
251
|
+
acc +. query.itemsTarget->Int.toFloat
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
let partitions = Dict.make()
|
|
255
|
+
queries->Array.forEach((query: FetchState.query) =>
|
|
256
|
+
partitions->Dict.set(
|
|
257
|
+
query.partitionId,
|
|
258
|
+
{
|
|
259
|
+
"fromBlock": query.fromBlock,
|
|
260
|
+
"targetBlock": query.toBlock,
|
|
261
|
+
"targetEvents": query.itemsTarget,
|
|
262
|
+
},
|
|
263
|
+
)
|
|
264
|
+
)
|
|
265
|
+
Logging.trace({
|
|
266
|
+
"msg": "Started querying",
|
|
267
|
+
"chainId": chainId,
|
|
268
|
+
"partitions": partitions,
|
|
269
|
+
})
|
|
270
|
+
|
|
271
|
+
actionByChain->Utils.Dict.setByInt(chainId, FetchState.Ready(queries))
|
|
272
|
+
// Mark the queries in flight and reserve their size against the
|
|
273
|
+
// shared budget; released as each response lands in
|
|
274
|
+
// handleQueryResult.
|
|
275
|
+
cs->ChainState.startFetchingQueries(~queries)
|
|
276
|
+
remaining := Pervasives.max(0., remaining.contents -. consumed)
|
|
277
|
+
}
|
|
278
|
+
}
|
|
238
279
|
}
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
candidates->Array.sort(compareByProgress)
|
|
242
|
-
|
|
243
|
-
// Admit furthest-behind first until the budget runs out. The condition is
|
|
244
|
-
// checked before each query, so as long as there's any budget left we admit
|
|
245
|
-
// the next one even when its estimate alone exceeds the remainder — otherwise a
|
|
246
|
-
// chain whose only query is bigger than the budget would never make progress.
|
|
247
|
-
let admittedByChain = Dict.make()
|
|
248
|
-
let running = ref(0.)
|
|
249
|
-
let remainingF = remaining->Int.toFloat
|
|
250
|
-
let idx = ref(0)
|
|
251
|
-
while running.contents < remainingF && idx.contents < candidates->Array.length {
|
|
252
|
-
let query = candidates->Array.getUnsafe(idx.contents)
|
|
253
|
-
admittedByChain->Utils.Dict.push(query.chainId->Int.toString, query)
|
|
254
|
-
running := running.contents +. query.estResponseSize
|
|
255
|
-
idx := idx.contents + 1
|
|
256
|
-
}
|
|
257
|
-
admittedByChain->Dict.forEachWithKey((queries, chainId) => {
|
|
258
|
-
let partitions = Dict.make()
|
|
259
|
-
queries->Array.forEach((query: FetchState.query) =>
|
|
260
|
-
partitions->Dict.set(
|
|
261
|
-
query.partitionId,
|
|
262
|
-
{
|
|
263
|
-
"fromBlock": query.fromBlock,
|
|
264
|
-
"targetBlock": query.toBlock,
|
|
265
|
-
"targetEvents": query.estResponseSize->Math.round->Float.toInt,
|
|
266
|
-
},
|
|
267
|
-
)
|
|
268
|
-
)
|
|
269
|
-
Logging.trace({
|
|
270
|
-
"msg": "Started querying",
|
|
271
|
-
"chainId": chainId->Int.fromString->Option.getUnsafe,
|
|
272
|
-
"partitions": partitions,
|
|
273
|
-
})
|
|
274
|
-
actionByChain->Dict.set(chainId, FetchState.Ready(queries))
|
|
275
|
-
// Mark the admitted queries in flight and reserve their size against the
|
|
276
|
-
// shared budget; released as each response lands in handleQueryResult.
|
|
277
|
-
crossChainState
|
|
278
|
-
->getChainState(chainId->Int.fromString->Option.getUnsafe)
|
|
279
|
-
->ChainState.startFetchingQueries(~queries)
|
|
280
280
|
})
|
|
281
281
|
|
|
282
282
|
let promises = []
|
|
283
|
-
for i in 0 to chainIds->Array.length - 1 {
|
|
284
|
-
let chainId = chainIds->Array.getUnsafe(i)
|
|
283
|
+
for i in 0 to crossChainState.chainIds->Array.length - 1 {
|
|
284
|
+
let chainId = crossChainState.chainIds->Array.getUnsafe(i)
|
|
285
285
|
switch actionByChain->Utils.Dict.dangerouslyGetByIntNonOption(chainId) {
|
|
286
286
|
| Some(NothingToQuery)
|
|
287
287
|
| None => ()
|
|
@@ -7,9 +7,7 @@ import * as Logging from "./Logging.res.mjs";
|
|
|
7
7
|
import * as ChainMap from "./ChainMap.res.mjs";
|
|
8
8
|
import * as ChainState from "./ChainState.res.mjs";
|
|
9
9
|
import * as Prometheus from "./Prometheus.res.mjs";
|
|
10
|
-
import * as
|
|
11
|
-
import * as Stdlib_Dict from "@rescript/runtime/lib/es6/Stdlib_Dict.js";
|
|
12
|
-
import * as Primitive_int from "@rescript/runtime/lib/es6/Primitive_int.js";
|
|
10
|
+
import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
|
|
13
11
|
import * as Primitive_float from "@rescript/runtime/lib/es6/Primitive_float.js";
|
|
14
12
|
import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
|
|
15
13
|
|
|
@@ -95,12 +93,12 @@ function enterReorgThreshold(crossChainState) {
|
|
|
95
93
|
crossChainState.isInReorgThreshold = true;
|
|
96
94
|
}
|
|
97
95
|
|
|
98
|
-
function applyBatchProgress(crossChainState, batch) {
|
|
96
|
+
function applyBatchProgress(crossChainState, batch, blockTimestampName) {
|
|
99
97
|
let chainIds = crossChainState.chainIds;
|
|
100
98
|
let everyChainCaughtUp = true;
|
|
101
99
|
for (let i = 0, i_finish = chainIds.length; i < i_finish; ++i) {
|
|
102
100
|
let cs = crossChainState.chainStates[chainIds[i]];
|
|
103
|
-
ChainState.applyBatchProgress(cs, batch);
|
|
101
|
+
ChainState.applyBatchProgress(cs, batch, blockTimestampName);
|
|
104
102
|
if (!(ChainState.hasProcessedToEndblock(cs) || ChainState.isProgressAtHead(cs))) {
|
|
105
103
|
everyChainCaughtUp = false;
|
|
106
104
|
}
|
|
@@ -135,73 +133,68 @@ function totalReservedSize(crossChainState) {
|
|
|
135
133
|
return total;
|
|
136
134
|
}
|
|
137
135
|
|
|
138
|
-
function compareByProgress(a, b) {
|
|
139
|
-
return Primitive_float.compare(a.progress, b.progress);
|
|
140
|
-
}
|
|
141
|
-
|
|
142
136
|
async function checkAndFetch(crossChainState, dispatchChain) {
|
|
143
|
-
let remaining =
|
|
144
|
-
|
|
137
|
+
let remaining = {
|
|
138
|
+
contents: Primitive_float.max(0, crossChainState.targetBufferSize - totalReadyCount(crossChainState) - totalReservedSize(crossChainState))
|
|
139
|
+
};
|
|
145
140
|
let actionByChain = {};
|
|
146
|
-
let
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
let
|
|
141
|
+
let maxProgress = {
|
|
142
|
+
contents: undefined
|
|
143
|
+
};
|
|
144
|
+
priorityOrder(crossChainState).forEach(cs => {
|
|
145
|
+
let chainId = ChainState.chainConfig(cs).id;
|
|
146
|
+
if (ChainState.knownHeight(cs) === 0) {
|
|
147
|
+
actionByChain[chainId] = "WaitingForNewBlock";
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
let chainTargetItems = remaining.contents + ChainState.pendingBudget(cs);
|
|
151
|
+
let progress = maxProgress.contents;
|
|
152
|
+
let maxTargetBlock = progress !== undefined ? ChainState.blockAtProgress(cs, progress) : (maxProgress.contents = ChainState.progressAtBlock(cs, ChainState.targetBlock(cs, chainTargetItems)), undefined);
|
|
153
|
+
let action = ChainState.getNextQuery(cs, chainTargetItems, maxTargetBlock);
|
|
151
154
|
if (typeof action !== "object") {
|
|
155
|
+
if (action === "WaitingForNewBlock") {
|
|
156
|
+
actionByChain[chainId] = action;
|
|
157
|
+
return;
|
|
158
|
+
}
|
|
152
159
|
actionByChain[chainId] = action;
|
|
160
|
+
return;
|
|
153
161
|
} else {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
162
|
+
let queries = action._0;
|
|
163
|
+
let consumed = Stdlib_Array.reduce(queries, 0, (acc, query) => acc + query.itemsTarget);
|
|
164
|
+
let partitions = {};
|
|
165
|
+
queries.forEach(query => {
|
|
166
|
+
partitions[query.partitionId] = {
|
|
167
|
+
fromBlock: query.fromBlock,
|
|
168
|
+
targetBlock: query.toBlock,
|
|
169
|
+
targetEvents: query.itemsTarget
|
|
170
|
+
};
|
|
159
171
|
});
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
Utils.Dict.push(admittedByChain, query.chainId.toString(), query);
|
|
169
|
-
running = running + query.estResponseSize;
|
|
170
|
-
idx = idx + 1 | 0;
|
|
171
|
-
};
|
|
172
|
-
Stdlib_Dict.forEachWithKey(admittedByChain, (queries, chainId) => {
|
|
173
|
-
let partitions = {};
|
|
174
|
-
queries.forEach(query => {
|
|
175
|
-
partitions[query.partitionId] = {
|
|
176
|
-
fromBlock: query.fromBlock,
|
|
177
|
-
targetBlock: query.toBlock,
|
|
178
|
-
targetEvents: Math.round(query.estResponseSize) | 0
|
|
172
|
+
Logging.trace({
|
|
173
|
+
msg: "Started querying",
|
|
174
|
+
chainId: chainId,
|
|
175
|
+
partitions: partitions
|
|
176
|
+
});
|
|
177
|
+
actionByChain[chainId] = {
|
|
178
|
+
TAG: "Ready",
|
|
179
|
+
_0: queries
|
|
179
180
|
};
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
partitions: partitions
|
|
185
|
-
});
|
|
186
|
-
actionByChain[chainId] = {
|
|
187
|
-
TAG: "Ready",
|
|
188
|
-
_0: queries
|
|
189
|
-
};
|
|
190
|
-
let chainId$1 = Stdlib_Int.fromString(chainId, undefined);
|
|
191
|
-
ChainState.startFetchingQueries(crossChainState.chainStates[chainId$1], queries);
|
|
181
|
+
ChainState.startFetchingQueries(cs, queries);
|
|
182
|
+
remaining.contents = Primitive_float.max(0, remaining.contents - consumed);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
192
185
|
});
|
|
193
186
|
let promises = [];
|
|
194
|
-
for (let i
|
|
195
|
-
let chainId
|
|
196
|
-
let action
|
|
197
|
-
if (action
|
|
187
|
+
for (let i = 0, i_finish = crossChainState.chainIds.length; i < i_finish; ++i) {
|
|
188
|
+
let chainId = crossChainState.chainIds[i];
|
|
189
|
+
let action = actionByChain[chainId];
|
|
190
|
+
if (action !== undefined) {
|
|
198
191
|
let exit = 0;
|
|
199
|
-
if (typeof action
|
|
192
|
+
if (typeof action === "object" || action === "WaitingForNewBlock") {
|
|
200
193
|
exit = 1;
|
|
201
194
|
}
|
|
202
195
|
if (exit === 1) {
|
|
203
|
-
let chain = ChainMap.Chain.makeUnsafe(chainId
|
|
204
|
-
promises.push(dispatchChain(chain, action
|
|
196
|
+
let chain = ChainMap.Chain.makeUnsafe(chainId);
|
|
197
|
+
promises.push(dispatchChain(chain, action));
|
|
205
198
|
}
|
|
206
199
|
}
|
|
207
200
|
}
|
package/src/CrossChainState.resi
CHANGED
|
@@ -29,7 +29,7 @@ let createBatch: (
|
|
|
29
29
|
~isRollback: bool,
|
|
30
30
|
) => Batch.t
|
|
31
31
|
let enterReorgThreshold: t => unit
|
|
32
|
-
let applyBatchProgress: (t, ~batch: Batch.t) => unit
|
|
32
|
+
let applyBatchProgress: (t, ~batch: Batch.t, ~blockTimestampName: string) => unit
|
|
33
33
|
|
|
34
34
|
// Fetch control.
|
|
35
35
|
let priorityOrder: t => array<ChainState.t>
|
package/src/Ecosystem.res
CHANGED
|
@@ -69,12 +69,12 @@ let getItemLogger = {
|
|
|
69
69
|
| None =>
|
|
70
70
|
let logger = switch item {
|
|
71
71
|
| Internal.Event(_) => ecosystem.toEventLogger(item->Internal.castUnsafeEventItem)
|
|
72
|
-
| Block({blockNumber,
|
|
72
|
+
| Block({blockNumber, onBlockRegistration}) =>
|
|
73
73
|
Logging.createChildFrom(
|
|
74
74
|
~logger=ecosystem.logger,
|
|
75
75
|
~params={
|
|
76
|
-
"onBlock":
|
|
77
|
-
"chainId":
|
|
76
|
+
"onBlock": onBlockRegistration.name,
|
|
77
|
+
"chainId": onBlockRegistration.chainId,
|
|
78
78
|
"block": blockNumber,
|
|
79
79
|
},
|
|
80
80
|
)
|
package/src/Ecosystem.res.mjs
CHANGED
|
@@ -26,10 +26,10 @@ function getItemLogger(item, ecosystem) {
|
|
|
26
26
|
if (item.kind === 0) {
|
|
27
27
|
logger$1 = ecosystem.toEventLogger(item);
|
|
28
28
|
} else {
|
|
29
|
-
let
|
|
29
|
+
let onBlockRegistration = item.onBlockRegistration;
|
|
30
30
|
logger$1 = Logging.createChildFrom(ecosystem.logger, {
|
|
31
|
-
onBlock:
|
|
32
|
-
chainId:
|
|
31
|
+
onBlock: onBlockRegistration.name,
|
|
32
|
+
chainId: onBlockRegistration.chainId,
|
|
33
33
|
block: item.blockNumber
|
|
34
34
|
});
|
|
35
35
|
}
|
package/src/Envio.res
CHANGED
|
@@ -66,18 +66,21 @@ type svmLog = {
|
|
|
66
66
|
message: string,
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
-
/** Block context for a matched instruction. `
|
|
70
|
-
|
|
71
|
-
|
|
69
|
+
/** Block context for a matched instruction. `slot`/`hash` are always present;
|
|
70
|
+
`time` can still be absent — HyperSync/Solana may not report a block time for
|
|
71
|
+
every slot. Every other field is opt-in via `field_selection.block_fields`,
|
|
72
|
+
materialised from the per-chain block store at batch prep. */
|
|
72
73
|
type svmInstructionBlock = {
|
|
73
74
|
/** Slot this instruction's block was matched in. */
|
|
74
75
|
slot: int,
|
|
75
|
-
/** Unix block time (seconds).
|
|
76
|
-
|
|
77
|
-
time: int,
|
|
78
|
-
/** Block hash. Currently always empty — populated by the future
|
|
79
|
-
reorg-guard `queryBlockHash(slot)` route. */
|
|
76
|
+
/** Unix block time (seconds). */
|
|
77
|
+
time?: int,
|
|
80
78
|
hash: string,
|
|
79
|
+
/** Block height (distinct from slot). Absent when not selected via
|
|
80
|
+
`field_selection.block_fields`. */
|
|
81
|
+
height?: int,
|
|
82
|
+
parentSlot?: int,
|
|
83
|
+
parentHash?: string,
|
|
81
84
|
}
|
|
82
85
|
|
|
83
86
|
/** The per-instruction payload handlers receive as their `instruction`
|
|
@@ -111,7 +114,9 @@ type svmInstruction = {
|
|
|
111
114
|
/** Program log entries scoped to this instruction. Absent when the
|
|
112
115
|
per-instruction `include_logs` flag is `false`. */
|
|
113
116
|
logs?: array<svmLog>,
|
|
114
|
-
|
|
117
|
+
// Omitted on construction; materialised onto the payload at batch prep,
|
|
118
|
+
// before a handler ever reads it.
|
|
119
|
+
block?: svmInstructionBlock,
|
|
115
120
|
}
|
|
116
121
|
|
|
117
122
|
/** Arguments passed to handlers registered via `indexer.onInstruction`. */
|