envio 3.0.0-alpha.2 → 3.0.0-alpha.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +44 -33
- package/fuel.schema.json +32 -21
- package/index.d.ts +1 -0
- package/package.json +7 -6
- package/src/Batch.res.mjs +1 -1
- package/src/Benchmark.res +394 -0
- package/src/Benchmark.res.mjs +398 -0
- package/src/ChainFetcher.res +459 -0
- package/src/ChainFetcher.res.mjs +281 -0
- package/src/ChainManager.res +179 -0
- package/src/ChainManager.res.mjs +139 -0
- package/src/Config.res +15 -1
- package/src/Config.res.mjs +27 -4
- package/src/Ecosystem.res +9 -124
- package/src/Ecosystem.res.mjs +19 -160
- package/src/Env.res +0 -1
- package/src/Env.res.mjs +0 -3
- package/src/Envio.gen.ts +9 -1
- package/src/Envio.res +12 -9
- package/src/EventProcessing.res +476 -0
- package/src/EventProcessing.res.mjs +341 -0
- package/src/FetchState.res +54 -29
- package/src/FetchState.res.mjs +62 -35
- package/src/GlobalState.res +1169 -0
- package/src/GlobalState.res.mjs +1196 -0
- package/src/Internal.res +2 -1
- package/src/LoadLayer.res +444 -0
- package/src/LoadLayer.res.mjs +296 -0
- package/src/LoadLayer.resi +32 -0
- package/src/Prometheus.res +8 -8
- package/src/Prometheus.res.mjs +10 -10
- package/src/ReorgDetection.res +6 -10
- package/src/ReorgDetection.res.mjs +6 -6
- package/src/UserContext.res +356 -0
- package/src/UserContext.res.mjs +238 -0
- package/src/bindings/DateFns.res +71 -0
- package/src/bindings/DateFns.res.mjs +22 -0
- package/src/sources/Evm.res +87 -0
- package/src/sources/Evm.res.mjs +105 -0
- package/src/sources/EvmChain.res +95 -0
- package/src/sources/EvmChain.res.mjs +61 -0
- package/src/sources/Fuel.res +19 -34
- package/src/sources/Fuel.res.mjs +34 -16
- package/src/sources/FuelSDK.res +37 -0
- package/src/sources/FuelSDK.res.mjs +29 -0
- package/src/sources/HyperFuel.res +2 -2
- package/src/sources/HyperFuel.resi +1 -1
- package/src/sources/HyperFuelClient.res +2 -2
- package/src/sources/HyperFuelSource.res +8 -8
- package/src/sources/HyperFuelSource.res.mjs +5 -5
- package/src/sources/HyperSyncSource.res +5 -5
- package/src/sources/HyperSyncSource.res.mjs +5 -5
- package/src/sources/RpcSource.res +4 -4
- package/src/sources/RpcSource.res.mjs +3 -3
- package/src/sources/Solana.res +59 -0
- package/src/sources/Solana.res.mjs +79 -0
- package/src/sources/Source.res +2 -2
- package/src/sources/SourceManager.res +24 -32
- package/src/sources/SourceManager.res.mjs +20 -20
- package/src/sources/SourceManager.resi +4 -5
|
@@ -186,7 +186,7 @@ function make(param) {
|
|
|
186
186
|
var endpointUrl = param.endpointUrl;
|
|
187
187
|
var chain = param.chain;
|
|
188
188
|
var getSelectionConfig = memoGetSelectionConfig(chain);
|
|
189
|
-
var getItemsOrThrow = async function (fromBlock, toBlock, addressesByContractName, indexingContracts,
|
|
189
|
+
var getItemsOrThrow = async function (fromBlock, toBlock, addressesByContractName, indexingContracts, knownHeight, param, selection, retry, logger) {
|
|
190
190
|
var mkLogAndRaise = function (extra, extra$1) {
|
|
191
191
|
return ErrorHandling.mkLogAndRaise(logger, extra, extra$1);
|
|
192
192
|
};
|
|
@@ -221,7 +221,7 @@ function make(param) {
|
|
|
221
221
|
_1: {
|
|
222
222
|
TAG: "FailedGettingItems",
|
|
223
223
|
exn: null,
|
|
224
|
-
attemptedToBlock: Belt_Option.getWithDefault(toBlock,
|
|
224
|
+
attemptedToBlock: Belt_Option.getWithDefault(toBlock, knownHeight),
|
|
225
225
|
retry: tmp
|
|
226
226
|
},
|
|
227
227
|
Error: new Error()
|
|
@@ -232,7 +232,7 @@ function make(param) {
|
|
|
232
232
|
_1: {
|
|
233
233
|
TAG: "FailedGettingItems",
|
|
234
234
|
exn: error,
|
|
235
|
-
attemptedToBlock: Belt_Option.getWithDefault(toBlock,
|
|
235
|
+
attemptedToBlock: Belt_Option.getWithDefault(toBlock, knownHeight),
|
|
236
236
|
retry: {
|
|
237
237
|
TAG: "WithBackoff",
|
|
238
238
|
message: "Unexpected issue while fetching events from HyperFuel client. Attempt a retry.",
|
|
@@ -243,7 +243,7 @@ function make(param) {
|
|
|
243
243
|
};
|
|
244
244
|
}
|
|
245
245
|
var pageFetchTime = Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(startFetchingBatchTimeRef)));
|
|
246
|
-
var
|
|
246
|
+
var knownHeight$1 = pageUnsafe.archiveHeight;
|
|
247
247
|
var heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;
|
|
248
248
|
var match = Belt_Array.get(pageUnsafe.items, pageUnsafe.items.length - 1 | 0);
|
|
249
249
|
var lastBlockQueriedPromise;
|
|
@@ -407,7 +407,7 @@ function make(param) {
|
|
|
407
407
|
"page fetch time (ms)": stats_page$unknownfetch$unknowntime$unknown$lparms$rpar
|
|
408
408
|
};
|
|
409
409
|
return {
|
|
410
|
-
|
|
410
|
+
knownHeight: knownHeight$1,
|
|
411
411
|
reorgGuard: reorgGuard,
|
|
412
412
|
parsedQueueItems: parsedQueueItems,
|
|
413
413
|
fromBlockQueried: fromBlock,
|
|
@@ -249,7 +249,7 @@ let make = (
|
|
|
249
249
|
~toBlock,
|
|
250
250
|
~addressesByContractName,
|
|
251
251
|
~indexingContracts,
|
|
252
|
-
~
|
|
252
|
+
~knownHeight,
|
|
253
253
|
~partitionId as _,
|
|
254
254
|
~selection,
|
|
255
255
|
~retry,
|
|
@@ -282,7 +282,7 @@ let make = (
|
|
|
282
282
|
Source.GetItemsError(
|
|
283
283
|
Source.FailedGettingItems({
|
|
284
284
|
exn: %raw(`null`),
|
|
285
|
-
attemptedToBlock: toBlock->Option.getWithDefault(
|
|
285
|
+
attemptedToBlock: toBlock->Option.getWithDefault(knownHeight),
|
|
286
286
|
retry: switch error {
|
|
287
287
|
| WrongInstance =>
|
|
288
288
|
let backoffMillis = switch retry {
|
|
@@ -308,7 +308,7 @@ let make = (
|
|
|
308
308
|
Source.GetItemsError(
|
|
309
309
|
Source.FailedGettingItems({
|
|
310
310
|
exn,
|
|
311
|
-
attemptedToBlock: toBlock->Option.getWithDefault(
|
|
311
|
+
attemptedToBlock: toBlock->Option.getWithDefault(knownHeight),
|
|
312
312
|
retry: WithBackoff({
|
|
313
313
|
message: `Unexpected issue while fetching events from HyperSync client. Attempt a retry.`,
|
|
314
314
|
backoffMillis: switch retry {
|
|
@@ -325,7 +325,7 @@ let make = (
|
|
|
325
325
|
startFetchingBatchTimeRef->Hrtime.timeSince->Hrtime.toMillis->Hrtime.intFromMillis
|
|
326
326
|
|
|
327
327
|
//set height and next from block
|
|
328
|
-
let
|
|
328
|
+
let knownHeight = pageUnsafe.archiveHeight
|
|
329
329
|
|
|
330
330
|
//The heighest (biggest) blocknumber that was accounted for in
|
|
331
331
|
//Our query. Not necessarily the blocknumber of the last log returned
|
|
@@ -537,7 +537,7 @@ let make = (
|
|
|
537
537
|
parsedQueueItems,
|
|
538
538
|
latestFetchedBlockNumber: rangeLastBlock.blockNumber,
|
|
539
539
|
stats,
|
|
540
|
-
|
|
540
|
+
knownHeight,
|
|
541
541
|
reorgGuard,
|
|
542
542
|
fromBlockQueried: fromBlock,
|
|
543
543
|
}
|
|
@@ -179,7 +179,7 @@ function make(param) {
|
|
|
179
179
|
Belt_Array.forEach(param.contracts, (function (contract) {
|
|
180
180
|
contractNameAbiMapping[contract.name] = contract.abi;
|
|
181
181
|
}));
|
|
182
|
-
var getItemsOrThrow = async function (fromBlock, toBlock, addressesByContractName, indexingContracts,
|
|
182
|
+
var getItemsOrThrow = async function (fromBlock, toBlock, addressesByContractName, indexingContracts, knownHeight, param, selection, retry, logger) {
|
|
183
183
|
var mkLogAndRaise = function (extra, extra$1) {
|
|
184
184
|
return ErrorHandling.mkLogAndRaise(logger, extra, extra$1);
|
|
185
185
|
};
|
|
@@ -221,7 +221,7 @@ function make(param) {
|
|
|
221
221
|
_1: {
|
|
222
222
|
TAG: "FailedGettingItems",
|
|
223
223
|
exn: null,
|
|
224
|
-
attemptedToBlock: Belt_Option.getWithDefault(toBlock,
|
|
224
|
+
attemptedToBlock: Belt_Option.getWithDefault(toBlock, knownHeight),
|
|
225
225
|
retry: tmp
|
|
226
226
|
},
|
|
227
227
|
Error: new Error()
|
|
@@ -232,7 +232,7 @@ function make(param) {
|
|
|
232
232
|
_1: {
|
|
233
233
|
TAG: "FailedGettingItems",
|
|
234
234
|
exn: error,
|
|
235
|
-
attemptedToBlock: Belt_Option.getWithDefault(toBlock,
|
|
235
|
+
attemptedToBlock: Belt_Option.getWithDefault(toBlock, knownHeight),
|
|
236
236
|
retry: {
|
|
237
237
|
TAG: "WithBackoff",
|
|
238
238
|
message: "Unexpected issue while fetching events from HyperSync client. Attempt a retry.",
|
|
@@ -243,7 +243,7 @@ function make(param) {
|
|
|
243
243
|
};
|
|
244
244
|
}
|
|
245
245
|
var pageFetchTime = Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(startFetchingBatchTimeRef)));
|
|
246
|
-
var
|
|
246
|
+
var knownHeight$1 = pageUnsafe.archiveHeight;
|
|
247
247
|
var heighestBlockQueried = pageUnsafe.nextBlock - 1 | 0;
|
|
248
248
|
var match = pageUnsafe.rollbackGuard;
|
|
249
249
|
var lastBlockQueriedPromise;
|
|
@@ -371,7 +371,7 @@ function make(param) {
|
|
|
371
371
|
"page fetch time (ms)": stats_page$unknownfetch$unknowntime$unknown$lparms$rpar
|
|
372
372
|
};
|
|
373
373
|
return {
|
|
374
|
-
|
|
374
|
+
knownHeight: knownHeight$1,
|
|
375
375
|
reorgGuard: reorgGuard,
|
|
376
376
|
parsedQueueItems: parsedQueueItems,
|
|
377
377
|
fromBlockQueried: fromBlock,
|
|
@@ -601,7 +601,7 @@ let make = (
|
|
|
601
601
|
~toBlock,
|
|
602
602
|
~addressesByContractName,
|
|
603
603
|
~indexingContracts,
|
|
604
|
-
~
|
|
604
|
+
~knownHeight,
|
|
605
605
|
~partitionId,
|
|
606
606
|
~selection: FetchState.selection,
|
|
607
607
|
~retry as _,
|
|
@@ -621,8 +621,8 @@ let make = (
|
|
|
621
621
|
|
|
622
622
|
// Always have a toBlock for an RPC worker
|
|
623
623
|
let toBlock = switch toBlock {
|
|
624
|
-
| Some(toBlock) => Pervasives.min(toBlock,
|
|
625
|
-
| None =>
|
|
624
|
+
| Some(toBlock) => Pervasives.min(toBlock, knownHeight)
|
|
625
|
+
| None => knownHeight
|
|
626
626
|
}
|
|
627
627
|
|
|
628
628
|
let suggestedToBlock = Pervasives.min(fromBlock + suggestedBlockInterval - 1, toBlock)
|
|
@@ -874,7 +874,7 @@ let make = (
|
|
|
874
874
|
stats: {
|
|
875
875
|
totalTimeElapsed: totalTimeElapsed,
|
|
876
876
|
},
|
|
877
|
-
|
|
877
|
+
knownHeight,
|
|
878
878
|
reorgGuard,
|
|
879
879
|
fromBlockQueried: fromBlock,
|
|
880
880
|
}
|
|
@@ -604,11 +604,11 @@ function make(param) {
|
|
|
604
604
|
return HyperSyncClient.Decoder.fromSignatures(allEventSignatures);
|
|
605
605
|
}
|
|
606
606
|
};
|
|
607
|
-
var getItemsOrThrow = async function (fromBlock, toBlock, addressesByContractName, indexingContracts,
|
|
607
|
+
var getItemsOrThrow = async function (fromBlock, toBlock, addressesByContractName, indexingContracts, knownHeight, partitionId, selection, param, param$1) {
|
|
608
608
|
var startFetchingBatchTimeRef = Hrtime.makeTimer();
|
|
609
609
|
var maxSuggestedBlockInterval = mutSuggestedBlockIntervals[maxSuggestedBlockIntervalKey];
|
|
610
610
|
var suggestedBlockInterval = maxSuggestedBlockInterval !== undefined ? maxSuggestedBlockInterval : Belt_Option.getWithDefault(mutSuggestedBlockIntervals[partitionId], syncConfig.initialBlockInterval);
|
|
611
|
-
var toBlock$1 = toBlock !== undefined && toBlock <
|
|
611
|
+
var toBlock$1 = toBlock !== undefined && toBlock < knownHeight ? toBlock : knownHeight;
|
|
612
612
|
var suggestedToBlock = Caml.int_max(Caml.int_min((fromBlock + suggestedBlockInterval | 0) - 1 | 0, toBlock$1), fromBlock);
|
|
613
613
|
var firstBlockParentPromise = fromBlock > 0 ? LazyLoader.get(blockLoader.contents, fromBlock - 1 | 0).then(function (res) {
|
|
614
614
|
return res;
|
|
@@ -785,7 +785,7 @@ function make(param) {
|
|
|
785
785
|
prevRangeLastBlock: reorgGuard_prevRangeLastBlock
|
|
786
786
|
};
|
|
787
787
|
return {
|
|
788
|
-
|
|
788
|
+
knownHeight: knownHeight,
|
|
789
789
|
reorgGuard: reorgGuard,
|
|
790
790
|
parsedQueueItems: parsedQueueItems,
|
|
791
791
|
fromBlockQueried: fromBlock,
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
@get external getNumber: Internal.eventBlock => int = "height"
|
|
2
|
+
@get external getTimestamp: Internal.eventBlock => int = "time"
|
|
3
|
+
@get external getId: Internal.eventBlock => string = "hash"
|
|
4
|
+
|
|
5
|
+
let cleanUpRawEventFieldsInPlace: Js.Json.t => unit = %raw(`fields => {
|
|
6
|
+
delete fields.hash
|
|
7
|
+
delete fields.height
|
|
8
|
+
delete fields.time
|
|
9
|
+
}`)
|
|
10
|
+
|
|
11
|
+
let ecosystem: Ecosystem.t = {
|
|
12
|
+
name: Solana,
|
|
13
|
+
blockFields: ["slot"],
|
|
14
|
+
transactionFields: [],
|
|
15
|
+
blockNumberName: "height",
|
|
16
|
+
blockTimestampName: "time",
|
|
17
|
+
blockHashName: "hash",
|
|
18
|
+
getNumber,
|
|
19
|
+
getTimestamp,
|
|
20
|
+
getId,
|
|
21
|
+
cleanUpRawEventFieldsInPlace,
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
module GetFinalizedSlot = {
|
|
25
|
+
let route = Rpc.makeRpcRoute(
|
|
26
|
+
"getSlot",
|
|
27
|
+
S.tuple(s => {
|
|
28
|
+
s.tag(0, {"commitment": "finalized"})
|
|
29
|
+
()
|
|
30
|
+
}),
|
|
31
|
+
S.int,
|
|
32
|
+
)
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
let makeRPCSource = (~chain, ~rpc: string): Source.t => {
|
|
36
|
+
let client = Rest.client(rpc)
|
|
37
|
+
|
|
38
|
+
{
|
|
39
|
+
name: "Solana",
|
|
40
|
+
sourceFor: Sync,
|
|
41
|
+
chain,
|
|
42
|
+
poweredByHyperSync: false,
|
|
43
|
+
pollingInterval: 10_000,
|
|
44
|
+
getBlockHashes: (~blockNumbers as _, ~logger as _) =>
|
|
45
|
+
Js.Exn.raiseError("Solana does not support getting block hashes"),
|
|
46
|
+
getHeightOrThrow: () => GetFinalizedSlot.route->Rest.fetch((), ~client),
|
|
47
|
+
getItemsOrThrow: (
|
|
48
|
+
~fromBlock as _,
|
|
49
|
+
~toBlock as _,
|
|
50
|
+
~addressesByContractName as _,
|
|
51
|
+
~indexingContracts as _,
|
|
52
|
+
~knownHeight as _,
|
|
53
|
+
~partitionId as _,
|
|
54
|
+
~selection as _,
|
|
55
|
+
~retry as _,
|
|
56
|
+
~logger as _,
|
|
57
|
+
) => Js.Exn.raiseError("Solana does not support getting items"),
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// Generated by ReScript, PLEASE EDIT WITH CARE
|
|
2
|
+
|
|
3
|
+
import * as Rpc from "./Rpc.res.mjs";
|
|
4
|
+
import * as Rest from "../vendored/Rest.res.mjs";
|
|
5
|
+
import * as Js_exn from "rescript/lib/es6/js_exn.js";
|
|
6
|
+
import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
|
|
7
|
+
|
|
8
|
+
var cleanUpRawEventFieldsInPlace = (fields => {
|
|
9
|
+
delete fields.hash
|
|
10
|
+
delete fields.height
|
|
11
|
+
delete fields.time
|
|
12
|
+
});
|
|
13
|
+
|
|
14
|
+
var ecosystem_blockFields = ["slot"];
|
|
15
|
+
|
|
16
|
+
var ecosystem_transactionFields = [];
|
|
17
|
+
|
|
18
|
+
function ecosystem_getNumber(prim) {
|
|
19
|
+
return prim.height;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function ecosystem_getTimestamp(prim) {
|
|
23
|
+
return prim.time;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function ecosystem_getId(prim) {
|
|
27
|
+
return prim.hash;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
var ecosystem = {
|
|
31
|
+
name: "solana",
|
|
32
|
+
blockFields: ecosystem_blockFields,
|
|
33
|
+
transactionFields: ecosystem_transactionFields,
|
|
34
|
+
blockNumberName: "height",
|
|
35
|
+
blockTimestampName: "time",
|
|
36
|
+
blockHashName: "hash",
|
|
37
|
+
getNumber: ecosystem_getNumber,
|
|
38
|
+
getTimestamp: ecosystem_getTimestamp,
|
|
39
|
+
getId: ecosystem_getId,
|
|
40
|
+
cleanUpRawEventFieldsInPlace: cleanUpRawEventFieldsInPlace
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
var route = Rpc.makeRpcRoute("getSlot", S$RescriptSchema.tuple(function (s) {
|
|
44
|
+
s.tag(0, {
|
|
45
|
+
commitment: "finalized"
|
|
46
|
+
});
|
|
47
|
+
}), S$RescriptSchema.$$int);
|
|
48
|
+
|
|
49
|
+
var GetFinalizedSlot = {
|
|
50
|
+
route: route
|
|
51
|
+
};
|
|
52
|
+
|
|
53
|
+
function makeRPCSource(chain, rpc) {
|
|
54
|
+
var client = Rest.client(rpc, undefined);
|
|
55
|
+
return {
|
|
56
|
+
name: "Solana",
|
|
57
|
+
sourceFor: "Sync",
|
|
58
|
+
chain: chain,
|
|
59
|
+
poweredByHyperSync: false,
|
|
60
|
+
pollingInterval: 10000,
|
|
61
|
+
getBlockHashes: (function (param, param$1) {
|
|
62
|
+
return Js_exn.raiseError("Solana does not support getting block hashes");
|
|
63
|
+
}),
|
|
64
|
+
getHeightOrThrow: (function () {
|
|
65
|
+
return Rest.$$fetch(route, undefined, client);
|
|
66
|
+
}),
|
|
67
|
+
getItemsOrThrow: (function (param, param$1, param$2, param$3, param$4, param$5, param$6, param$7, param$8) {
|
|
68
|
+
return Js_exn.raiseError("Solana does not support getting items");
|
|
69
|
+
})
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export {
|
|
74
|
+
cleanUpRawEventFieldsInPlace ,
|
|
75
|
+
ecosystem ,
|
|
76
|
+
GetFinalizedSlot ,
|
|
77
|
+
makeRPCSource ,
|
|
78
|
+
}
|
|
79
|
+
/* route Not a pure module */
|
package/src/sources/Source.res
CHANGED
|
@@ -11,7 +11,7 @@ type blockRangeFetchStats = {
|
|
|
11
11
|
Thes response returned from a block range fetch
|
|
12
12
|
*/
|
|
13
13
|
type blockRangeFetchResponse = {
|
|
14
|
-
|
|
14
|
+
knownHeight: int,
|
|
15
15
|
reorgGuard: ReorgDetection.reorgGuard,
|
|
16
16
|
parsedQueueItems: array<Internal.item>,
|
|
17
17
|
fromBlockQueried: int,
|
|
@@ -50,7 +50,7 @@ type t = {
|
|
|
50
50
|
~toBlock: option<int>,
|
|
51
51
|
~addressesByContractName: dict<array<Address.t>>,
|
|
52
52
|
~indexingContracts: dict<Internal.indexingContract>,
|
|
53
|
-
~
|
|
53
|
+
~knownHeight: int,
|
|
54
54
|
~partitionId: string,
|
|
55
55
|
~selection: FetchState.selection,
|
|
56
56
|
~retry: int,
|
|
@@ -92,7 +92,6 @@ let trackNewStatus = (sourceManager: t, ~newStatus) => {
|
|
|
92
92
|
let fetchNext = async (
|
|
93
93
|
sourceManager: t,
|
|
94
94
|
~fetchState: FetchState.t,
|
|
95
|
-
~currentBlockHeight,
|
|
96
95
|
~executeQuery,
|
|
97
96
|
~waitForNewBlock,
|
|
98
97
|
~onNewBlock,
|
|
@@ -100,13 +99,14 @@ let fetchNext = async (
|
|
|
100
99
|
) => {
|
|
101
100
|
let {maxPartitionConcurrency} = sourceManager
|
|
102
101
|
|
|
103
|
-
|
|
102
|
+
let nextQuery = fetchState->FetchState.getNextQuery(
|
|
104
103
|
~concurrencyLimit={
|
|
105
104
|
maxPartitionConcurrency - sourceManager.fetchingPartitionsCount
|
|
106
105
|
},
|
|
107
|
-
~currentBlockHeight,
|
|
108
106
|
~stateId,
|
|
109
|
-
)
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
switch nextQuery {
|
|
110
110
|
| ReachedMaxConcurrency
|
|
111
111
|
| NothingToQuery => ()
|
|
112
112
|
| WaitingForNewBlock =>
|
|
@@ -116,12 +116,12 @@ let fetchNext = async (
|
|
|
116
116
|
| None =>
|
|
117
117
|
sourceManager->trackNewStatus(~newStatus=WaitingForNewBlock)
|
|
118
118
|
sourceManager.waitingForNewBlockStateId = Some(stateId)
|
|
119
|
-
let
|
|
119
|
+
let knownHeight = await waitForNewBlock(~knownHeight=fetchState.knownHeight)
|
|
120
120
|
switch sourceManager.waitingForNewBlockStateId {
|
|
121
121
|
| Some(waitingStateId) if waitingStateId === stateId => {
|
|
122
122
|
sourceManager->trackNewStatus(~newStatus=Idle)
|
|
123
123
|
sourceManager.waitingForNewBlockStateId = None
|
|
124
|
-
onNewBlock(~
|
|
124
|
+
onNewBlock(~knownHeight)
|
|
125
125
|
}
|
|
126
126
|
| Some(_) // Don't reset it if we are waiting for another state
|
|
127
127
|
| None => ()
|
|
@@ -162,14 +162,14 @@ type status = Active | Stalled | Done
|
|
|
162
162
|
let getSourceNewHeight = async (
|
|
163
163
|
sourceManager,
|
|
164
164
|
~source: Source.t,
|
|
165
|
-
~
|
|
165
|
+
~knownHeight,
|
|
166
166
|
~status: ref<status>,
|
|
167
167
|
~logger,
|
|
168
168
|
) => {
|
|
169
169
|
let newHeight = ref(0)
|
|
170
170
|
let retry = ref(0)
|
|
171
171
|
|
|
172
|
-
while newHeight.contents <=
|
|
172
|
+
while newHeight.contents <= knownHeight && status.contents !== Done {
|
|
173
173
|
try {
|
|
174
174
|
// Use to detect if the source is taking too long to respond
|
|
175
175
|
let endTimer = Prometheus.SourceGetHeightDuration.startTimer({
|
|
@@ -180,7 +180,7 @@ let getSourceNewHeight = async (
|
|
|
180
180
|
endTimer()
|
|
181
181
|
|
|
182
182
|
newHeight := height
|
|
183
|
-
if height <=
|
|
183
|
+
if height <= knownHeight {
|
|
184
184
|
retry := 0
|
|
185
185
|
// Slowdown polling when the chain isn't progressing
|
|
186
186
|
let pollingInterval = if status.contents === Stalled {
|
|
@@ -211,35 +211,35 @@ let getSourceNewHeight = async (
|
|
|
211
211
|
}
|
|
212
212
|
|
|
213
213
|
// Polls for a block height greater than the given block number to ensure a new block is available for indexing.
|
|
214
|
-
let waitForNewBlock = async (sourceManager: t, ~
|
|
214
|
+
let waitForNewBlock = async (sourceManager: t, ~knownHeight) => {
|
|
215
215
|
let {sources} = sourceManager
|
|
216
216
|
|
|
217
217
|
let logger = Logging.createChild(
|
|
218
218
|
~params={
|
|
219
219
|
"chainId": sourceManager.activeSource.chain->ChainMap.Chain.toChainId,
|
|
220
|
-
"
|
|
220
|
+
"knownHeight": knownHeight,
|
|
221
221
|
},
|
|
222
222
|
)
|
|
223
223
|
logger->Logging.childTrace("Initiating check for new blocks.")
|
|
224
224
|
|
|
225
225
|
// Only include Live sources if we've actually synced some blocks
|
|
226
|
-
// (
|
|
226
|
+
// (knownHeight > 0 means we've fetched at least one batch)
|
|
227
227
|
// This prevents Live RPC from winning the initial height race and
|
|
228
228
|
// becoming activeSource, which would bypass HyperSync's smart block detection
|
|
229
|
-
let isInitialHeightFetch =
|
|
229
|
+
let isInitialHeightFetch = knownHeight === 0
|
|
230
230
|
|
|
231
231
|
let syncSources = []
|
|
232
232
|
let fallbackSources = []
|
|
233
233
|
sources->Utils.Set.forEach(source => {
|
|
234
234
|
if (
|
|
235
235
|
source.sourceFor === Sync ||
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
236
|
+
// Include Live sources only after initial sync has started
|
|
237
|
+
// Live sources are optimized for real-time indexing with lower latency
|
|
238
|
+
source.sourceFor === Live && !isInitialHeightFetch ||
|
|
239
|
+
// Even if the active source is a fallback, still include
|
|
240
|
+
// it to the list. So we don't wait for a timeout again
|
|
241
|
+
// if all main sync sources are still not valid
|
|
242
|
+
source === sourceManager.activeSource
|
|
243
243
|
) {
|
|
244
244
|
syncSources->Array.push(source)
|
|
245
245
|
} else {
|
|
@@ -252,10 +252,7 @@ let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
|
|
|
252
252
|
let (source, newBlockHeight) = await Promise.race(
|
|
253
253
|
syncSources
|
|
254
254
|
->Array.map(async source => {
|
|
255
|
-
(
|
|
256
|
-
source,
|
|
257
|
-
await sourceManager->getSourceNewHeight(~source, ~currentBlockHeight, ~status, ~logger),
|
|
258
|
-
)
|
|
255
|
+
(source, await sourceManager->getSourceNewHeight(~source, ~knownHeight, ~status, ~logger))
|
|
259
256
|
})
|
|
260
257
|
->Array.concat([
|
|
261
258
|
Utils.delay(sourceManager.newBlockFallbackStallTimeout)->Promise.then(() => {
|
|
@@ -281,12 +278,7 @@ let waitForNewBlock = async (sourceManager: t, ~currentBlockHeight) => {
|
|
|
281
278
|
fallbackSources->Array.map(async source => {
|
|
282
279
|
(
|
|
283
280
|
source,
|
|
284
|
-
await sourceManager->getSourceNewHeight(
|
|
285
|
-
~source,
|
|
286
|
-
~currentBlockHeight,
|
|
287
|
-
~status,
|
|
288
|
-
~logger,
|
|
289
|
-
),
|
|
281
|
+
await sourceManager->getSourceNewHeight(~source, ~knownHeight, ~status, ~logger),
|
|
290
282
|
)
|
|
291
283
|
}),
|
|
292
284
|
)
|
|
@@ -349,7 +341,7 @@ let getNextSyncSource = (
|
|
|
349
341
|
}
|
|
350
342
|
}
|
|
351
343
|
|
|
352
|
-
let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~
|
|
344
|
+
let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~knownHeight) => {
|
|
353
345
|
let toBlockRef = ref(
|
|
354
346
|
switch query.target {
|
|
355
347
|
| Head => None
|
|
@@ -389,7 +381,7 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
|
|
|
389
381
|
~addressesByContractName=query.addressesByContractName,
|
|
390
382
|
~indexingContracts=query.indexingContracts,
|
|
391
383
|
~partitionId=query.partitionId,
|
|
392
|
-
~
|
|
384
|
+
~knownHeight,
|
|
393
385
|
~selection=query.selection,
|
|
394
386
|
~retry,
|
|
395
387
|
~logger,
|
|
@@ -68,10 +68,10 @@ function trackNewStatus(sourceManager, newStatus) {
|
|
|
68
68
|
sourceManager.status = newStatus;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
-
async function fetchNext(sourceManager, fetchState,
|
|
72
|
-
var
|
|
73
|
-
if (typeof
|
|
74
|
-
switch (
|
|
71
|
+
async function fetchNext(sourceManager, fetchState, executeQuery, waitForNewBlock, onNewBlock, stateId) {
|
|
72
|
+
var nextQuery = FetchState.getNextQuery(fetchState, sourceManager.maxPartitionConcurrency - sourceManager.fetchingPartitionsCount | 0, stateId);
|
|
73
|
+
if (typeof nextQuery !== "object") {
|
|
74
|
+
switch (nextQuery) {
|
|
75
75
|
case "WaitingForNewBlock" :
|
|
76
76
|
var waitingStateId = sourceManager.waitingForNewBlockStateId;
|
|
77
77
|
if (waitingStateId !== undefined && waitingStateId >= stateId) {
|
|
@@ -79,12 +79,12 @@ async function fetchNext(sourceManager, fetchState, currentBlockHeight, executeQ
|
|
|
79
79
|
}
|
|
80
80
|
trackNewStatus(sourceManager, "WaitingForNewBlock");
|
|
81
81
|
sourceManager.waitingForNewBlockStateId = stateId;
|
|
82
|
-
var
|
|
82
|
+
var knownHeight = await waitForNewBlock(fetchState.knownHeight);
|
|
83
83
|
var waitingStateId$1 = sourceManager.waitingForNewBlockStateId;
|
|
84
84
|
if (waitingStateId$1 !== undefined && waitingStateId$1 === stateId) {
|
|
85
85
|
trackNewStatus(sourceManager, "Idle");
|
|
86
86
|
sourceManager.waitingForNewBlockStateId = undefined;
|
|
87
|
-
return onNewBlock(
|
|
87
|
+
return onNewBlock(knownHeight);
|
|
88
88
|
} else {
|
|
89
89
|
return ;
|
|
90
90
|
}
|
|
@@ -94,12 +94,12 @@ async function fetchNext(sourceManager, fetchState, currentBlockHeight, executeQ
|
|
|
94
94
|
|
|
95
95
|
}
|
|
96
96
|
} else {
|
|
97
|
-
var queries
|
|
98
|
-
FetchState.startFetchingQueries(fetchState, queries
|
|
99
|
-
sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries
|
|
97
|
+
var queries = nextQuery._0;
|
|
98
|
+
FetchState.startFetchingQueries(fetchState, queries, stateId);
|
|
99
|
+
sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount + queries.length | 0;
|
|
100
100
|
Prometheus.IndexingConcurrency.set(sourceManager.fetchingPartitionsCount, sourceManager.activeSource.chain);
|
|
101
101
|
trackNewStatus(sourceManager, "Querieng");
|
|
102
|
-
await Promise.all(Belt_Array.map(queries
|
|
102
|
+
await Promise.all(Belt_Array.map(queries, (function (q) {
|
|
103
103
|
var promise = executeQuery(q);
|
|
104
104
|
promise.then(function (param) {
|
|
105
105
|
sourceManager.fetchingPartitionsCount = sourceManager.fetchingPartitionsCount - 1 | 0;
|
|
@@ -115,10 +115,10 @@ async function fetchNext(sourceManager, fetchState, currentBlockHeight, executeQ
|
|
|
115
115
|
}
|
|
116
116
|
}
|
|
117
117
|
|
|
118
|
-
async function getSourceNewHeight(sourceManager, source,
|
|
118
|
+
async function getSourceNewHeight(sourceManager, source, knownHeight, status, logger) {
|
|
119
119
|
var newHeight = 0;
|
|
120
120
|
var retry = 0;
|
|
121
|
-
while(newHeight <=
|
|
121
|
+
while(newHeight <= knownHeight && status.contents !== "Done") {
|
|
122
122
|
try {
|
|
123
123
|
var endTimer = Prometheus.SourceGetHeightDuration.startTimer({
|
|
124
124
|
source: source.name,
|
|
@@ -127,7 +127,7 @@ async function getSourceNewHeight(sourceManager, source, currentBlockHeight, sta
|
|
|
127
127
|
var height = await source.getHeightOrThrow();
|
|
128
128
|
endTimer();
|
|
129
129
|
newHeight = height;
|
|
130
|
-
if (height <=
|
|
130
|
+
if (height <= knownHeight) {
|
|
131
131
|
retry = 0;
|
|
132
132
|
var pollingInterval = status.contents === "Stalled" ? sourceManager.stalledPollingInterval : source.pollingInterval;
|
|
133
133
|
await Utils.delay(pollingInterval);
|
|
@@ -150,13 +150,13 @@ async function getSourceNewHeight(sourceManager, source, currentBlockHeight, sta
|
|
|
150
150
|
return newHeight;
|
|
151
151
|
}
|
|
152
152
|
|
|
153
|
-
async function waitForNewBlock(sourceManager,
|
|
153
|
+
async function waitForNewBlock(sourceManager, knownHeight) {
|
|
154
154
|
var logger = Logging.createChild({
|
|
155
155
|
chainId: sourceManager.activeSource.chain,
|
|
156
|
-
|
|
156
|
+
knownHeight: knownHeight
|
|
157
157
|
});
|
|
158
158
|
Logging.childTrace(logger, "Initiating check for new blocks.");
|
|
159
|
-
var isInitialHeightFetch =
|
|
159
|
+
var isInitialHeightFetch = knownHeight === 0;
|
|
160
160
|
var syncSources = [];
|
|
161
161
|
var fallbackSources = [];
|
|
162
162
|
sourceManager.sources.forEach(function (source) {
|
|
@@ -172,7 +172,7 @@ async function waitForNewBlock(sourceManager, currentBlockHeight) {
|
|
|
172
172
|
var match = await Promise.race(Belt_Array.concat(Belt_Array.map(syncSources, (async function (source) {
|
|
173
173
|
return [
|
|
174
174
|
source,
|
|
175
|
-
await getSourceNewHeight(sourceManager, source,
|
|
175
|
+
await getSourceNewHeight(sourceManager, source, knownHeight, status, logger)
|
|
176
176
|
];
|
|
177
177
|
})), [Utils.delay(sourceManager.newBlockFallbackStallTimeout).then(function () {
|
|
178
178
|
if (status.contents !== "Done") {
|
|
@@ -186,7 +186,7 @@ async function waitForNewBlock(sourceManager, currentBlockHeight) {
|
|
|
186
186
|
return Promise.race(Belt_Array.map(fallbackSources, (async function (source) {
|
|
187
187
|
return [
|
|
188
188
|
source,
|
|
189
|
-
await getSourceNewHeight(sourceManager, source,
|
|
189
|
+
await getSourceNewHeight(sourceManager, source, knownHeight, status, logger)
|
|
190
190
|
];
|
|
191
191
|
})));
|
|
192
192
|
})]));
|
|
@@ -247,7 +247,7 @@ function getNextSyncSource(sourceManager, initialSource, currentSource, attemptF
|
|
|
247
247
|
}
|
|
248
248
|
}
|
|
249
249
|
|
|
250
|
-
async function executeQuery(sourceManager, query,
|
|
250
|
+
async function executeQuery(sourceManager, query, knownHeight) {
|
|
251
251
|
var match = query.target;
|
|
252
252
|
var toBlockRef;
|
|
253
253
|
toBlockRef = typeof match !== "object" ? undefined : (
|
|
@@ -273,7 +273,7 @@ async function executeQuery(sourceManager, query, currentBlockHeight) {
|
|
|
273
273
|
retry: retry
|
|
274
274
|
});
|
|
275
275
|
try {
|
|
276
|
-
var response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, query.indexingContracts,
|
|
276
|
+
var response = await source.getItemsOrThrow(query.fromBlock, toBlock, query.addressesByContractName, query.indexingContracts, knownHeight, query.partitionId, query.selection, retry, logger);
|
|
277
277
|
Logging.childTrace(logger, {
|
|
278
278
|
msg: "Fetched block range from server",
|
|
279
279
|
toBlock: response.latestFetchedBlockNumber,
|
|
@@ -13,19 +13,18 @@ let getActiveSource: t => Source.t
|
|
|
13
13
|
let fetchNext: (
|
|
14
14
|
t,
|
|
15
15
|
~fetchState: FetchState.t,
|
|
16
|
-
~currentBlockHeight: int,
|
|
17
16
|
~executeQuery: FetchState.query => promise<unit>,
|
|
18
|
-
~waitForNewBlock: (~
|
|
19
|
-
~onNewBlock: (~
|
|
17
|
+
~waitForNewBlock: (~knownHeight: int) => promise<int>,
|
|
18
|
+
~onNewBlock: (~knownHeight: int) => unit,
|
|
20
19
|
~stateId: int,
|
|
21
20
|
) => promise<unit>
|
|
22
21
|
|
|
23
|
-
let waitForNewBlock: (t, ~
|
|
22
|
+
let waitForNewBlock: (t, ~knownHeight: int) => promise<int>
|
|
24
23
|
|
|
25
24
|
let executeQuery: (
|
|
26
25
|
t,
|
|
27
26
|
~query: FetchState.query,
|
|
28
|
-
~
|
|
27
|
+
~knownHeight: int,
|
|
29
28
|
) => promise<Source.blockRangeFetchResponse>
|
|
30
29
|
|
|
31
30
|
let makeGetHeightRetryInterval: (
|