envio 3.3.0-alpha.6 → 3.3.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +23 -5
- package/package.json +6 -6
- package/src/Address.res +5 -2
- package/src/Address.res.mjs +3 -1
- package/src/Api.res +1 -1
- package/src/Api.res.mjs +1 -1
- package/src/ChainFetching.res +34 -34
- package/src/ChainFetching.res.mjs +29 -29
- package/src/ChainState.res +328 -165
- package/src/ChainState.res.mjs +205 -100
- package/src/ChainState.resi +16 -7
- package/src/Config.res +55 -27
- package/src/Config.res.mjs +46 -7
- package/src/ContractRegisterContext.res +2 -12
- package/src/ContractRegisterContext.res.mjs +3 -5
- package/src/Core.res +7 -0
- package/src/CrossChainState.res +54 -79
- package/src/CrossChainState.res.mjs +45 -60
- package/src/CrossChainState.resi +1 -1
- package/src/Ecosystem.res +3 -3
- package/src/Ecosystem.res.mjs +3 -3
- package/src/Envio.res +14 -9
- package/src/EventConfigBuilder.res +105 -56
- package/src/EventConfigBuilder.res.mjs +69 -33
- package/src/EventProcessing.res +19 -15
- package/src/EventProcessing.res.mjs +13 -12
- package/src/FetchState.res +383 -182
- package/src/FetchState.res.mjs +260 -264
- package/src/HandlerLoader.res +7 -112
- package/src/HandlerLoader.res.mjs +1 -87
- package/src/HandlerRegister.res +209 -6
- package/src/HandlerRegister.res.mjs +90 -5
- package/src/HandlerRegister.resi +13 -2
- package/src/IndexerState.res +6 -3
- package/src/IndexerState.res.mjs +3 -3
- package/src/IndexerState.resi +1 -1
- package/src/IndexingAddresses.res +10 -7
- package/src/IndexingAddresses.res.mjs +8 -7
- package/src/IndexingAddresses.resi +3 -1
- package/src/Internal.res +82 -36
- package/src/Internal.res.mjs +11 -1
- package/src/Main.res +22 -23
- package/src/Main.res.mjs +15 -17
- package/src/Metrics.res +43 -2
- package/src/Metrics.res.mjs +39 -3
- package/src/Prometheus.res +0 -35
- package/src/Prometheus.res.mjs +33 -68
- package/src/RawEvent.res +7 -2
- package/src/RawEvent.res.mjs +4 -4
- package/src/SimulateItems.res +26 -8
- package/src/SimulateItems.res.mjs +11 -14
- package/src/TestIndexer.res +1 -1
- package/src/TestIndexer.res.mjs +1 -1
- package/src/sources/BlockStore.res +46 -0
- package/src/sources/BlockStore.res.mjs +24 -0
- package/src/sources/EventRouter.res +19 -12
- package/src/sources/EventRouter.res.mjs +7 -5
- package/src/sources/Evm.res +54 -4
- package/src/sources/Evm.res.mjs +43 -4
- package/src/sources/EvmChain.res +14 -18
- package/src/sources/EvmChain.res.mjs +12 -13
- package/src/sources/FieldMask.res +39 -0
- package/src/sources/FieldMask.res.mjs +42 -0
- package/src/sources/Fuel.res +5 -2
- package/src/sources/Fuel.res.mjs +4 -3
- package/src/sources/HyperFuelSource.res +32 -40
- package/src/sources/HyperFuelSource.res.mjs +53 -45
- package/src/sources/HyperSync.res +35 -9
- package/src/sources/HyperSync.res.mjs +52 -35
- package/src/sources/HyperSync.resi +11 -3
- package/src/sources/HyperSyncClient.res +15 -4
- package/src/sources/HyperSyncHeightStream.res +1 -8
- package/src/sources/HyperSyncHeightStream.res.mjs +0 -2
- package/src/sources/HyperSyncSource.res +39 -58
- package/src/sources/HyperSyncSource.res.mjs +44 -44
- package/src/sources/RpcSource.res +114 -92
- package/src/sources/RpcSource.res.mjs +77 -46
- package/src/sources/RpcWebSocketHeightStream.res +0 -5
- package/src/sources/RpcWebSocketHeightStream.res.mjs +0 -2
- package/src/sources/SimulateSource.res +6 -3
- package/src/sources/SimulateSource.res.mjs +13 -5
- package/src/sources/Source.res +25 -5
- package/src/sources/SourceManager.res +66 -3
- package/src/sources/SourceManager.res.mjs +55 -7
- package/src/sources/SourceManager.resi +10 -0
- package/src/sources/Svm.res +15 -10
- package/src/sources/Svm.res.mjs +24 -7
- package/src/sources/SvmHyperSyncClient.res +5 -6
- package/src/sources/SvmHyperSyncSource.res +84 -41
- package/src/sources/SvmHyperSyncSource.res.mjs +90 -43
- package/src/sources/TransactionStore.res +6 -107
- package/src/sources/TransactionStore.res.mjs +5 -86
- package/svm.schema.json +19 -0
|
@@ -48,18 +48,16 @@ let getKnownRawBlockWithBackoff = async (
|
|
|
48
48
|
~chain,
|
|
49
49
|
~blockNumber,
|
|
50
50
|
~backoffMsOnFailure,
|
|
51
|
+
~recordRequest: (~method: string, ~seconds: float) => unit,
|
|
51
52
|
) => {
|
|
52
53
|
let currentBackoff = ref(backoffMsOnFailure)
|
|
53
54
|
let result = ref(None)
|
|
54
55
|
|
|
55
56
|
while result.contents->Option.isNone {
|
|
56
|
-
|
|
57
|
-
~sourceName,
|
|
58
|
-
~chainId=chain->ChainMap.Chain.toChainId,
|
|
59
|
-
~method="eth_getBlockByNumber",
|
|
60
|
-
)
|
|
57
|
+
let timerRef = Performance.now()
|
|
61
58
|
switch await getKnownRawBlock(~client, ~blockNumber) {
|
|
62
59
|
| exception err =>
|
|
60
|
+
recordRequest(~method="eth_getBlockByNumber", ~seconds=timerRef->Performance.secondsSince)
|
|
63
61
|
Logging.warn({
|
|
64
62
|
"err": err->Utils.prettifyExn,
|
|
65
63
|
"msg": `Issue while running fetching batch of events from the RPC. Will wait ${currentBackoff.contents->Int.toString}ms and try again.`,
|
|
@@ -69,7 +67,9 @@ let getKnownRawBlockWithBackoff = async (
|
|
|
69
67
|
})
|
|
70
68
|
await Time.resolvePromiseAfterDelay(~delayMilliseconds=currentBackoff.contents)
|
|
71
69
|
currentBackoff := currentBackoff.contents * 2
|
|
72
|
-
| json =>
|
|
70
|
+
| json =>
|
|
71
|
+
recordRequest(~method="eth_getBlockByNumber", ~seconds=timerRef->Performance.secondsSince)
|
|
72
|
+
result := Some(json)
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
75
|
result.contents->Option.getOrThrow
|
|
@@ -97,16 +97,24 @@ let getErrorMessage = (exn: exn): option<string> =>
|
|
|
97
97
|
// never helps — the same range always re-trips the same cap. The reaction is to
|
|
98
98
|
// shrink the range and retry immediately, ratcheting the max range down.
|
|
99
99
|
let isResponseTooLargeError = {
|
|
100
|
+
// Provider-specific "too many logs" messages, matched loosely since each names
|
|
101
|
+
// it differently: HyperRPC ("More than 50000 logs returned"), ZkEVM ("query
|
|
102
|
+
// returned more than N results"), LlamaRPC ("query exceeds max results"),
|
|
103
|
+
// 1RPC ("response size should not..."), Optimism ("(backend) response too
|
|
104
|
+
// large"), Arbitrum ("logs matched by query exceeds limit"), Ankr ("block
|
|
105
|
+
// range is too wide"). Kept as one block rather than per-line comments — the
|
|
106
|
+
// ReScript formatter doesn't preserve comments attached to regex literals in
|
|
107
|
+
// an array.
|
|
100
108
|
let patterns = [
|
|
101
|
-
/more than \d+ logs/i,
|
|
109
|
+
/more than \d+ logs/i,
|
|
102
110
|
/\d+ logs returned/i,
|
|
103
111
|
/too many logs/i,
|
|
104
|
-
/query returned more than \d+ results/i,
|
|
105
|
-
/query exceeds max results/i,
|
|
106
|
-
/response size should not/i,
|
|
107
|
-
/(backend )?response too large/i,
|
|
108
|
-
/logs matched by query exceeds limit/i,
|
|
109
|
-
/block range is too wide/i,
|
|
112
|
+
/query returned more than \d+ results/i,
|
|
113
|
+
/query exceeds max results/i,
|
|
114
|
+
/response size should not/i,
|
|
115
|
+
/(backend )?response too large/i,
|
|
116
|
+
/logs matched by query exceeds limit/i,
|
|
117
|
+
/block range is too wide/i,
|
|
110
118
|
]
|
|
111
119
|
(exn: exn) =>
|
|
112
120
|
switch exn->getErrorMessage {
|
|
@@ -292,8 +300,7 @@ let getNextPage = (
|
|
|
292
300
|
~rpcClient: EvmRpcClient.t,
|
|
293
301
|
~mutSuggestedBlockIntervals,
|
|
294
302
|
~partitionId,
|
|
295
|
-
~
|
|
296
|
-
~chainId,
|
|
303
|
+
~recordRequest: (~method: string, ~seconds: float) => unit,
|
|
297
304
|
): promise<eventBatchQuery> => {
|
|
298
305
|
//If the query hangs for longer than this, reject this promise to reduce the block interval
|
|
299
306
|
let queryTimoutPromise =
|
|
@@ -306,7 +313,7 @@ let getNextPage = (
|
|
|
306
313
|
let latestFetchedBlockPromise = loadBlock(toBlock)
|
|
307
314
|
|
|
308
315
|
let queryLogs = ({addresses, topicQuery}: logSelection) => {
|
|
309
|
-
|
|
316
|
+
let timerRef = Performance.now()
|
|
310
317
|
rpcClient.getLogs({
|
|
311
318
|
fromBlock,
|
|
312
319
|
toBlock,
|
|
@@ -318,6 +325,9 @@ let getNextPage = (
|
|
|
318
325
|
| Multiple(topics) => Nullable.make(topics)
|
|
319
326
|
}
|
|
320
327
|
),
|
|
328
|
+
})->Promise.thenResolve(items => {
|
|
329
|
+
recordRequest(~method="eth_getLogs", ~seconds=timerRef->Performance.secondsSince)
|
|
330
|
+
items
|
|
321
331
|
})
|
|
322
332
|
}
|
|
323
333
|
|
|
@@ -353,8 +363,10 @@ let getNextPage = (
|
|
|
353
363
|
->Promise.race
|
|
354
364
|
->Promise.catch(err => {
|
|
355
365
|
let executedBlockInterval = toBlock - fromBlock + 1
|
|
356
|
-
let shrunkBlockInterval =
|
|
357
|
-
|
|
366
|
+
let shrunkBlockInterval = Pervasives.max(
|
|
367
|
+
1,
|
|
368
|
+
(executedBlockInterval->Int.toFloat *. sc.backoffMultiplicative)->Float.toInt,
|
|
369
|
+
)
|
|
358
370
|
|
|
359
371
|
let throwFailedGettingItems = retry =>
|
|
360
372
|
throw(Source.GetItemsError(FailedGettingItems({exn: err, attemptedToBlock: toBlock, retry})))
|
|
@@ -401,12 +413,12 @@ type selectionConfig = {
|
|
|
401
413
|
}
|
|
402
414
|
|
|
403
415
|
let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
|
|
404
|
-
let
|
|
405
|
-
selection.
|
|
406
|
-
Utils.magic: array<Internal.
|
|
416
|
+
let evmOnEventRegistrations =
|
|
417
|
+
selection.onEventRegistrations->(
|
|
418
|
+
Utils.magic: array<Internal.onEventRegistration> => array<Internal.evmOnEventRegistration>
|
|
407
419
|
)
|
|
408
420
|
|
|
409
|
-
if
|
|
421
|
+
if evmOnEventRegistrations->Utils.Array.isEmpty {
|
|
410
422
|
throw(
|
|
411
423
|
Source.GetItemsError(
|
|
412
424
|
UnsupportedSelection({
|
|
@@ -429,12 +441,9 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
|
|
|
429
441
|
let dynamicWildcardByContract = Dict.make()
|
|
430
442
|
let contractNames = Utils.Set.make()
|
|
431
443
|
|
|
432
|
-
|
|
433
|
-
contractName
|
|
434
|
-
isWildcard,
|
|
435
|
-
dependsOnAddresses,
|
|
436
|
-
getEventFiltersOrThrow,
|
|
437
|
-
}) => {
|
|
444
|
+
evmOnEventRegistrations->Array.forEach(reg => {
|
|
445
|
+
let contractName = reg.eventConfig.contractName
|
|
446
|
+
let {isWildcard, dependsOnAddresses, getEventFiltersOrThrow} = reg
|
|
438
447
|
let eventFilters = getEventFiltersOrThrow(chain)
|
|
439
448
|
if dependsOnAddresses {
|
|
440
449
|
contractNames->Utils.Set.add(contractName)->ignore
|
|
@@ -664,11 +673,9 @@ let makeThrowingGetEventBlock = (
|
|
|
664
673
|
}
|
|
665
674
|
}
|
|
666
675
|
|
|
667
|
-
// `number
|
|
668
|
-
//
|
|
676
|
+
// `number` is always part of the selected block fields, so it can be read
|
|
677
|
+
// from the assembled block for the item's own `blockNumber`.
|
|
669
678
|
@get external getBlockNumber: Internal.eventBlock => int = "number"
|
|
670
|
-
@get external getBlockTimestamp: Internal.eventBlock => int = "timestamp"
|
|
671
|
-
@get external getBlockHash: Internal.eventBlock => string = "hash"
|
|
672
679
|
|
|
673
680
|
// Field source classification for RPC calls
|
|
674
681
|
type fieldSource = TransactionOnly | ReceiptOnly | Both
|
|
@@ -984,7 +991,7 @@ type options = {
|
|
|
984
991
|
syncConfig: Config.sourceSync,
|
|
985
992
|
url: string,
|
|
986
993
|
chain: ChainMap.Chain.t,
|
|
987
|
-
eventRouter: EventRouter.t<Internal.
|
|
994
|
+
eventRouter: EventRouter.t<Internal.evmOnEventRegistration>,
|
|
988
995
|
allEventParams: array<HyperSyncClient.Decoder.eventParamsInput>,
|
|
989
996
|
lowercaseAddresses: bool,
|
|
990
997
|
ws?: string,
|
|
@@ -1030,15 +1037,37 @@ let make = (
|
|
|
1030
1037
|
~headers?,
|
|
1031
1038
|
)
|
|
1032
1039
|
|
|
1040
|
+
// Requests are made from shared, memoized loaders, so they can't be
|
|
1041
|
+
// attributed to a single getItemsOrThrow/getHeightOrThrow/getBlockHashes
|
|
1042
|
+
// call at its call site. Every actual request (cache/dedup hits never reach
|
|
1043
|
+
// recordRequest) pushes here; each method drains whatever is pending when it
|
|
1044
|
+
// returns. Since a push always lands in exactly one drain, per-source totals
|
|
1045
|
+
// stay exact even with concurrent in-flight calls — which call happens to
|
|
1046
|
+
// drain a given entry doesn't matter, since SourceManager aggregates by
|
|
1047
|
+
// (source, method) regardless of which call returned it.
|
|
1048
|
+
let pendingRequestStats: array<Source.requestStat> = []
|
|
1049
|
+
let recordRequest = (~method, ~seconds) => {
|
|
1050
|
+
pendingRequestStats->Array.push({Source.method, seconds})->ignore
|
|
1051
|
+
}
|
|
1052
|
+
let drainRequestStats = () => {
|
|
1053
|
+
let stats = pendingRequestStats->Utils.Array.copy
|
|
1054
|
+
pendingRequestStats->Utils.Array.clearInPlace
|
|
1055
|
+
stats
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1033
1058
|
let makeTransactionLoader = () =>
|
|
1034
1059
|
LazyLoader.make(
|
|
1035
1060
|
~loaderFn=transactionHash => {
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1061
|
+
let timerRef = Performance.now()
|
|
1062
|
+
Rpc.GetTransactionByHash.rawRoute
|
|
1063
|
+
->Rest.fetch(transactionHash, ~client)
|
|
1064
|
+
->Promise.thenResolve(res => {
|
|
1065
|
+
recordRequest(
|
|
1066
|
+
~method="eth_getTransactionByHash",
|
|
1067
|
+
~seconds=timerRef->Performance.secondsSince,
|
|
1068
|
+
)
|
|
1069
|
+
res
|
|
1070
|
+
})
|
|
1042
1071
|
},
|
|
1043
1072
|
~onError=(am, ~exn) => {
|
|
1044
1073
|
Logging.error({
|
|
@@ -1066,6 +1095,7 @@ let make = (
|
|
|
1066
1095
|
~chain,
|
|
1067
1096
|
~backoffMsOnFailure=1000,
|
|
1068
1097
|
~blockNumber,
|
|
1098
|
+
~recordRequest,
|
|
1069
1099
|
)
|
|
1070
1100
|
},
|
|
1071
1101
|
~onError=(am, ~exn) => {
|
|
@@ -1088,12 +1118,16 @@ let make = (
|
|
|
1088
1118
|
let makeReceiptLoader = () =>
|
|
1089
1119
|
LazyLoader.make(
|
|
1090
1120
|
~loaderFn=transactionHash => {
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1121
|
+
let timerRef = Performance.now()
|
|
1122
|
+
Rpc.GetTransactionReceipt.rawRoute
|
|
1123
|
+
->Rest.fetch(transactionHash, ~client)
|
|
1124
|
+
->Promise.thenResolve(res => {
|
|
1125
|
+
recordRequest(
|
|
1126
|
+
~method="eth_getTransactionReceipt",
|
|
1127
|
+
~seconds=timerRef->Performance.secondsSince,
|
|
1128
|
+
)
|
|
1129
|
+
res
|
|
1130
|
+
})
|
|
1097
1131
|
},
|
|
1098
1132
|
~onError=(am, ~exn) => {
|
|
1099
1133
|
Logging.error({
|
|
@@ -1152,13 +1186,16 @@ let make = (
|
|
|
1152
1186
|
~knownHeight,
|
|
1153
1187
|
~partitionId,
|
|
1154
1188
|
~selection: FetchState.selection,
|
|
1189
|
+
~itemsTarget as _,
|
|
1155
1190
|
~retry,
|
|
1156
1191
|
~logger as _,
|
|
1157
1192
|
) => {
|
|
1158
1193
|
let startFetchingBatchTimeRef = Performance.now()
|
|
1159
1194
|
|
|
1160
1195
|
let sourceMaxBlockInterval =
|
|
1161
|
-
mutSuggestedBlockIntervals->getSourceMaxBlockInterval(
|
|
1196
|
+
mutSuggestedBlockIntervals->getSourceMaxBlockInterval(
|
|
1197
|
+
~intervalCeiling=syncConfig.intervalCeiling,
|
|
1198
|
+
)
|
|
1162
1199
|
let suggestedBlockInterval = Pervasives.min(
|
|
1163
1200
|
mutSuggestedBlockIntervals
|
|
1164
1201
|
->Utils.Dict.dangerouslyGetNonOption(partitionId)
|
|
@@ -1198,8 +1235,7 @@ let make = (
|
|
|
1198
1235
|
~rpcClient,
|
|
1199
1236
|
~mutSuggestedBlockIntervals,
|
|
1200
1237
|
~partitionId,
|
|
1201
|
-
~
|
|
1202
|
-
~chainId=chain->ChainMap.Chain.toChainId,
|
|
1238
|
+
~recordRequest,
|
|
1203
1239
|
)
|
|
1204
1240
|
|
|
1205
1241
|
let executedBlockInterval = suggestedToBlock - fromBlock + 1
|
|
@@ -1233,7 +1269,11 @@ let make = (
|
|
|
1233
1269
|
~contractAddress=routedAddress,
|
|
1234
1270
|
) {
|
|
1235
1271
|
| None => None
|
|
1236
|
-
| Some(
|
|
1272
|
+
| Some(onEventRegistration) =>
|
|
1273
|
+
let eventConfig =
|
|
1274
|
+
onEventRegistration.eventConfig->(
|
|
1275
|
+
Utils.magic: Internal.eventConfig => Internal.evmEventConfig
|
|
1276
|
+
)
|
|
1237
1277
|
switch maybeDecodedEvent
|
|
1238
1278
|
->Nullable.toOption
|
|
1239
1279
|
->Option.flatMap(Dict.get(_, eventConfig.contractName)) {
|
|
@@ -1280,10 +1320,8 @@ let make = (
|
|
|
1280
1320
|
}
|
|
1281
1321
|
|
|
1282
1322
|
Internal.Event({
|
|
1283
|
-
|
|
1284
|
-
timestamp: block->getBlockTimestamp,
|
|
1323
|
+
onEventRegistration: (onEventRegistration :> Internal.onEventRegistration),
|
|
1285
1324
|
blockNumber: block->getBlockNumber,
|
|
1286
|
-
blockHash: block->getBlockHash,
|
|
1287
1325
|
chain,
|
|
1288
1326
|
logIndex: log.logIndex,
|
|
1289
1327
|
transactionIndex: log.transactionIndex,
|
|
@@ -1337,14 +1375,16 @@ let make = (
|
|
|
1337
1375
|
latestFetchedBlockTimestamp: latestFetchedBlockInfo.timestamp,
|
|
1338
1376
|
latestFetchedBlockNumber: latestFetchedBlockInfo.number,
|
|
1339
1377
|
parsedQueueItems,
|
|
1340
|
-
// RPC keeps the transaction inline on the payload; no store
|
|
1378
|
+
// RPC keeps the transaction and block inline on the payload; no store pages.
|
|
1341
1379
|
transactionStore: None,
|
|
1380
|
+
blockStore: None,
|
|
1342
1381
|
stats: {
|
|
1343
1382
|
totalTimeElapsed: totalTimeElapsed,
|
|
1344
1383
|
},
|
|
1345
1384
|
knownHeight,
|
|
1346
1385
|
blockHashes,
|
|
1347
1386
|
fromBlockQueried: fromBlock,
|
|
1387
|
+
requestStats: drainRequestStats(),
|
|
1348
1388
|
}
|
|
1349
1389
|
}
|
|
1350
1390
|
|
|
@@ -1361,21 +1401,25 @@ let make = (
|
|
|
1361
1401
|
->Array.map(blockNum => blockLoader.contents->LazyLoader.get(blockNum))
|
|
1362
1402
|
->Promise.all
|
|
1363
1403
|
->Promise.thenResolve(rawBlocks => {
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1404
|
+
let result =
|
|
1405
|
+
rawBlocks
|
|
1406
|
+
->Array.map(json => {
|
|
1407
|
+
let b = parseBlockInfo(json)
|
|
1408
|
+
|
|
1409
|
+
(
|
|
1410
|
+
{
|
|
1411
|
+
blockNumber: b.number,
|
|
1412
|
+
blockHash: b.hash,
|
|
1413
|
+
blockTimestamp: b.timestamp,
|
|
1414
|
+
}: ReorgDetection.blockDataWithTimestamp
|
|
1415
|
+
)
|
|
1416
|
+
})
|
|
1417
|
+
->Ok
|
|
1418
|
+
{Source.result, requestStats: drainRequestStats()}
|
|
1377
1419
|
})
|
|
1378
|
-
->Promise.catch(exn =>
|
|
1420
|
+
->Promise.catch(exn =>
|
|
1421
|
+
{Source.result: Error(exn), requestStats: drainRequestStats()}->Promise.resolve
|
|
1422
|
+
)
|
|
1379
1423
|
}
|
|
1380
1424
|
|
|
1381
1425
|
let createHeightSubscription =
|
|
@@ -1397,33 +1441,11 @@ let make = (
|
|
|
1397
1441
|
await rpcClient.getHeight()
|
|
1398
1442
|
} catch {
|
|
1399
1443
|
| exn =>
|
|
1400
|
-
|
|
1401
|
-
Prometheus.SourceRequestCount.increment(
|
|
1402
|
-
~sourceName=name,
|
|
1403
|
-
~chainId=chain->ChainMap.Chain.toChainId,
|
|
1404
|
-
~method="eth_blockNumber",
|
|
1405
|
-
)
|
|
1406
|
-
Prometheus.SourceRequestCount.addSeconds(
|
|
1407
|
-
~sourceName=name,
|
|
1408
|
-
~chainId=chain->ChainMap.Chain.toChainId,
|
|
1409
|
-
~method="eth_blockNumber",
|
|
1410
|
-
~seconds,
|
|
1411
|
-
)
|
|
1444
|
+
recordRequest(~method="eth_blockNumber", ~seconds=timerRef->Performance.secondsSince)
|
|
1412
1445
|
exn->throw
|
|
1413
1446
|
}
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
~sourceName=name,
|
|
1417
|
-
~chainId=chain->ChainMap.Chain.toChainId,
|
|
1418
|
-
~method="eth_blockNumber",
|
|
1419
|
-
)
|
|
1420
|
-
Prometheus.SourceRequestCount.addSeconds(
|
|
1421
|
-
~sourceName=name,
|
|
1422
|
-
~chainId=chain->ChainMap.Chain.toChainId,
|
|
1423
|
-
~method="eth_blockNumber",
|
|
1424
|
-
~seconds,
|
|
1425
|
-
)
|
|
1426
|
-
height
|
|
1447
|
+
recordRequest(~method="eth_blockNumber", ~seconds=timerRef->Performance.secondsSince)
|
|
1448
|
+
{height, requestStats: drainRequestStats()}
|
|
1427
1449
|
},
|
|
1428
1450
|
getItemsOrThrow,
|
|
1429
1451
|
?createHeightSubscription,
|
|
@@ -9,7 +9,6 @@ import * as Address from "../Address.res.mjs";
|
|
|
9
9
|
import * as Logging from "../Logging.res.mjs";
|
|
10
10
|
import * as Internal from "../Internal.res.mjs";
|
|
11
11
|
import * as LazyLoader from "../LazyLoader.res.mjs";
|
|
12
|
-
import * as Prometheus from "../Prometheus.res.mjs";
|
|
13
12
|
import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
|
|
14
13
|
import * as EventRouter from "./EventRouter.res.mjs";
|
|
15
14
|
import * as Performance from "../bindings/Performance.res.mjs";
|
|
@@ -47,11 +46,11 @@ function parseBlockInfo(json) {
|
|
|
47
46
|
};
|
|
48
47
|
}
|
|
49
48
|
|
|
50
|
-
async function getKnownRawBlockWithBackoff(client, sourceName, chain, blockNumber, backoffMsOnFailure) {
|
|
49
|
+
async function getKnownRawBlockWithBackoff(client, sourceName, chain, blockNumber, backoffMsOnFailure, recordRequest) {
|
|
51
50
|
let currentBackoff = backoffMsOnFailure;
|
|
52
51
|
let result;
|
|
53
52
|
while (Stdlib_Option.isNone(result)) {
|
|
54
|
-
|
|
53
|
+
let timerRef = Performance.now();
|
|
55
54
|
let exit = 0;
|
|
56
55
|
let json;
|
|
57
56
|
try {
|
|
@@ -59,6 +58,7 @@ async function getKnownRawBlockWithBackoff(client, sourceName, chain, blockNumbe
|
|
|
59
58
|
exit = 1;
|
|
60
59
|
} catch (raw_err) {
|
|
61
60
|
let err = Primitive_exceptions.internalToException(raw_err);
|
|
61
|
+
recordRequest("eth_getBlockByNumber", Performance.secondsSince(timerRef));
|
|
62
62
|
Logging.warn({
|
|
63
63
|
err: Utils.prettifyExn(err),
|
|
64
64
|
msg: `Issue while running fetching batch of events from the RPC. Will wait ` + currentBackoff.toString() + `ms and try again.`,
|
|
@@ -70,6 +70,7 @@ async function getKnownRawBlockWithBackoff(client, sourceName, chain, blockNumbe
|
|
|
70
70
|
currentBackoff = (currentBackoff << 1);
|
|
71
71
|
}
|
|
72
72
|
if (exit === 1) {
|
|
73
|
+
recordRequest("eth_getBlockByNumber", Performance.secondsSince(timerRef));
|
|
73
74
|
result = json;
|
|
74
75
|
}
|
|
75
76
|
};
|
|
@@ -265,14 +266,14 @@ function mergeAndDedupItems(itemsPerSelection) {
|
|
|
265
266
|
return merged;
|
|
266
267
|
}
|
|
267
268
|
|
|
268
|
-
function getNextPage(fromBlock, toBlock, logSelections, loadBlock, sc, rpcClient, mutSuggestedBlockIntervals, partitionId,
|
|
269
|
+
function getNextPage(fromBlock, toBlock, logSelections, loadBlock, sc, rpcClient, mutSuggestedBlockIntervals, partitionId, recordRequest) {
|
|
269
270
|
let queryTimoutPromise = Time.resolvePromiseAfterDelay(sc.queryTimeoutMillis).then(() => Promise.reject({
|
|
270
271
|
RE_EXN_ID: QueryTimout,
|
|
271
272
|
_1: `Query took longer than ` + (sc.queryTimeoutMillis / 1000 | 0).toString() + ` seconds`
|
|
272
273
|
}));
|
|
273
274
|
let latestFetchedBlockPromise = loadBlock(toBlock);
|
|
274
275
|
let queryLogs = param => {
|
|
275
|
-
|
|
276
|
+
let timerRef = Performance.now();
|
|
276
277
|
return rpcClient.getLogs({
|
|
277
278
|
fromBlock: fromBlock,
|
|
278
279
|
toBlock: toBlock,
|
|
@@ -286,6 +287,9 @@ function getNextPage(fromBlock, toBlock, logSelections, loadBlock, sc, rpcClient
|
|
|
286
287
|
return filter;
|
|
287
288
|
}
|
|
288
289
|
})
|
|
290
|
+
}).then(items => {
|
|
291
|
+
recordRequest("eth_getLogs", Performance.secondsSince(timerRef));
|
|
292
|
+
return items;
|
|
289
293
|
});
|
|
290
294
|
};
|
|
291
295
|
let len = logSelections.length;
|
|
@@ -353,8 +357,8 @@ function getNextPage(fromBlock, toBlock, logSelections, loadBlock, sc, rpcClient
|
|
|
353
357
|
}
|
|
354
358
|
|
|
355
359
|
function getSelectionConfig(selection, chain) {
|
|
356
|
-
let
|
|
357
|
-
if (Utils.$$Array.isEmpty(
|
|
360
|
+
let evmOnEventRegistrations = selection.onEventRegistrations;
|
|
361
|
+
if (Utils.$$Array.isEmpty(evmOnEventRegistrations)) {
|
|
358
362
|
throw {
|
|
359
363
|
RE_EXN_ID: Source.GetItemsError,
|
|
360
364
|
_1: {
|
|
@@ -369,15 +373,15 @@ function getSelectionConfig(selection, chain) {
|
|
|
369
373
|
let dynamicByContract = {};
|
|
370
374
|
let dynamicWildcardByContract = {};
|
|
371
375
|
let contractNames = new Set();
|
|
372
|
-
|
|
373
|
-
let contractName =
|
|
374
|
-
let eventFilters =
|
|
375
|
-
if (
|
|
376
|
+
evmOnEventRegistrations.forEach(reg => {
|
|
377
|
+
let contractName = reg.eventConfig.contractName;
|
|
378
|
+
let eventFilters = reg.getEventFiltersOrThrow(chain);
|
|
379
|
+
if (reg.dependsOnAddresses) {
|
|
376
380
|
contractNames.add(contractName);
|
|
377
381
|
if (eventFilters.TAG === "Static") {
|
|
378
382
|
return Utils.Dict.pushMany(staticByContract, contractName, eventFilters._0);
|
|
379
383
|
} else {
|
|
380
|
-
return Utils.Dict.push(
|
|
384
|
+
return Utils.Dict.push(reg.isWildcard ? dynamicWildcardByContract : dynamicByContract, contractName, eventFilters._0);
|
|
381
385
|
}
|
|
382
386
|
}
|
|
383
387
|
let tmp;
|
|
@@ -973,9 +977,24 @@ function make(param) {
|
|
|
973
977
|
let mutSuggestedBlockIntervals = {};
|
|
974
978
|
let client = Rpc.makeClient(url, headers);
|
|
975
979
|
let rpcClient = EvmRpcClient.make(url, !lowercaseAddresses, undefined, headers, param.allEventParams);
|
|
980
|
+
let pendingRequestStats = [];
|
|
981
|
+
let recordRequest = (method, seconds) => {
|
|
982
|
+
pendingRequestStats.push({
|
|
983
|
+
method: method,
|
|
984
|
+
seconds: seconds
|
|
985
|
+
});
|
|
986
|
+
};
|
|
987
|
+
let drainRequestStats = () => {
|
|
988
|
+
let stats = pendingRequestStats.slice();
|
|
989
|
+
Utils.$$Array.clearInPlace(pendingRequestStats);
|
|
990
|
+
return stats;
|
|
991
|
+
};
|
|
976
992
|
let makeTransactionLoader = () => LazyLoader.make(transactionHash => {
|
|
977
|
-
|
|
978
|
-
return Rest.fetch(Rpc.GetTransactionByHash.rawRoute, transactionHash, client)
|
|
993
|
+
let timerRef = Performance.now();
|
|
994
|
+
return Rest.fetch(Rpc.GetTransactionByHash.rawRoute, transactionHash, client).then(res => {
|
|
995
|
+
recordRequest("eth_getTransactionByHash", Performance.secondsSince(timerRef));
|
|
996
|
+
return res;
|
|
997
|
+
});
|
|
979
998
|
}, (am, exn) => Logging.error({
|
|
980
999
|
err: Utils.prettifyExn(exn),
|
|
981
1000
|
msg: `Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ` + (am._retryDelayMillis / 1000 | 0).toString() + ` seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the "suggestedFix" in the metadata of this command`,
|
|
@@ -986,7 +1005,7 @@ function make(param) {
|
|
|
986
1005
|
suggestedFix: "This likely means the RPC url you are using is not responding correctly. Please try another RPC endipoint."
|
|
987
1006
|
}
|
|
988
1007
|
}), undefined, undefined, undefined, undefined);
|
|
989
|
-
let makeBlockLoader = () => LazyLoader.make(blockNumber => getKnownRawBlockWithBackoff(client, name, chain, blockNumber, 1000), (am, exn) => Logging.error({
|
|
1008
|
+
let makeBlockLoader = () => LazyLoader.make(blockNumber => getKnownRawBlockWithBackoff(client, name, chain, blockNumber, 1000, recordRequest), (am, exn) => Logging.error({
|
|
990
1009
|
err: Utils.prettifyExn(exn),
|
|
991
1010
|
msg: `Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ` + (am._retryDelayMillis / 1000 | 0).toString() + ` seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the "suggestedFix" in the metadata of this command`,
|
|
992
1011
|
source: name,
|
|
@@ -997,8 +1016,11 @@ function make(param) {
|
|
|
997
1016
|
}
|
|
998
1017
|
}), undefined, undefined, undefined, undefined);
|
|
999
1018
|
let makeReceiptLoader = () => LazyLoader.make(transactionHash => {
|
|
1000
|
-
|
|
1001
|
-
return Rest.fetch(Rpc.GetTransactionReceipt.rawRoute, transactionHash, client)
|
|
1019
|
+
let timerRef = Performance.now();
|
|
1020
|
+
return Rest.fetch(Rpc.GetTransactionReceipt.rawRoute, transactionHash, client).then(res => {
|
|
1021
|
+
recordRequest("eth_getTransactionReceipt", Performance.secondsSince(timerRef));
|
|
1022
|
+
return res;
|
|
1023
|
+
});
|
|
1002
1024
|
}, (am, exn) => Logging.error({
|
|
1003
1025
|
err: Utils.prettifyExn(exn),
|
|
1004
1026
|
msg: `Top level promise timeout reached. Please review other errors or warnings in the code. This function will retry in ` + (am._retryDelayMillis / 1000 | 0).toString() + ` seconds. It is highly likely that your indexer isn't syncing on one or more chains currently. Also take a look at the "suggestedFix" in the metadata of this command`,
|
|
@@ -1040,7 +1062,7 @@ function make(param) {
|
|
|
1040
1062
|
Error: new Error()
|
|
1041
1063
|
};
|
|
1042
1064
|
}, lowercaseAddresses);
|
|
1043
|
-
let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, partitionId, selection, retry, param) => {
|
|
1065
|
+
let getItemsOrThrow = async (fromBlock, toBlock, addressesByContractName, contractNameByAddress, knownHeight, partitionId, selection, param, retry, param$1) => {
|
|
1044
1066
|
let startFetchingBatchTimeRef = Performance.now();
|
|
1045
1067
|
let sourceMaxBlockInterval = getSourceMaxBlockInterval(mutSuggestedBlockIntervals, syncConfig.intervalCeiling);
|
|
1046
1068
|
let suggestedBlockInterval = Primitive_int.min(Stdlib_Option.getOr(mutSuggestedBlockIntervals[partitionId], syncConfig.initialBlockInterval), sourceMaxBlockInterval);
|
|
@@ -1049,7 +1071,7 @@ function make(param) {
|
|
|
1049
1071
|
let firstBlockParentPromise = fromBlock > 0 ? LazyLoader.get(blockLoader.contents, fromBlock - 1 | 0).then(json => parseBlockInfo(json)) : Promise.resolve(undefined);
|
|
1050
1072
|
let match = getSelectionConfig(selection);
|
|
1051
1073
|
let logSelections = match.getLogSelectionsOrThrow(addressesByContractName);
|
|
1052
|
-
let match$1 = await getNextPage(fromBlock, suggestedToBlock, logSelections, blockNumber => LazyLoader.get(blockLoader.contents, blockNumber).then(parseBlockInfo), syncConfig, rpcClient, mutSuggestedBlockIntervals, partitionId,
|
|
1074
|
+
let match$1 = await getNextPage(fromBlock, suggestedToBlock, logSelections, blockNumber => LazyLoader.get(blockLoader.contents, blockNumber).then(parseBlockInfo), syncConfig, rpcClient, mutSuggestedBlockIntervals, partitionId, recordRequest);
|
|
1053
1075
|
let latestFetchedBlockInfo = match$1.latestFetchedBlockInfo;
|
|
1054
1076
|
let items = match$1.items;
|
|
1055
1077
|
let executedBlockInterval = (suggestedToBlock - fromBlock | 0) + 1 | 0;
|
|
@@ -1060,10 +1082,11 @@ function make(param) {
|
|
|
1060
1082
|
let log = param.log;
|
|
1061
1083
|
let topic0 = Stdlib_Option.getOr(log.topics[0], "0x0");
|
|
1062
1084
|
let routedAddress = lowercaseAddresses ? Address.Evm.fromAddressLowercaseOrThrow(log.address) : Address.Evm.fromAddressOrThrow(log.address);
|
|
1063
|
-
let
|
|
1064
|
-
if (
|
|
1085
|
+
let onEventRegistration = EventRouter.get(eventRouter, EventRouter.getEvmEventId(topic0, log.topics.length), routedAddress, contractNameByAddress);
|
|
1086
|
+
if (onEventRegistration === undefined) {
|
|
1065
1087
|
return;
|
|
1066
1088
|
}
|
|
1089
|
+
let eventConfig = onEventRegistration.eventConfig;
|
|
1067
1090
|
let decoded = Stdlib_Option.flatMap(Primitive_option.fromNullable(param.params), __x => __x[eventConfig.contractName]);
|
|
1068
1091
|
if (decoded === undefined) {
|
|
1069
1092
|
return;
|
|
@@ -1110,11 +1133,9 @@ function make(param) {
|
|
|
1110
1133
|
let block = match[0];
|
|
1111
1134
|
return {
|
|
1112
1135
|
kind: 0,
|
|
1113
|
-
|
|
1114
|
-
timestamp: block.timestamp,
|
|
1136
|
+
onEventRegistration: onEventRegistration,
|
|
1115
1137
|
chain: chain,
|
|
1116
1138
|
blockNumber: block.number,
|
|
1117
|
-
blockHash: block.hash,
|
|
1118
1139
|
logIndex: log.logIndex,
|
|
1119
1140
|
transactionIndex: log.transactionIndex,
|
|
1120
1141
|
payload: {
|
|
@@ -1125,7 +1146,7 @@ function make(param) {
|
|
|
1125
1146
|
srcAddress: routedAddress,
|
|
1126
1147
|
logIndex: log.logIndex,
|
|
1127
1148
|
transaction: Primitive_option.some(match[1]),
|
|
1128
|
-
block: block
|
|
1149
|
+
block: Primitive_option.some(block)
|
|
1129
1150
|
}
|
|
1130
1151
|
};
|
|
1131
1152
|
})();
|
|
@@ -1162,12 +1183,14 @@ function make(param) {
|
|
|
1162
1183
|
blockHashes: blockHashes,
|
|
1163
1184
|
parsedQueueItems: parsedQueueItems,
|
|
1164
1185
|
transactionStore: undefined,
|
|
1186
|
+
blockStore: undefined,
|
|
1165
1187
|
fromBlockQueried: fromBlock,
|
|
1166
1188
|
latestFetchedBlockNumber: latestFetchedBlockInfo.number,
|
|
1167
1189
|
latestFetchedBlockTimestamp: latestFetchedBlockInfo.timestamp,
|
|
1168
1190
|
stats: {
|
|
1169
1191
|
"total time elapsed (s)": totalTimeElapsed
|
|
1170
|
-
}
|
|
1192
|
+
},
|
|
1193
|
+
requestStats: drainRequestStats()
|
|
1171
1194
|
};
|
|
1172
1195
|
};
|
|
1173
1196
|
let onReorg = param => {
|
|
@@ -1175,19 +1198,28 @@ function make(param) {
|
|
|
1175
1198
|
transactionLoader.contents = makeTransactionLoader();
|
|
1176
1199
|
receiptLoader.contents = makeReceiptLoader();
|
|
1177
1200
|
};
|
|
1178
|
-
let getBlockHashes = (blockNumbers, _currentlyUnusedLogger) => Stdlib_Promise.$$catch(Promise.all(blockNumbers.map(blockNum => LazyLoader.get(blockLoader.contents, blockNum))).then(rawBlocks =>
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1201
|
+
let getBlockHashes = (blockNumbers, _currentlyUnusedLogger) => Stdlib_Promise.$$catch(Promise.all(blockNumbers.map(blockNum => LazyLoader.get(blockLoader.contents, blockNum))).then(rawBlocks => {
|
|
1202
|
+
let result = {
|
|
1203
|
+
TAG: "Ok",
|
|
1204
|
+
_0: rawBlocks.map(json => {
|
|
1205
|
+
let b = parseBlockInfo(json);
|
|
1206
|
+
return {
|
|
1207
|
+
blockHash: b.hash,
|
|
1208
|
+
blockNumber: b.number,
|
|
1209
|
+
blockTimestamp: b.timestamp
|
|
1210
|
+
};
|
|
1211
|
+
})
|
|
1212
|
+
};
|
|
1213
|
+
return {
|
|
1214
|
+
result: result,
|
|
1215
|
+
requestStats: drainRequestStats()
|
|
1216
|
+
};
|
|
1217
|
+
}), exn => Promise.resolve({
|
|
1218
|
+
result: {
|
|
1219
|
+
TAG: "Error",
|
|
1220
|
+
_0: exn
|
|
1221
|
+
},
|
|
1222
|
+
requestStats: drainRequestStats()
|
|
1191
1223
|
}));
|
|
1192
1224
|
let createHeightSubscription = Stdlib_Option.map(param.ws, wsUrl => (onHeight => RpcWebSocketHeightStream.subscribe(wsUrl, chain, onHeight)));
|
|
1193
1225
|
return {
|
|
@@ -1203,15 +1235,14 @@ function make(param) {
|
|
|
1203
1235
|
try {
|
|
1204
1236
|
height = await rpcClient.getHeight();
|
|
1205
1237
|
} catch (exn) {
|
|
1206
|
-
|
|
1207
|
-
Prometheus.SourceRequestCount.increment(name, chain, "eth_blockNumber");
|
|
1208
|
-
Prometheus.SourceRequestCount.addSeconds(name, chain, "eth_blockNumber", seconds);
|
|
1238
|
+
recordRequest("eth_blockNumber", Performance.secondsSince(timerRef));
|
|
1209
1239
|
throw exn;
|
|
1210
1240
|
}
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1241
|
+
recordRequest("eth_blockNumber", Performance.secondsSince(timerRef));
|
|
1242
|
+
return {
|
|
1243
|
+
height: height,
|
|
1244
|
+
requestStats: drainRequestStats()
|
|
1245
|
+
};
|
|
1215
1246
|
},
|
|
1216
1247
|
getItemsOrThrow: getItemsOrThrow,
|
|
1217
1248
|
createHeightSubscription: createHeightSubscription,
|
|
@@ -108,11 +108,6 @@ let subscribe = (~wsUrl, ~chainId, ~onHeight: int => unit): (unit => unit) => {
|
|
|
108
108
|
| NewHead(blockNumber) =>
|
|
109
109
|
errorCount := 0
|
|
110
110
|
resetStaleTimeout()
|
|
111
|
-
Prometheus.SourceRequestCount.increment(
|
|
112
|
-
~sourceName="WebSocket",
|
|
113
|
-
~chainId,
|
|
114
|
-
~method="eth_subscribe",
|
|
115
|
-
)
|
|
116
111
|
onHeight(blockNumber)
|
|
117
112
|
| SubscriptionConfirmed(_) => resetStaleTimeout()
|
|
118
113
|
| ErrorResponse =>
|