envio 3.3.0-alpha.4 → 3.3.0-alpha.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/evm.schema.json +10 -0
- package/package.json +7 -7
- package/src/Batch.res.mjs +1 -1
- package/src/BatchProcessing.res +5 -0
- package/src/BatchProcessing.res.mjs +6 -0
- package/src/ChainState.res +41 -7
- package/src/ChainState.res.mjs +22 -18
- package/src/ChainState.resi +3 -1
- package/src/Config.res +3 -0
- package/src/Config.res.mjs +3 -1
- package/src/Core.res +0 -3
- package/src/ExitOnCaughtUp.res +10 -2
- package/src/ExitOnCaughtUp.res.mjs +10 -1
- package/src/FetchState.res +63 -133
- package/src/FetchState.res.mjs +107 -166
- package/src/IndexerState.res +4 -0
- package/src/IndexerState.res.mjs +8 -1
- package/src/IndexerState.resi +1 -0
- package/src/IndexingAddresses.res +105 -0
- package/src/IndexingAddresses.res.mjs +100 -0
- package/src/IndexingAddresses.resi +32 -0
- package/src/Main.res +4 -15
- package/src/Main.res.mjs +6 -14
- package/src/Metrics.res +33 -0
- package/src/Metrics.res.mjs +39 -0
- package/src/Prometheus.res +2 -20
- package/src/Prometheus.res.mjs +83 -95
- package/src/SimulateDeadInputTracker.res +85 -0
- package/src/SimulateDeadInputTracker.res.mjs +73 -0
- package/src/SimulateDeadInputTracker.resi +12 -0
- package/src/SimulateItems.res +23 -42
- package/src/SimulateItems.res.mjs +12 -30
- package/src/TestIndexer.res +3 -27
- package/src/TestIndexer.res.mjs +2 -9
- package/src/bindings/Viem.res +0 -41
- package/src/bindings/Viem.res.mjs +1 -43
- package/src/sources/Evm.res +7 -36
- package/src/sources/Evm.res.mjs +4 -36
- package/src/sources/EvmChain.res +4 -2
- package/src/sources/EvmChain.res.mjs +3 -2
- package/src/sources/EvmRpcClient.res +36 -5
- package/src/sources/EvmRpcClient.res.mjs +7 -4
- package/src/sources/HyperSyncClient.res +0 -35
- package/src/sources/HyperSyncClient.res.mjs +1 -8
- package/src/sources/Rpc.res +15 -47
- package/src/sources/Rpc.res.mjs +25 -56
- package/src/sources/RpcSource.res +289 -204
- package/src/sources/RpcSource.res.mjs +293 -303
- package/src/sources/SimulateSource.res +1 -0
- package/src/sources/SimulateSource.res.mjs +2 -1
- package/src/sources/Source.res +4 -0
- package/src/sources/Svm.res +7 -15
- package/src/sources/Svm.res.mjs +4 -15
- package/src/sources/TransactionStore.res +14 -2
- package/src/sources/TransactionStore.res.mjs +10 -2
|
@@ -74,6 +74,47 @@ let getKnownRawBlockWithBackoff = async (
|
|
|
74
74
|
}
|
|
75
75
|
result.contents->Option.getOrThrow
|
|
76
76
|
}
|
|
77
|
+
// Pulls the provider error message out of either a parsed Rpc.JsonRpcError or
|
|
78
|
+
// the raw napi JsExn shape (`error.error.message`), so classifiers below don't
|
|
79
|
+
// each have to re-derive it.
|
|
80
|
+
let getErrorMessage = (exn: exn): option<string> =>
|
|
81
|
+
switch exn {
|
|
82
|
+
| Rpc.JsonRpcError({message}) => Some(message)
|
|
83
|
+
| JsExn(error) =>
|
|
84
|
+
try {
|
|
85
|
+
let message: string = (error->Obj.magic)["error"]["message"]
|
|
86
|
+
message->S.assertOrThrow(S.string)
|
|
87
|
+
Some(message)
|
|
88
|
+
} catch {
|
|
89
|
+
| _ => None
|
|
90
|
+
}
|
|
91
|
+
| _ => None
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Deterministic "the range returned too much data" errors that carry no numeric
|
|
95
|
+
// block-range suggestion (HyperRPC's 50k-log cap, response-size and result-count
|
|
96
|
+
// limits). They depend on log density, not on a fixed block window, so waiting
|
|
97
|
+
// never helps — the same range always re-trips the same cap. The reaction is to
|
|
98
|
+
// shrink the range and retry immediately, ratcheting the max range down.
|
|
99
|
+
let isResponseTooLargeError = {
|
|
100
|
+
let patterns = [
|
|
101
|
+
/more than \d+ logs/i, // HyperRPC: "More than 50000 logs returned"
|
|
102
|
+
/\d+ logs returned/i,
|
|
103
|
+
/too many logs/i,
|
|
104
|
+
/query returned more than \d+ results/i, // ZkEVM
|
|
105
|
+
/query exceeds max results/i, // LlamaRPC
|
|
106
|
+
/response size should not/i, // 1RPC
|
|
107
|
+
/(backend )?response too large/i, // Optimism
|
|
108
|
+
/logs matched by query exceeds limit/i, // Arbitrum
|
|
109
|
+
/block range is too wide/i, // Ankr
|
|
110
|
+
]
|
|
111
|
+
(exn: exn) =>
|
|
112
|
+
switch exn->getErrorMessage {
|
|
113
|
+
| Some(message) => patterns->Array.some(re => re->RegExp.test(message))
|
|
114
|
+
| None => false
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
77
118
|
let getSuggestedBlockIntervalFromExn = {
|
|
78
119
|
// Unknown provider: "retry with the range 123-456"
|
|
79
120
|
let suggestedRangeRegExp = /retry with the range (\d+)-(\d+)/
|
|
@@ -118,14 +159,6 @@ let getSuggestedBlockIntervalFromExn = {
|
|
|
118
159
|
// when we send request with numeric block range instead of hex
|
|
119
160
|
// Infura, ZkSync: "Try with this block range [0x123,0x456]"
|
|
120
161
|
|
|
121
|
-
// Future handling needed for these providers that don't suggest ranges:
|
|
122
|
-
// - Ankr: "block range is too wide"
|
|
123
|
-
// - 1RPC: "response size should not greater than 10000000 bytes"
|
|
124
|
-
// - ZkEVM: "query returned more than 10000 results"
|
|
125
|
-
// - LlamaRPC: "query exceeds max results"
|
|
126
|
-
// - Optimism: "backend response too large" or "Block range is too large"
|
|
127
|
-
// - Arbitrum: "logs matched by query exceeds limit of 10000"
|
|
128
|
-
|
|
129
162
|
let parseMessageForBlockRange = (message: string) => {
|
|
130
163
|
// Helper to extract block range from regex match
|
|
131
164
|
let extractBlockRange = (execResult, ~isMaxRange) =>
|
|
@@ -209,35 +242,54 @@ let getSuggestedBlockIntervalFromExn = {
|
|
|
209
242
|
// Whether it's the max range that the provider allows
|
|
210
243
|
bool,
|
|
211
244
|
)> =>
|
|
212
|
-
switch exn {
|
|
213
|
-
|
|
|
214
|
-
|
|
|
215
|
-
try {
|
|
216
|
-
let message: string = (error->Obj.magic)["error"]["message"]
|
|
217
|
-
message->S.assertOrThrow(S.string)
|
|
218
|
-
parseMessageForBlockRange(message)
|
|
219
|
-
} catch {
|
|
220
|
-
| _ => None
|
|
221
|
-
}
|
|
222
|
-
| _ => None
|
|
245
|
+
switch exn->getErrorMessage {
|
|
246
|
+
| Some(message) => parseMessageForBlockRange(message)
|
|
247
|
+
| None => None
|
|
223
248
|
}
|
|
224
249
|
}
|
|
225
250
|
|
|
226
251
|
type eventBatchQuery = {
|
|
227
|
-
|
|
252
|
+
items: array<EvmRpcClient.rpcEventItem>,
|
|
228
253
|
latestFetchedBlockInfo: blockInfo,
|
|
229
254
|
}
|
|
230
255
|
|
|
231
256
|
let maxSuggestedBlockIntervalKey = "max"
|
|
232
257
|
|
|
258
|
+
let getSourceMaxBlockInterval = (mutSuggestedBlockIntervals, ~intervalCeiling) =>
|
|
259
|
+
mutSuggestedBlockIntervals
|
|
260
|
+
->Utils.Dict.dangerouslyGetNonOption(maxSuggestedBlockIntervalKey)
|
|
261
|
+
->Option.getOr(intervalCeiling)
|
|
262
|
+
|
|
263
|
+
type logSelection = {
|
|
264
|
+
addresses: option<array<Address.t>>,
|
|
265
|
+
topicQuery: Rpc.GetLogs.topicQuery,
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// A log can satisfy more than one selection when a single event's `where` is an
|
|
269
|
+
// OR of param groups, so dedup the fanned-out responses by (blockNumber,
|
|
270
|
+
// logIndex) — unique per chain — keeping the first occurrence.
|
|
271
|
+
let mergeAndDedupItems = (itemsPerSelection: array<array<EvmRpcClient.rpcEventItem>>) => {
|
|
272
|
+
let seen = Utils.Set.make()
|
|
273
|
+
let merged = []
|
|
274
|
+
itemsPerSelection->Array.forEach(items =>
|
|
275
|
+
items->Array.forEach((item: EvmRpcClient.rpcEventItem) => {
|
|
276
|
+
let key = `${item.log.blockNumber->Int.toString}-${item.log.logIndex->Int.toString}`
|
|
277
|
+
if seen->Utils.Set.has(key)->not {
|
|
278
|
+
seen->Utils.Set.add(key)->ignore
|
|
279
|
+
merged->Array.push(item)->ignore
|
|
280
|
+
}
|
|
281
|
+
})
|
|
282
|
+
)
|
|
283
|
+
merged
|
|
284
|
+
}
|
|
285
|
+
|
|
233
286
|
let getNextPage = (
|
|
234
287
|
~fromBlock,
|
|
235
288
|
~toBlock,
|
|
236
|
-
~
|
|
237
|
-
~topicQuery,
|
|
289
|
+
~logSelections: array<logSelection>,
|
|
238
290
|
~loadBlock,
|
|
239
291
|
~syncConfig as sc: Config.sourceSync,
|
|
240
|
-
~
|
|
292
|
+
~rpcClient: EvmRpcClient.t,
|
|
241
293
|
~mutSuggestedBlockIntervals,
|
|
242
294
|
~partitionId,
|
|
243
295
|
~sourceName,
|
|
@@ -252,90 +304,109 @@ let getNextPage = (
|
|
|
252
304
|
)
|
|
253
305
|
|
|
254
306
|
let latestFetchedBlockPromise = loadBlock(toBlock)
|
|
255
|
-
|
|
256
|
-
let
|
|
257
|
-
~
|
|
258
|
-
|
|
259
|
-
address: ?addresses,
|
|
260
|
-
topics: topicQuery,
|
|
307
|
+
|
|
308
|
+
let queryLogs = ({addresses, topicQuery}: logSelection) => {
|
|
309
|
+
Prometheus.SourceRequestCount.increment(~sourceName, ~chainId, ~method="eth_getLogs")
|
|
310
|
+
rpcClient.getLogs({
|
|
261
311
|
fromBlock,
|
|
262
312
|
toBlock,
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
313
|
+
?addresses,
|
|
314
|
+
topics: topicQuery->Array.map(filter =>
|
|
315
|
+
switch filter {
|
|
316
|
+
| Rpc.GetLogs.Null => Nullable.null
|
|
317
|
+
| Single(topic) => Nullable.make([topic])
|
|
318
|
+
| Multiple(topics) => Nullable.make(topics)
|
|
319
|
+
}
|
|
320
|
+
),
|
|
321
|
+
})
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
let logsPromise = switch logSelections {
|
|
325
|
+
| [] =>
|
|
326
|
+
latestFetchedBlockPromise->Promise.thenResolve((latestFetchedBlockInfo): eventBatchQuery => {
|
|
327
|
+
items: [],
|
|
328
|
+
latestFetchedBlockInfo,
|
|
329
|
+
})
|
|
330
|
+
// Fast path: a single selection needs no cross-request merge or dedup.
|
|
331
|
+
| [logSelection] =>
|
|
332
|
+
logSelection
|
|
333
|
+
->queryLogs
|
|
334
|
+
->Promise.then(async items => {
|
|
335
|
+
{
|
|
336
|
+
items,
|
|
337
|
+
latestFetchedBlockInfo: await latestFetchedBlockPromise,
|
|
338
|
+
}
|
|
339
|
+
})
|
|
340
|
+
| _ =>
|
|
341
|
+
logSelections
|
|
342
|
+
->Array.map(queryLogs)
|
|
343
|
+
->Promise.all
|
|
344
|
+
->Promise.then(async itemsPerSelection => {
|
|
345
|
+
{
|
|
346
|
+
items: itemsPerSelection->mergeAndDedupItems,
|
|
347
|
+
latestFetchedBlockInfo: await latestFetchedBlockPromise,
|
|
348
|
+
}
|
|
349
|
+
})
|
|
350
|
+
}
|
|
270
351
|
|
|
271
352
|
[queryTimoutPromise, logsPromise]
|
|
272
353
|
->Promise.race
|
|
273
354
|
->Promise.catch(err => {
|
|
355
|
+
let executedBlockInterval = toBlock - fromBlock + 1
|
|
356
|
+
let shrunkBlockInterval =
|
|
357
|
+
Pervasives.max(1, (executedBlockInterval->Int.toFloat *. sc.backoffMultiplicative)->Float.toInt)
|
|
358
|
+
|
|
359
|
+
let throwFailedGettingItems = retry =>
|
|
360
|
+
throw(Source.GetItemsError(FailedGettingItems({exn: err, attemptedToBlock: toBlock, retry})))
|
|
361
|
+
let throwResize = interval =>
|
|
362
|
+
throwFailedGettingItems(WithSuggestedToBlock({toBlock: fromBlock + interval - 1}))
|
|
363
|
+
|
|
274
364
|
switch getSuggestedBlockIntervalFromExn(err) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
throw(
|
|
281
|
-
Source.GetItemsError(
|
|
282
|
-
FailedGettingItems({
|
|
283
|
-
exn: err,
|
|
284
|
-
attemptedToBlock: toBlock,
|
|
285
|
-
retry: WithSuggestedToBlock({
|
|
286
|
-
toBlock: fromBlock + nextBlockIntervalTry - 1,
|
|
287
|
-
}),
|
|
288
|
-
}),
|
|
289
|
-
),
|
|
365
|
+
// "limited to N blocks" — a structural cap on the whole source; only tighten.
|
|
366
|
+
| Some((interval, true)) =>
|
|
367
|
+
let capped = Pervasives.min(
|
|
368
|
+
mutSuggestedBlockIntervals->getSourceMaxBlockInterval(~intervalCeiling=sc.intervalCeiling),
|
|
369
|
+
interval,
|
|
290
370
|
)
|
|
371
|
+
mutSuggestedBlockIntervals->Dict.set(maxSuggestedBlockIntervalKey, capped)
|
|
372
|
+
throwResize(capped)
|
|
373
|
+
// A one-off suggested range ("retry with range X-Y") — apply to this partition.
|
|
374
|
+
| Some((interval, false)) =>
|
|
375
|
+
mutSuggestedBlockIntervals->Dict.set(partitionId, interval)
|
|
376
|
+
throwResize(interval)
|
|
377
|
+
// Density cap with no suggested number (too many logs / response too large):
|
|
378
|
+
// shrink THIS partition and retry immediately (no wait); acceleration
|
|
379
|
+
// re-adapts on the next successful query. The interval>1 guard avoids a
|
|
380
|
+
// no-progress tight loop on a single over-cap block.
|
|
381
|
+
| None if executedBlockInterval > 1 && err->isResponseTooLargeError =>
|
|
382
|
+
mutSuggestedBlockIntervals->Dict.set(partitionId, shrunkBlockInterval)
|
|
383
|
+
throwResize(shrunkBlockInterval)
|
|
384
|
+
// Transient/unknown — shrink this partition and back off.
|
|
291
385
|
| None =>
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
(
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
Source.FailedGettingItems({
|
|
299
|
-
exn: err,
|
|
300
|
-
attemptedToBlock: toBlock,
|
|
301
|
-
retry: WithBackoff({
|
|
302
|
-
message: `Failed getting data for the block range. Will try smaller block range for the next attempt.`,
|
|
303
|
-
backoffMillis: sc.backoffMillis,
|
|
304
|
-
}),
|
|
305
|
-
}),
|
|
306
|
-
),
|
|
386
|
+
mutSuggestedBlockIntervals->Dict.set(partitionId, shrunkBlockInterval)
|
|
387
|
+
throwFailedGettingItems(
|
|
388
|
+
WithBackoff({
|
|
389
|
+
message: `Failed getting data for the block range. Will try smaller block range for the next attempt.`,
|
|
390
|
+
backoffMillis: sc.backoffMillis,
|
|
391
|
+
}),
|
|
307
392
|
)
|
|
308
393
|
}
|
|
309
394
|
})
|
|
310
395
|
}
|
|
311
396
|
|
|
312
|
-
type logSelection = {
|
|
313
|
-
addresses: option<array<Address.t>>,
|
|
314
|
-
topicQuery: Rpc.GetLogs.topicQuery,
|
|
315
|
-
}
|
|
316
|
-
|
|
317
397
|
type selectionConfig = {
|
|
318
|
-
|
|
398
|
+
getLogSelectionsOrThrow: (
|
|
399
|
+
~addressesByContractName: dict<array<Address.t>>,
|
|
400
|
+
) => array<logSelection>,
|
|
319
401
|
}
|
|
320
402
|
|
|
321
403
|
let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
|
|
322
|
-
let
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
->(Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>)
|
|
327
|
-
->Array.forEach(({getEventFiltersOrThrow}) => {
|
|
328
|
-
switch getEventFiltersOrThrow(chain) {
|
|
329
|
-
| Static(s) => staticTopicSelections->Array.pushMany(s)->ignore
|
|
330
|
-
| Dynamic(fn) => dynamicEventFilters->Array.push(fn)->ignore
|
|
331
|
-
}
|
|
332
|
-
})
|
|
404
|
+
let evmEventConfigs =
|
|
405
|
+
selection.eventConfigs->(
|
|
406
|
+
Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
|
|
407
|
+
)
|
|
333
408
|
|
|
334
|
-
|
|
335
|
-
staticTopicSelections->LogSelection.compressTopicSelections,
|
|
336
|
-
dynamicEventFilters,
|
|
337
|
-
) {
|
|
338
|
-
| ([], []) =>
|
|
409
|
+
if evmEventConfigs->Utils.Array.isEmpty {
|
|
339
410
|
throw(
|
|
340
411
|
Source.GetItemsError(
|
|
341
412
|
UnsupportedSelection({
|
|
@@ -343,48 +414,110 @@ let getSelectionConfig = (selection: FetchState.selection, ~chain) => {
|
|
|
343
414
|
}),
|
|
344
415
|
),
|
|
345
416
|
)
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// eth_getLogs takes one address list and one topic selection per request, so
|
|
420
|
+
// fan out to one request per selection. Each address-bound event is grouped by
|
|
421
|
+
// its contract and later scoped to that contract's own addresses — pooling all
|
|
422
|
+
// contracts' addresses would let one contract's query fetch a sibling's logs,
|
|
423
|
+
// which route back by address and bypass the sibling's filter (routing never
|
|
424
|
+
// re-applies it). Pure-wildcard events carry no address constraint, so they're
|
|
425
|
+
// pooled and resolved once.
|
|
426
|
+
let noAddressTopicSelections = []
|
|
427
|
+
let staticByContract = Dict.make()
|
|
428
|
+
let dynamicByContract = Dict.make()
|
|
429
|
+
let dynamicWildcardByContract = Dict.make()
|
|
430
|
+
let contractNames = Utils.Set.make()
|
|
431
|
+
|
|
432
|
+
evmEventConfigs->Array.forEach(({
|
|
433
|
+
contractName,
|
|
434
|
+
isWildcard,
|
|
435
|
+
dependsOnAddresses,
|
|
436
|
+
getEventFiltersOrThrow,
|
|
437
|
+
}) => {
|
|
438
|
+
let eventFilters = getEventFiltersOrThrow(chain)
|
|
439
|
+
if dependsOnAddresses {
|
|
440
|
+
contractNames->Utils.Set.add(contractName)->ignore
|
|
441
|
+
switch eventFilters {
|
|
442
|
+
| Internal.Static(topicSelections) =>
|
|
443
|
+
staticByContract->Utils.Dict.pushMany(contractName, topicSelections)
|
|
444
|
+
| Dynamic(fn) =>
|
|
445
|
+
(isWildcard ? dynamicWildcardByContract : dynamicByContract)->Utils.Dict.push(
|
|
446
|
+
contractName,
|
|
447
|
+
fn,
|
|
448
|
+
)
|
|
354
449
|
}
|
|
450
|
+
} else {
|
|
451
|
+
noAddressTopicSelections
|
|
452
|
+
->Array.pushMany(
|
|
453
|
+
switch eventFilters {
|
|
454
|
+
| Static(s) => s
|
|
455
|
+
| Dynamic(fn) => fn([])
|
|
456
|
+
},
|
|
457
|
+
)
|
|
458
|
+
->ignore
|
|
355
459
|
}
|
|
356
|
-
|
|
357
|
-
let eventConfig = selection.eventConfigs->Utils.Array.firstUnsafe
|
|
460
|
+
})
|
|
358
461
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
462
|
+
// `compressTopicSelections` folds the filter-less events into a single topic0
|
|
463
|
+
// OR-set, keeping the common case at one request.
|
|
464
|
+
let toLogSelections = (~addresses, topicSelections): array<logSelection> =>
|
|
465
|
+
topicSelections
|
|
466
|
+
->LogSelection.compressTopicSelections
|
|
467
|
+
->Array.map(topicSelection => {
|
|
468
|
+
addresses,
|
|
469
|
+
topicQuery: topicSelection->Rpc.GetLogs.mapTopicQuery,
|
|
470
|
+
})
|
|
471
|
+
|
|
472
|
+
// Address-independent, so resolve once (the wildcard partition reuses this).
|
|
473
|
+
let noAddressLogSelections = toLogSelections(~addresses=None, noAddressTopicSelections)
|
|
474
|
+
|
|
475
|
+
let getLogSelectionsOrThrow = if contractNames->Utils.Set.size === 0 {
|
|
476
|
+
(~addressesByContractName as _) => noAddressLogSelections
|
|
477
|
+
} else {
|
|
478
|
+
(~addressesByContractName): array<logSelection> => {
|
|
479
|
+
let logSelections = noAddressLogSelections->Array.copy
|
|
480
|
+
contractNames->Utils.Set.forEach(contractName => {
|
|
481
|
+
switch addressesByContractName->Utils.Dict.dangerouslyGetNonOption(contractName) {
|
|
482
|
+
| None
|
|
483
|
+
| Some([]) => ()
|
|
484
|
+
| Some(addresses) =>
|
|
485
|
+
// Static + dynamic non-wildcard filters, scoped to this contract's addresses.
|
|
486
|
+
let addressedTopicSelections = []
|
|
487
|
+
switch staticByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
|
|
488
|
+
| Some(s) => addressedTopicSelections->Array.pushMany(s)->ignore
|
|
489
|
+
| None => ()
|
|
490
|
+
}
|
|
491
|
+
switch dynamicByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
|
|
492
|
+
| Some(fns) =>
|
|
493
|
+
fns->Array.forEach(fn =>
|
|
494
|
+
addressedTopicSelections->Array.pushMany(fn(addresses))->ignore
|
|
495
|
+
)
|
|
496
|
+
| None => ()
|
|
497
|
+
}
|
|
498
|
+
logSelections
|
|
499
|
+
->Array.pushMany(toLogSelections(~addresses=Some(addresses), addressedTopicSelections))
|
|
500
|
+
->ignore
|
|
501
|
+
|
|
502
|
+
// Dynamic wildcard-by-address filters fold the address into the topics,
|
|
503
|
+
// so they still match any address.
|
|
504
|
+
switch dynamicWildcardByContract->Utils.Dict.dangerouslyGetNonOption(contractName) {
|
|
505
|
+
| Some(fns) =>
|
|
506
|
+
logSelections
|
|
507
|
+
->Array.pushMany(
|
|
508
|
+
toLogSelections(~addresses=None, fns->Array.flatMap(fn => fn(addresses))),
|
|
509
|
+
)
|
|
510
|
+
->ignore
|
|
511
|
+
| None => ()
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
})
|
|
515
|
+
logSelections
|
|
375
516
|
}
|
|
376
|
-
| _ =>
|
|
377
|
-
throw(
|
|
378
|
-
Source.GetItemsError(
|
|
379
|
-
UnsupportedSelection({
|
|
380
|
-
message: "RPC data-source currently supports event filters only when there's a single wildcard event. Please, create a GitHub issue if it's a blocker for you.",
|
|
381
|
-
}),
|
|
382
|
-
),
|
|
383
|
-
)
|
|
384
517
|
}
|
|
385
518
|
|
|
386
519
|
{
|
|
387
|
-
|
|
520
|
+
getLogSelectionsOrThrow: getLogSelectionsOrThrow,
|
|
388
521
|
}
|
|
389
522
|
}
|
|
390
523
|
|
|
@@ -855,6 +988,7 @@ type options = {
|
|
|
855
988
|
allEventParams: array<HyperSyncClient.Decoder.eventParamsInput>,
|
|
856
989
|
lowercaseAddresses: bool,
|
|
857
990
|
ws?: string,
|
|
991
|
+
headers?: dict<string>,
|
|
858
992
|
}
|
|
859
993
|
|
|
860
994
|
let make = (
|
|
@@ -867,6 +1001,7 @@ let make = (
|
|
|
867
1001
|
allEventParams,
|
|
868
1002
|
lowercaseAddresses,
|
|
869
1003
|
?ws,
|
|
1004
|
+
?headers,
|
|
870
1005
|
}: options,
|
|
871
1006
|
): t => {
|
|
872
1007
|
let chainId = chain->ChainMap.Chain.toChainId
|
|
@@ -881,10 +1016,19 @@ let make = (
|
|
|
881
1016
|
|
|
882
1017
|
let getSelectionConfig = memoGetSelectionConfig(~chain)
|
|
883
1018
|
|
|
1019
|
+
// Per-partition adaptive block interval (AIMD), keyed by partitionId. The
|
|
1020
|
+
// `max` key holds a source-wide ceiling that only ever tightens, set by
|
|
1021
|
+
// structural provider limits ("limited to N blocks"). A partition's own entry
|
|
1022
|
+
// can go stale when partitions merge/split — acceptable, it re-adapts.
|
|
884
1023
|
let mutSuggestedBlockIntervals = Dict.make()
|
|
885
1024
|
|
|
886
|
-
let client = Rpc.makeClient(url)
|
|
887
|
-
let rpcClient = EvmRpcClient.make(
|
|
1025
|
+
let client = Rpc.makeClient(url, ~headers?)
|
|
1026
|
+
let rpcClient = EvmRpcClient.make(
|
|
1027
|
+
~url,
|
|
1028
|
+
~allEventParams,
|
|
1029
|
+
~checksumAddresses=!lowercaseAddresses,
|
|
1030
|
+
~headers?,
|
|
1031
|
+
)
|
|
888
1032
|
|
|
889
1033
|
let makeTransactionLoader = () =>
|
|
890
1034
|
LazyLoader.make(
|
|
@@ -1000,36 +1144,6 @@ let make = (
|
|
|
1000
1144
|
~lowercaseAddresses,
|
|
1001
1145
|
)
|
|
1002
1146
|
|
|
1003
|
-
let convertLogToHyperSyncEvent = (log: Rpc.GetLogs.log): HyperSyncClient.ResponseTypes.event => {
|
|
1004
|
-
let hyperSyncLog: HyperSyncClient.ResponseTypes.log = {
|
|
1005
|
-
removed: log.removed,
|
|
1006
|
-
index: log.logIndex,
|
|
1007
|
-
transactionIndex: log.transactionIndex,
|
|
1008
|
-
transactionHash: log.transactionHash,
|
|
1009
|
-
blockHash: log.blockHash,
|
|
1010
|
-
blockNumber: log.blockNumber,
|
|
1011
|
-
address: log.address,
|
|
1012
|
-
data: log.data,
|
|
1013
|
-
topics: log.topics->(Utils.magic: array<string> => array<Nullable.t<EvmTypes.Hex.t>>),
|
|
1014
|
-
}
|
|
1015
|
-
{log: hyperSyncLog}
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
let hscDecoder: ref<option<HyperSyncClient.Decoder.tWithParams>> = ref(None)
|
|
1019
|
-
let getHscDecoder = () => {
|
|
1020
|
-
switch hscDecoder.contents {
|
|
1021
|
-
| Some(decoder) => decoder
|
|
1022
|
-
| None => {
|
|
1023
|
-
let decoder = HyperSyncClient.Decoder.fromParams(
|
|
1024
|
-
allEventParams,
|
|
1025
|
-
~checksumAddresses=!lowercaseAddresses,
|
|
1026
|
-
)
|
|
1027
|
-
hscDecoder := Some(decoder)
|
|
1028
|
-
decoder
|
|
1029
|
-
}
|
|
1030
|
-
}
|
|
1031
|
-
}
|
|
1032
|
-
|
|
1033
1147
|
let getItemsOrThrow = async (
|
|
1034
1148
|
~fromBlock,
|
|
1035
1149
|
~toBlock,
|
|
@@ -1043,15 +1157,14 @@ let make = (
|
|
|
1043
1157
|
) => {
|
|
1044
1158
|
let startFetchingBatchTimeRef = Performance.now()
|
|
1045
1159
|
|
|
1046
|
-
let
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
| Some(maxSuggestedBlockInterval) => maxSuggestedBlockInterval
|
|
1050
|
-
| None =>
|
|
1160
|
+
let sourceMaxBlockInterval =
|
|
1161
|
+
mutSuggestedBlockIntervals->getSourceMaxBlockInterval(~intervalCeiling=syncConfig.intervalCeiling)
|
|
1162
|
+
let suggestedBlockInterval = Pervasives.min(
|
|
1051
1163
|
mutSuggestedBlockIntervals
|
|
1052
1164
|
->Utils.Dict.dangerouslyGetNonOption(partitionId)
|
|
1053
|
-
->Option.getOr(syncConfig.initialBlockInterval)
|
|
1054
|
-
|
|
1165
|
+
->Option.getOr(syncConfig.initialBlockInterval),
|
|
1166
|
+
sourceMaxBlockInterval,
|
|
1167
|
+
)
|
|
1055
1168
|
|
|
1056
1169
|
// Always have a toBlock for an RPC worker
|
|
1057
1170
|
let toBlock = switch toBlock {
|
|
@@ -1070,20 +1183,19 @@ let make = (
|
|
|
1070
1183
|
->Promise.thenResolve(json => Some(parseBlockInfo(json)))
|
|
1071
1184
|
: Promise.resolve(None)
|
|
1072
1185
|
|
|
1073
|
-
let {
|
|
1074
|
-
let
|
|
1186
|
+
let {getLogSelectionsOrThrow} = getSelectionConfig(selection)
|
|
1187
|
+
let logSelections = getLogSelectionsOrThrow(~addressesByContractName)
|
|
1075
1188
|
|
|
1076
|
-
let {
|
|
1189
|
+
let {items, latestFetchedBlockInfo} = await getNextPage(
|
|
1077
1190
|
~fromBlock,
|
|
1078
1191
|
~toBlock=suggestedToBlock,
|
|
1079
|
-
~
|
|
1080
|
-
~topicQuery,
|
|
1192
|
+
~logSelections,
|
|
1081
1193
|
~loadBlock=blockNumber =>
|
|
1082
1194
|
blockLoader.contents
|
|
1083
1195
|
->LazyLoader.get(blockNumber)
|
|
1084
1196
|
->Promise.thenResolve(parseBlockInfo),
|
|
1085
1197
|
~syncConfig,
|
|
1086
|
-
~
|
|
1198
|
+
~rpcClient,
|
|
1087
1199
|
~mutSuggestedBlockIntervals,
|
|
1088
1200
|
~partitionId,
|
|
1089
1201
|
~sourceName=name,
|
|
@@ -1092,49 +1204,22 @@ let make = (
|
|
|
1092
1204
|
|
|
1093
1205
|
let executedBlockInterval = suggestedToBlock - fromBlock + 1
|
|
1094
1206
|
|
|
1095
|
-
//
|
|
1096
|
-
//
|
|
1097
|
-
//
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
!(mutSuggestedBlockIntervals->Dict.has(maxSuggestedBlockIntervalKey))
|
|
1101
|
-
) {
|
|
1102
|
-
// Increase batch size going forward, but do not increase past a configured maximum
|
|
1103
|
-
// See: https://en.wikipedia.org/wiki/Additive_increase/multiplicative_decrease
|
|
1207
|
+
// Grow this partition's interval only when the full suggested range was
|
|
1208
|
+
// actually applied (not clamped by a hard toBlock). The min clamps to the
|
|
1209
|
+
// source-wide ceiling, which also stops growth once a structural cap tightened it.
|
|
1210
|
+
// See: https://en.wikipedia.org/wiki/Additive_increase/multiplicative_decrease
|
|
1211
|
+
if executedBlockInterval >= suggestedBlockInterval {
|
|
1104
1212
|
mutSuggestedBlockIntervals->Dict.set(
|
|
1105
1213
|
partitionId,
|
|
1106
1214
|
Pervasives.min(
|
|
1107
1215
|
executedBlockInterval + syncConfig.accelerationAdditive,
|
|
1108
|
-
|
|
1109
|
-
),
|
|
1110
|
-
)
|
|
1111
|
-
}
|
|
1112
|
-
|
|
1113
|
-
// Convert RPC logs to HyperSync events
|
|
1114
|
-
let hyperSyncEvents = logs->Array.map(convertLogToHyperSyncEvent)
|
|
1115
|
-
|
|
1116
|
-
// Decode using HyperSyncClient decoder
|
|
1117
|
-
let parsedEvents = try await getHscDecoder().decodeLogs(hyperSyncEvents) catch {
|
|
1118
|
-
| exn =>
|
|
1119
|
-
throw(
|
|
1120
|
-
Source.GetItemsError(
|
|
1121
|
-
FailedGettingItems({
|
|
1122
|
-
exn,
|
|
1123
|
-
attemptedToBlock: toBlock,
|
|
1124
|
-
retry: ImpossibleForTheQuery({
|
|
1125
|
-
message: "Failed to parse events using hypersync client decoder. Please double-check your ABI.",
|
|
1126
|
-
}),
|
|
1127
|
-
}),
|
|
1216
|
+
sourceMaxBlockInterval,
|
|
1128
1217
|
),
|
|
1129
1218
|
)
|
|
1130
1219
|
}
|
|
1131
1220
|
|
|
1132
|
-
let parsedQueueItems = await
|
|
1133
|
-
->Array.
|
|
1134
|
-
->Array.filterMap(((
|
|
1135
|
-
log: Rpc.GetLogs.log,
|
|
1136
|
-
maybeDecodedEvent: Nullable.t<dict<Internal.eventParams>>,
|
|
1137
|
-
)) => {
|
|
1221
|
+
let parsedQueueItems = await items
|
|
1222
|
+
->Array.filterMap(({log, params: maybeDecodedEvent}: EvmRpcClient.rpcEventItem) => {
|
|
1138
1223
|
let topic0 = log.topics[0]->Option.getOr("0x0")
|
|
1139
1224
|
let routedAddress = if lowercaseAddresses {
|
|
1140
1225
|
log.address->Address.Evm.fromAddressLowercaseOrThrow
|
|
@@ -1242,7 +1327,7 @@ let make = (
|
|
|
1242
1327
|
| Some(b) => pushBlockInfo(b)
|
|
1243
1328
|
| None => ()
|
|
1244
1329
|
}
|
|
1245
|
-
|
|
1330
|
+
items->Array.forEach(({log}) =>
|
|
1246
1331
|
blockHashes
|
|
1247
1332
|
->Array.push({ReorgDetection.blockNumber: log.blockNumber, blockHash: log.blockHash})
|
|
1248
1333
|
->ignore
|