envio 3.3.0-alpha.9 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.d.ts +19 -0
- package/licenses/CLA.md +35 -0
- package/licenses/EULA.md +67 -0
- package/licenses/LICENSE.md +45 -0
- package/licenses/README.md +35 -0
- package/package.json +9 -8
- package/src/Batch.res.mjs +1 -1
- package/src/BatchProcessing.res +0 -7
- package/src/BatchProcessing.res.mjs +1 -8
- package/src/ChainFetching.res +30 -18
- package/src/ChainFetching.res.mjs +23 -13
- package/src/ChainState.res +134 -55
- package/src/ChainState.res.mjs +78 -36
- package/src/ChainState.resi +14 -2
- package/src/Config.res +9 -5
- package/src/Config.res.mjs +2 -2
- package/src/Core.res +20 -0
- package/src/Core.res.mjs +12 -0
- package/src/CrossChainState.res +115 -34
- package/src/CrossChainState.res.mjs +37 -14
- package/src/EffectState.res +100 -0
- package/src/EffectState.res.mjs +74 -0
- package/src/EffectState.resi +32 -0
- package/src/Envio.res +25 -7
- package/src/Envio.res.mjs +15 -19
- package/src/EventConfigBuilder.res +21 -46
- package/src/EventConfigBuilder.res.mjs +18 -25
- package/src/EventProcessing.res +8 -5
- package/src/EventProcessing.res.mjs +2 -1
- package/src/FetchState.res +814 -467
- package/src/FetchState.res.mjs +541 -369
- package/src/HandlerRegister.res +3 -1
- package/src/HandlerRegister.res.mjs +3 -2
- package/src/InMemoryStore.res +14 -26
- package/src/InMemoryStore.res.mjs +6 -19
- package/src/IndexerState.res +15 -39
- package/src/IndexerState.res.mjs +18 -29
- package/src/IndexerState.resi +2 -10
- package/src/Internal.res +98 -14
- package/src/Internal.res.mjs +96 -2
- package/src/LoadLayer.res +44 -24
- package/src/LoadLayer.res.mjs +43 -20
- package/src/LoadLayer.resi +1 -0
- package/src/LogSelection.res +0 -62
- package/src/LogSelection.res.mjs +0 -74
- package/src/Metrics.res +1 -1
- package/src/Metrics.res.mjs +1 -1
- package/src/Persistence.res +12 -3
- package/src/PgStorage.res +155 -82
- package/src/PgStorage.res.mjs +130 -77
- package/src/Prometheus.res +20 -10
- package/src/Prometheus.res.mjs +22 -10
- package/src/PruneStaleHistory.res +146 -35
- package/src/PruneStaleHistory.res.mjs +114 -20
- package/src/RawEvent.res +2 -2
- package/src/Rollback.res +32 -13
- package/src/Rollback.res.mjs +24 -11
- package/src/SimulateDeadInputTracker.res +1 -1
- package/src/SimulateItems.res +38 -6
- package/src/SimulateItems.res.mjs +28 -9
- package/src/TestIndexer.res +2 -2
- package/src/TestIndexer.res.mjs +2 -2
- package/src/TopicFilter.res +1 -25
- package/src/TopicFilter.res.mjs +0 -74
- package/src/UserContext.res +62 -23
- package/src/UserContext.res.mjs +26 -6
- package/src/Writing.res +60 -21
- package/src/Writing.res.mjs +27 -9
- package/src/bindings/NodeJs.res +5 -0
- package/src/bindings/Viem.res +0 -5
- package/src/sources/EventRouter.res +0 -21
- package/src/sources/EventRouter.res.mjs +0 -12
- package/src/sources/EvmChain.res +2 -27
- package/src/sources/EvmChain.res.mjs +2 -23
- package/src/sources/EvmRpcClient.res +58 -23
- package/src/sources/EvmRpcClient.res.mjs +11 -5
- package/src/sources/HyperSync.res +9 -38
- package/src/sources/HyperSync.res.mjs +16 -28
- package/src/sources/HyperSync.resi +2 -2
- package/src/sources/HyperSyncClient.res +88 -11
- package/src/sources/HyperSyncClient.res.mjs +39 -6
- package/src/sources/HyperSyncSource.res +18 -199
- package/src/sources/HyperSyncSource.res.mjs +9 -128
- package/src/sources/Rpc.res +0 -32
- package/src/sources/Rpc.res.mjs +1 -46
- package/src/sources/RpcSource.res +129 -522
- package/src/sources/RpcSource.res.mjs +161 -366
- package/src/sources/SimulateSource.res +37 -19
- package/src/sources/SimulateSource.res.mjs +27 -10
- package/src/sources/SourceManager.res +3 -7
- package/src/sources/SourceManager.res.mjs +2 -7
- package/src/sources/SourceManager.resi +0 -2
- package/src/sources/SvmHyperSyncSource.res +13 -3
- package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
open Source
|
|
2
2
|
|
|
3
|
-
exception QueryTimout(string)
|
|
4
|
-
|
|
5
3
|
// eth_getTransactionByHash/eth_getTransactionReceipt returning null is usually
|
|
6
4
|
// transient: a load-balanced provider can route the lookup to a node that
|
|
7
5
|
// hasn't caught up with the one that served eth_getLogs. Must stay retryable,
|
|
@@ -74,454 +72,91 @@ let getKnownRawBlockWithBackoff = async (
|
|
|
74
72
|
}
|
|
75
73
|
result.contents->Option.getOrThrow
|
|
76
74
|
}
|
|
77
|
-
// Pulls the provider error message out of
|
|
78
|
-
//
|
|
79
|
-
//
|
|
75
|
+
// Pulls the underlying provider error message back out of a caught exn, for
|
|
76
|
+
// logging/debugging. Provider JSON-RPC errors surface as `Rpc.JsonRpcError`;
|
|
77
|
+
// the paging retry decision (see `parseGetNextPageRetryError` below) surfaces
|
|
78
|
+
// as a napi `JsExn` whose message is the JSON payload `EvmRpcClient.getNextPage`
|
|
79
|
+
// throws, carrying the classified message (if any) under `errorMessage`.
|
|
80
80
|
let getErrorMessage = (exn: exn): option<string> =>
|
|
81
81
|
switch exn {
|
|
82
82
|
| Rpc.JsonRpcError({message}) => Some(message)
|
|
83
|
-
| JsExn(
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
// Deterministic "the range returned too much data" errors that carry no numeric
|
|
95
|
-
// block-range suggestion (HyperRPC's 50k-log cap, response-size and result-count
|
|
96
|
-
// limits). They depend on log density, not on a fixed block window, so waiting
|
|
97
|
-
// never helps — the same range always re-trips the same cap. The reaction is to
|
|
98
|
-
// shrink the range and retry immediately, ratcheting the max range down.
|
|
99
|
-
let isResponseTooLargeError = {
|
|
100
|
-
// Provider-specific "too many logs" messages, matched loosely since each names
|
|
101
|
-
// it differently: HyperRPC ("More than 50000 logs returned"), ZkEVM ("query
|
|
102
|
-
// returned more than N results"), LlamaRPC ("query exceeds max results"),
|
|
103
|
-
// 1RPC ("response size should not..."), Optimism ("(backend) response too
|
|
104
|
-
// large"), Arbitrum ("logs matched by query exceeds limit"), Ankr ("block
|
|
105
|
-
// range is too wide"). Kept as one block rather than per-line comments — the
|
|
106
|
-
// ReScript formatter doesn't preserve comments attached to regex literals in
|
|
107
|
-
// an array.
|
|
108
|
-
let patterns = [
|
|
109
|
-
/more than \d+ logs/i,
|
|
110
|
-
/\d+ logs returned/i,
|
|
111
|
-
/too many logs/i,
|
|
112
|
-
/query returned more than \d+ results/i,
|
|
113
|
-
/query exceeds max results/i,
|
|
114
|
-
/response size should not/i,
|
|
115
|
-
/(backend )?response too large/i,
|
|
116
|
-
/logs matched by query exceeds limit/i,
|
|
117
|
-
/block range is too wide/i,
|
|
118
|
-
]
|
|
119
|
-
(exn: exn) =>
|
|
120
|
-
switch exn->getErrorMessage {
|
|
121
|
-
| Some(message) => patterns->Array.some(re => re->RegExp.test(message))
|
|
122
|
-
| None => false
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
let getSuggestedBlockIntervalFromExn = {
|
|
127
|
-
// Unknown provider: "retry with the range 123-456"
|
|
128
|
-
let suggestedRangeRegExp = /retry with the range (\d+)-(\d+)/
|
|
129
|
-
|
|
130
|
-
// QuickNode, 1RPC, Blast: "limited to a 1000 blocks range"
|
|
131
|
-
let blockRangeLimitRegExp = /limited to a (\d+) blocks range/
|
|
132
|
-
|
|
133
|
-
// Alchemy: "up to a 500 block range"
|
|
134
|
-
let alchemyRangeRegExp = /up to a (\d+) block range/
|
|
135
|
-
|
|
136
|
-
// Cloudflare: "Max range: 3500"
|
|
137
|
-
let cloudflareRangeRegExp = /Max range: (\d+)/
|
|
138
|
-
|
|
139
|
-
// Thirdweb: "Maximum allowed number of requested blocks is 3500"
|
|
140
|
-
let thirdwebRangeRegExp = /Maximum allowed number of requested blocks is (\d+)/
|
|
141
|
-
|
|
142
|
-
// BlockPI: "limited to 2000 block"
|
|
143
|
-
let blockpiRangeRegExp = /limited to (\d+) block/
|
|
144
|
-
|
|
145
|
-
// Base: "block range too large" - fixed 2000 block limit
|
|
146
|
-
let baseRangeRegExp = /block range too large/
|
|
147
|
-
|
|
148
|
-
// evm-rpc.sei-apis.com: "block range too large (2000), maximum allowed is 1000 blocks"
|
|
149
|
-
let maxAllowedBlocksRegExp = /maximum allowed is (\d+) blocks/
|
|
150
|
-
|
|
151
|
-
// Blast (paid): "exceeds the range allowed for your plan (5000 > 3000)"
|
|
152
|
-
let blastPaidRegExp = /exceeds the range allowed for your plan \(\d+ > (\d+)\)/
|
|
153
|
-
|
|
154
|
-
// Chainstack: "Block range limit exceeded" - 10000 block limit
|
|
155
|
-
let chainstackRegExp = /Block range limit exceeded./
|
|
156
|
-
|
|
157
|
-
// Coinbase: "please limit the query to at most 1000 blocks"
|
|
158
|
-
let coinbaseRegExp = /please limit the query to at most (\d+) blocks/
|
|
159
|
-
|
|
160
|
-
// PublicNode: "maximum block range: 2000"
|
|
161
|
-
let publicNodeRegExp = /maximum block range: (\d+)/
|
|
162
|
-
|
|
163
|
-
// Hyperliquid: "query exceeds max block range 1000"
|
|
164
|
-
let hyperliquidRegExp = /query exceeds max block range (\d+)/
|
|
165
|
-
|
|
166
|
-
// TODO: Reproduce how the error message looks like
|
|
167
|
-
// when we send request with numeric block range instead of hex
|
|
168
|
-
// Infura, ZkSync: "Try with this block range [0x123,0x456]"
|
|
169
|
-
|
|
170
|
-
let parseMessageForBlockRange = (message: string) => {
|
|
171
|
-
// Helper to extract block range from regex match
|
|
172
|
-
let extractBlockRange = (execResult, ~isMaxRange) =>
|
|
173
|
-
switch execResult->RegExp.Result.matches {
|
|
174
|
-
| [Some(blockRangeLimit)] =>
|
|
175
|
-
switch blockRangeLimit->Int.fromString {
|
|
176
|
-
| Some(blockRangeLimit) if blockRangeLimit > 0 => Some(blockRangeLimit, isMaxRange)
|
|
177
|
-
| _ => None
|
|
178
|
-
}
|
|
179
|
-
| _ => None
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
// Try each regex pattern in order
|
|
183
|
-
switch suggestedRangeRegExp->RegExp.exec(message) {
|
|
184
|
-
| Some(execResult) =>
|
|
185
|
-
switch execResult->RegExp.Result.matches {
|
|
186
|
-
| [Some(fromBlock), Some(toBlock)] =>
|
|
187
|
-
switch (fromBlock->Int.fromString, toBlock->Int.fromString) {
|
|
188
|
-
| (Some(fromBlock), Some(toBlock)) if toBlock >= fromBlock =>
|
|
189
|
-
Some(toBlock - fromBlock + 1, false)
|
|
83
|
+
| JsExn(e) =>
|
|
84
|
+
switch e->JsExn.message {
|
|
85
|
+
| Some(msg) =>
|
|
86
|
+
switch msg->JSON.parseOrThrow->JSON.Decode.object {
|
|
87
|
+
| exception _ => None
|
|
88
|
+
| None => None
|
|
89
|
+
| Some(obj) =>
|
|
90
|
+
switch obj->Dict.get("errorMessage") {
|
|
91
|
+
| Some(String(message)) => Some(message)
|
|
190
92
|
| _ => None
|
|
191
93
|
}
|
|
192
|
-
| _ => None
|
|
193
|
-
}
|
|
194
|
-
| None =>
|
|
195
|
-
// Try each provider's specific error pattern
|
|
196
|
-
switch blockRangeLimitRegExp->RegExp.exec(message) {
|
|
197
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
198
|
-
| None =>
|
|
199
|
-
switch alchemyRangeRegExp->RegExp.exec(message) {
|
|
200
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
201
|
-
| None =>
|
|
202
|
-
switch cloudflareRangeRegExp->RegExp.exec(message) {
|
|
203
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
204
|
-
| None =>
|
|
205
|
-
switch thirdwebRangeRegExp->RegExp.exec(message) {
|
|
206
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
207
|
-
| None =>
|
|
208
|
-
switch blockpiRangeRegExp->RegExp.exec(message) {
|
|
209
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
210
|
-
| None =>
|
|
211
|
-
switch maxAllowedBlocksRegExp->RegExp.exec(message) {
|
|
212
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
213
|
-
| None =>
|
|
214
|
-
switch baseRangeRegExp->RegExp.exec(message) {
|
|
215
|
-
| Some(_) => Some(2000, true)
|
|
216
|
-
| None =>
|
|
217
|
-
switch blastPaidRegExp->RegExp.exec(message) {
|
|
218
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
219
|
-
| None =>
|
|
220
|
-
switch chainstackRegExp->RegExp.exec(message) {
|
|
221
|
-
| Some(_) => Some(10000, true)
|
|
222
|
-
| None =>
|
|
223
|
-
switch coinbaseRegExp->RegExp.exec(message) {
|
|
224
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
225
|
-
| None =>
|
|
226
|
-
switch publicNodeRegExp->RegExp.exec(message) {
|
|
227
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
228
|
-
| None =>
|
|
229
|
-
switch hyperliquidRegExp->RegExp.exec(message) {
|
|
230
|
-
| Some(execResult) => extractBlockRange(execResult, ~isMaxRange=true)
|
|
231
|
-
| None => None
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
94
|
}
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
(exn): option<(
|
|
248
|
-
// The suggested block range
|
|
249
|
-
int,
|
|
250
|
-
// Whether it's the max range that the provider allows
|
|
251
|
-
bool,
|
|
252
|
-
)> =>
|
|
253
|
-
switch exn->getErrorMessage {
|
|
254
|
-
| Some(message) => parseMessageForBlockRange(message)
|
|
255
95
|
| None => None
|
|
256
96
|
}
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
type eventBatchQuery = {
|
|
260
|
-
items: array<EvmRpcClient.rpcEventItem>,
|
|
261
|
-
latestFetchedBlockInfo: blockInfo,
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
let maxSuggestedBlockIntervalKey = "max"
|
|
265
|
-
|
|
266
|
-
let getSourceMaxBlockInterval = (mutSuggestedBlockIntervals, ~intervalCeiling) =>
|
|
267
|
-
mutSuggestedBlockIntervals
|
|
268
|
-
->Utils.Dict.dangerouslyGetNonOption(maxSuggestedBlockIntervalKey)
|
|
269
|
-
->Option.getOr(intervalCeiling)
|
|
270
|
-
|
|
271
|
-
type logSelection = {
|
|
272
|
-
addresses: option<array<Address.t>>,
|
|
273
|
-
topicQuery: Rpc.GetLogs.topicQuery,
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
// A log can satisfy more than one selection when a single event's `where` is an
|
|
277
|
-
// OR of param groups, so dedup the fanned-out responses by (blockNumber,
|
|
278
|
-
// logIndex) — unique per chain — keeping the first occurrence.
|
|
279
|
-
let mergeAndDedupItems = (itemsPerSelection: array<array<EvmRpcClient.rpcEventItem>>) => {
|
|
280
|
-
let seen = Utils.Set.make()
|
|
281
|
-
let merged = []
|
|
282
|
-
itemsPerSelection->Array.forEach(items =>
|
|
283
|
-
items->Array.forEach((item: EvmRpcClient.rpcEventItem) => {
|
|
284
|
-
let key = `${item.log.blockNumber->Int.toString}-${item.log.logIndex->Int.toString}`
|
|
285
|
-
if seen->Utils.Set.has(key)->not {
|
|
286
|
-
seen->Utils.Set.add(key)->ignore
|
|
287
|
-
merged->Array.push(item)->ignore
|
|
288
|
-
}
|
|
289
|
-
})
|
|
290
|
-
)
|
|
291
|
-
merged
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
let getNextPage = (
|
|
295
|
-
~fromBlock,
|
|
296
|
-
~toBlock,
|
|
297
|
-
~logSelections: array<logSelection>,
|
|
298
|
-
~loadBlock,
|
|
299
|
-
~syncConfig as sc: Config.sourceSync,
|
|
300
|
-
~rpcClient: EvmRpcClient.t,
|
|
301
|
-
~mutSuggestedBlockIntervals,
|
|
302
|
-
~partitionId,
|
|
303
|
-
~recordRequest: (~method: string, ~seconds: float) => unit,
|
|
304
|
-
): promise<eventBatchQuery> => {
|
|
305
|
-
//If the query hangs for longer than this, reject this promise to reduce the block interval
|
|
306
|
-
let queryTimoutPromise =
|
|
307
|
-
Time.resolvePromiseAfterDelay(~delayMilliseconds=sc.queryTimeoutMillis)->Promise.then(() =>
|
|
308
|
-
Promise.reject(
|
|
309
|
-
QueryTimout(`Query took longer than ${Int.toString(sc.queryTimeoutMillis / 1000)} seconds`),
|
|
310
|
-
)
|
|
311
|
-
)
|
|
312
|
-
|
|
313
|
-
let latestFetchedBlockPromise = loadBlock(toBlock)
|
|
314
|
-
|
|
315
|
-
let queryLogs = ({addresses, topicQuery}: logSelection) => {
|
|
316
|
-
let timerRef = Performance.now()
|
|
317
|
-
rpcClient.getLogs({
|
|
318
|
-
fromBlock,
|
|
319
|
-
toBlock,
|
|
320
|
-
?addresses,
|
|
321
|
-
topics: topicQuery->Array.map(filter =>
|
|
322
|
-
switch filter {
|
|
323
|
-
| Rpc.GetLogs.Null => Nullable.null
|
|
324
|
-
| Single(topic) => Nullable.make([topic])
|
|
325
|
-
| Multiple(topics) => Nullable.make(topics)
|
|
326
|
-
}
|
|
327
|
-
),
|
|
328
|
-
})->Promise.thenResolve(items => {
|
|
329
|
-
recordRequest(~method="eth_getLogs", ~seconds=timerRef->Performance.secondsSince)
|
|
330
|
-
items
|
|
331
|
-
})
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
let logsPromise = switch logSelections {
|
|
335
|
-
| [] =>
|
|
336
|
-
latestFetchedBlockPromise->Promise.thenResolve((latestFetchedBlockInfo): eventBatchQuery => {
|
|
337
|
-
items: [],
|
|
338
|
-
latestFetchedBlockInfo,
|
|
339
|
-
})
|
|
340
|
-
// Fast path: a single selection needs no cross-request merge or dedup.
|
|
341
|
-
| [logSelection] =>
|
|
342
|
-
logSelection
|
|
343
|
-
->queryLogs
|
|
344
|
-
->Promise.then(async items => {
|
|
345
|
-
{
|
|
346
|
-
items,
|
|
347
|
-
latestFetchedBlockInfo: await latestFetchedBlockPromise,
|
|
348
|
-
}
|
|
349
|
-
})
|
|
350
|
-
| _ =>
|
|
351
|
-
logSelections
|
|
352
|
-
->Array.map(queryLogs)
|
|
353
|
-
->Promise.all
|
|
354
|
-
->Promise.then(async itemsPerSelection => {
|
|
355
|
-
{
|
|
356
|
-
items: itemsPerSelection->mergeAndDedupItems,
|
|
357
|
-
latestFetchedBlockInfo: await latestFetchedBlockPromise,
|
|
358
|
-
}
|
|
359
|
-
})
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
[queryTimoutPromise, logsPromise]
|
|
363
|
-
->Promise.race
|
|
364
|
-
->Promise.catch(err => {
|
|
365
|
-
let executedBlockInterval = toBlock - fromBlock + 1
|
|
366
|
-
let shrunkBlockInterval = Pervasives.max(
|
|
367
|
-
1,
|
|
368
|
-
(executedBlockInterval->Int.toFloat *. sc.backoffMultiplicative)->Float.toInt,
|
|
369
|
-
)
|
|
370
|
-
|
|
371
|
-
let throwFailedGettingItems = retry =>
|
|
372
|
-
throw(Source.GetItemsError(FailedGettingItems({exn: err, attemptedToBlock: toBlock, retry})))
|
|
373
|
-
let throwResize = interval =>
|
|
374
|
-
throwFailedGettingItems(WithSuggestedToBlock({toBlock: fromBlock + interval - 1}))
|
|
375
|
-
|
|
376
|
-
switch getSuggestedBlockIntervalFromExn(err) {
|
|
377
|
-
// "limited to N blocks" — a structural cap on the whole source; only tighten.
|
|
378
|
-
| Some((interval, true)) =>
|
|
379
|
-
let capped = Pervasives.min(
|
|
380
|
-
mutSuggestedBlockIntervals->getSourceMaxBlockInterval(~intervalCeiling=sc.intervalCeiling),
|
|
381
|
-
interval,
|
|
382
|
-
)
|
|
383
|
-
mutSuggestedBlockIntervals->Dict.set(maxSuggestedBlockIntervalKey, capped)
|
|
384
|
-
throwResize(capped)
|
|
385
|
-
// A one-off suggested range ("retry with range X-Y") — apply to this partition.
|
|
386
|
-
| Some((interval, false)) =>
|
|
387
|
-
mutSuggestedBlockIntervals->Dict.set(partitionId, interval)
|
|
388
|
-
throwResize(interval)
|
|
389
|
-
// Density cap with no suggested number (too many logs / response too large):
|
|
390
|
-
// shrink THIS partition and retry immediately (no wait); acceleration
|
|
391
|
-
// re-adapts on the next successful query. The interval>1 guard avoids a
|
|
392
|
-
// no-progress tight loop on a single over-cap block.
|
|
393
|
-
| None if executedBlockInterval > 1 && err->isResponseTooLargeError =>
|
|
394
|
-
mutSuggestedBlockIntervals->Dict.set(partitionId, shrunkBlockInterval)
|
|
395
|
-
throwResize(shrunkBlockInterval)
|
|
396
|
-
// Transient/unknown — shrink this partition and back off.
|
|
397
|
-
| None =>
|
|
398
|
-
mutSuggestedBlockIntervals->Dict.set(partitionId, shrunkBlockInterval)
|
|
399
|
-
throwFailedGettingItems(
|
|
400
|
-
WithBackoff({
|
|
401
|
-
message: `Failed getting data for the block range. Will try smaller block range for the next attempt.`,
|
|
402
|
-
backoffMillis: sc.backoffMillis,
|
|
403
|
-
}),
|
|
404
|
-
)
|
|
405
|
-
}
|
|
406
|
-
})
|
|
407
|
-
}
|
|
408
|
-
|
|
409
|
-
type selectionConfig = {
|
|
410
|
-
getLogSelectionsOrThrow: (
|
|
411
|
-
~addressesByContractName: dict<array<Address.t>>,
|
|
412
|
-
) => array<logSelection>,
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
let getSelectionConfig = (selection: FetchState.selection) => {
|
|
416
|
-
let evmOnEventRegistrations =
|
|
417
|
-
selection.onEventRegistrations->(
|
|
418
|
-
Utils.magic: array<Internal.onEventRegistration> => array<Internal.evmOnEventRegistration>
|
|
419
|
-
)
|
|
420
|
-
|
|
421
|
-
if evmOnEventRegistrations->Utils.Array.isEmpty {
|
|
422
|
-
throw(
|
|
423
|
-
Source.GetItemsError(
|
|
424
|
-
UnsupportedSelection({
|
|
425
|
-
message: "Invalid events configuration for the partition. Nothing to fetch. Please, report to the Envio team.",
|
|
426
|
-
}),
|
|
427
|
-
),
|
|
428
|
-
)
|
|
97
|
+
| _ => None
|
|
429
98
|
}
|
|
430
99
|
|
|
431
|
-
// eth_getLogs takes one address list and one topic selection per request, so
|
|
432
|
-
// fan out to one request per selection. Each address-bound event is grouped by
|
|
433
|
-
// its contract and later scoped to that contract's own addresses — pooling all
|
|
434
|
-
// contracts' addresses would let one contract's query fetch a sibling's logs,
|
|
435
|
-
// which route back by address and bypass the sibling's filter (routing never
|
|
436
|
-
// re-applies it). Pure-wildcard events carry no address constraint, so they're
|
|
437
|
-
// pooled and resolved once.
|
|
438
|
-
let noAddressTopicSelections = []
|
|
439
|
-
let byContract = Dict.make()
|
|
440
|
-
let wildcardByContract = Dict.make()
|
|
441
|
-
let contractNames = Utils.Set.make()
|
|
442
|
-
|
|
443
|
-
evmOnEventRegistrations->Array.forEach(reg => {
|
|
444
|
-
let contractName = reg.eventConfig.contractName
|
|
445
|
-
let {isWildcard, dependsOnAddresses, resolvedWhere} = reg
|
|
446
|
-
if dependsOnAddresses {
|
|
447
|
-
contractNames->Utils.Set.add(contractName)->ignore
|
|
448
|
-
(isWildcard ? wildcardByContract : byContract)->Utils.Dict.pushMany(
|
|
449
|
-
contractName,
|
|
450
|
-
resolvedWhere.topicSelections,
|
|
451
|
-
)
|
|
452
|
-
} else {
|
|
453
|
-
noAddressTopicSelections
|
|
454
|
-
->Array.pushMany(
|
|
455
|
-
resolvedWhere.topicSelections->LogSelection.materializeTopicSelections(~addresses=[]),
|
|
456
|
-
)
|
|
457
|
-
->ignore
|
|
458
|
-
}
|
|
459
|
-
})
|
|
460
|
-
|
|
461
|
-
// `compressTopicSelections` folds the filter-less events into a single topic0
|
|
462
|
-
// OR-set, keeping the common case at one request.
|
|
463
|
-
let toLogSelections = (~addresses, topicSelections): array<logSelection> =>
|
|
464
|
-
topicSelections
|
|
465
|
-
->LogSelection.compressTopicSelections
|
|
466
|
-
->Array.map(topicSelection => {
|
|
467
|
-
addresses,
|
|
468
|
-
topicQuery: topicSelection->Rpc.GetLogs.mapTopicQuery,
|
|
469
|
-
})
|
|
470
|
-
|
|
471
|
-
// Address-independent, so resolve once (the wildcard partition reuses this).
|
|
472
|
-
let noAddressLogSelections = toLogSelections(~addresses=None, noAddressTopicSelections)
|
|
473
100
|
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
101
|
+
// `EvmRpcClient.getNextPage` throws a napi error whose message is a JSON
|
|
102
|
+
// payload describing the retry decision:
|
|
103
|
+
// `{"kind":"Retry","attemptedToBlock":int,"errorMessage":string|null,
|
|
104
|
+
// "requestStats":[{"method":string,"seconds":float}],"retry":
|
|
105
|
+
// {"tag":"WithSuggestedToBlock","toBlock":int} |
|
|
106
|
+
// {"tag":"WithBackoff","message":string,"backoffMillis":int}}`.
|
|
107
|
+
let parseGetNextPageRetryError = (exn: exn): option<(
|
|
108
|
+
int,
|
|
109
|
+
Source.getItemsRetry,
|
|
110
|
+
array<Source.requestStat>,
|
|
111
|
+
)> =>
|
|
112
|
+
switch exn {
|
|
113
|
+
| JsExn(e) =>
|
|
114
|
+
switch e->JsExn.message {
|
|
115
|
+
| Some(msg) =>
|
|
116
|
+
switch msg->JSON.parseOrThrow->JSON.Decode.object {
|
|
117
|
+
| exception _ => None
|
|
118
|
+
| None => None
|
|
119
|
+
| Some(obj) =>
|
|
120
|
+
switch (obj->Dict.get("kind"), obj->Dict.get("attemptedToBlock"), obj->Dict.get("retry")) {
|
|
121
|
+
| (Some(String("Retry")), Some(Number(attemptedToBlock)), Some(Object(retryObj))) =>
|
|
122
|
+
let requestStats = switch obj->Dict.get("requestStats") {
|
|
123
|
+
| Some(Array(stats)) =>
|
|
124
|
+
stats->Array.filterMap(s =>
|
|
125
|
+
switch s->JSON.Decode.object {
|
|
126
|
+
| Some(o) =>
|
|
127
|
+
switch (o->Dict.get("method"), o->Dict.get("seconds")) {
|
|
128
|
+
| (Some(String(method)), Some(Number(seconds))) => Some({Source.method, seconds})
|
|
129
|
+
| _ => None
|
|
130
|
+
}
|
|
131
|
+
| None => None
|
|
132
|
+
}
|
|
493
133
|
)
|
|
494
|
-
|
|
495
|
-
| None => ()
|
|
134
|
+
| _ => []
|
|
496
135
|
}
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
)
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
|
136
|
+
let retry = switch retryObj->Dict.get("tag") {
|
|
137
|
+
| Some(String("WithSuggestedToBlock")) =>
|
|
138
|
+
switch retryObj->Dict.get("toBlock") {
|
|
139
|
+
| Some(Number(toBlock)) =>
|
|
140
|
+
Some(Source.WithSuggestedToBlock({toBlock: toBlock->Float.toInt}))
|
|
141
|
+
| _ => None
|
|
142
|
+
}
|
|
143
|
+
| Some(String("WithBackoff")) =>
|
|
144
|
+
switch (retryObj->Dict.get("message"), retryObj->Dict.get("backoffMillis")) {
|
|
145
|
+
| (Some(String(message)), Some(Number(backoffMillis))) =>
|
|
146
|
+
Some(Source.WithBackoff({message, backoffMillis: backoffMillis->Float.toInt}))
|
|
147
|
+
| _ => None
|
|
148
|
+
}
|
|
149
|
+
| _ => None
|
|
511
150
|
}
|
|
151
|
+
retry->Option.map(retry => (attemptedToBlock->Float.toInt, retry, requestStats))
|
|
152
|
+
| _ => None
|
|
512
153
|
}
|
|
513
|
-
}
|
|
514
|
-
|
|
154
|
+
}
|
|
155
|
+
| None => None
|
|
515
156
|
}
|
|
157
|
+
| _ => None
|
|
516
158
|
}
|
|
517
159
|
|
|
518
|
-
{
|
|
519
|
-
getLogSelectionsOrThrow: getLogSelectionsOrThrow,
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
|
|
523
|
-
let memoGetSelectionConfig = () => Utils.WeakMap.memoize(getSelectionConfig)
|
|
524
|
-
|
|
525
160
|
// Type-erase a schema for storage in the field registry
|
|
526
161
|
external toFieldSchema: S.t<'a> => S.t<JSON.t> = "%identity"
|
|
527
162
|
|
|
@@ -980,8 +615,8 @@ type options = {
|
|
|
980
615
|
syncConfig: Config.sourceSync,
|
|
981
616
|
url: string,
|
|
982
617
|
chain: ChainMap.Chain.t,
|
|
983
|
-
|
|
984
|
-
|
|
618
|
+
// The chain's registrations, indexed by their sequential `index`.
|
|
619
|
+
onEventRegistrations: array<Internal.evmOnEventRegistration>,
|
|
985
620
|
lowercaseAddresses: bool,
|
|
986
621
|
ws?: string,
|
|
987
622
|
headers?: dict<string>,
|
|
@@ -993,8 +628,7 @@ let make = (
|
|
|
993
628
|
syncConfig,
|
|
994
629
|
url,
|
|
995
630
|
chain,
|
|
996
|
-
|
|
997
|
-
allEventParams,
|
|
631
|
+
onEventRegistrations,
|
|
998
632
|
lowercaseAddresses,
|
|
999
633
|
?ws,
|
|
1000
634
|
?headers,
|
|
@@ -1010,19 +644,14 @@ let make = (
|
|
|
1010
644
|
}
|
|
1011
645
|
let name = `RPC (${urlHost})`
|
|
1012
646
|
|
|
1013
|
-
let getSelectionConfig = memoGetSelectionConfig()
|
|
1014
|
-
|
|
1015
|
-
// Per-partition adaptive block interval (AIMD), keyed by partitionId. The
|
|
1016
|
-
// `max` key holds a source-wide ceiling that only ever tightens, set by
|
|
1017
|
-
// structural provider limits ("limited to N blocks"). A partition's own entry
|
|
1018
|
-
// can go stale when partitions merge/split — acceptable, it re-adapts.
|
|
1019
|
-
let mutSuggestedBlockIntervals = Dict.make()
|
|
1020
|
-
|
|
1021
647
|
let client = Rpc.makeClient(url, ~headers?)
|
|
1022
648
|
let rpcClient = EvmRpcClient.make(
|
|
1023
649
|
~url,
|
|
1024
|
-
~
|
|
650
|
+
~eventRegistrations=HyperSyncClient.Registration.fromOnEventRegistrations(
|
|
651
|
+
onEventRegistrations,
|
|
652
|
+
),
|
|
1025
653
|
~checksumAddresses=!lowercaseAddresses,
|
|
654
|
+
~syncConfig,
|
|
1026
655
|
~headers?,
|
|
1027
656
|
)
|
|
1028
657
|
|
|
@@ -1171,7 +800,7 @@ let make = (
|
|
|
1171
800
|
~fromBlock,
|
|
1172
801
|
~toBlock,
|
|
1173
802
|
~addressesByContractName,
|
|
1174
|
-
~contractNameByAddress,
|
|
803
|
+
~contractNameByAddress as _,
|
|
1175
804
|
~knownHeight,
|
|
1176
805
|
~partitionId,
|
|
1177
806
|
~selection: FetchState.selection,
|
|
@@ -1181,27 +810,12 @@ let make = (
|
|
|
1181
810
|
) => {
|
|
1182
811
|
let startFetchingBatchTimeRef = Performance.now()
|
|
1183
812
|
|
|
1184
|
-
let sourceMaxBlockInterval =
|
|
1185
|
-
mutSuggestedBlockIntervals->getSourceMaxBlockInterval(
|
|
1186
|
-
~intervalCeiling=syncConfig.intervalCeiling,
|
|
1187
|
-
)
|
|
1188
|
-
let suggestedBlockInterval = Pervasives.min(
|
|
1189
|
-
mutSuggestedBlockIntervals
|
|
1190
|
-
->Utils.Dict.dangerouslyGetNonOption(partitionId)
|
|
1191
|
-
->Option.getOr(syncConfig.initialBlockInterval),
|
|
1192
|
-
sourceMaxBlockInterval,
|
|
1193
|
-
)
|
|
1194
|
-
|
|
1195
813
|
// Always have a toBlock for an RPC worker
|
|
1196
814
|
let toBlock = switch toBlock {
|
|
1197
815
|
| Some(toBlock) => Pervasives.min(toBlock, knownHeight)
|
|
1198
816
|
| None => knownHeight
|
|
1199
817
|
}
|
|
1200
818
|
|
|
1201
|
-
let suggestedToBlock = Pervasives.min(fromBlock + suggestedBlockInterval - 1, toBlock)
|
|
1202
|
-
//Defensively ensure we never query a target block below fromBlock
|
|
1203
|
-
->Pervasives.max(fromBlock)
|
|
1204
|
-
|
|
1205
819
|
let firstBlockParentPromise =
|
|
1206
820
|
fromBlock > 0
|
|
1207
821
|
? blockLoader.contents
|
|
@@ -1209,67 +823,64 @@ let make = (
|
|
|
1209
823
|
->Promise.thenResolve(json => Some(parseBlockInfo(json)))
|
|
1210
824
|
: Promise.resolve(None)
|
|
1211
825
|
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
~logSelections,
|
|
1219
|
-
~loadBlock=blockNumber =>
|
|
1220
|
-
blockLoader.contents
|
|
1221
|
-
->LazyLoader.get(blockNumber)
|
|
1222
|
-
->Promise.thenResolve(parseBlockInfo),
|
|
1223
|
-
~syncConfig,
|
|
1224
|
-
~rpcClient,
|
|
1225
|
-
~mutSuggestedBlockIntervals,
|
|
1226
|
-
~partitionId,
|
|
1227
|
-
~recordRequest,
|
|
1228
|
-
)
|
|
1229
|
-
|
|
1230
|
-
let executedBlockInterval = suggestedToBlock - fromBlock + 1
|
|
1231
|
-
|
|
1232
|
-
// Grow this partition's interval only when the full suggested range was
|
|
1233
|
-
// actually applied (not clamped by a hard toBlock). The min clamps to the
|
|
1234
|
-
// source-wide ceiling, which also stops growth once a structural cap tightened it.
|
|
1235
|
-
// See: https://en.wikipedia.org/wiki/Additive_increase/multiplicative_decrease
|
|
1236
|
-
if executedBlockInterval >= suggestedBlockInterval {
|
|
1237
|
-
mutSuggestedBlockIntervals->Dict.set(
|
|
1238
|
-
partitionId,
|
|
1239
|
-
Pervasives.min(
|
|
1240
|
-
executedBlockInterval + syncConfig.accelerationAdditive,
|
|
1241
|
-
sourceMaxBlockInterval,
|
|
826
|
+
if selection.onEventRegistrations->Utils.Array.isEmpty {
|
|
827
|
+
throw(
|
|
828
|
+
Source.GetItemsError(
|
|
829
|
+
UnsupportedSelection({
|
|
830
|
+
message: "Invalid events configuration for the partition. Nothing to fetch. Please, report to the Envio team.",
|
|
831
|
+
}),
|
|
1242
832
|
),
|
|
1243
833
|
)
|
|
1244
834
|
}
|
|
1245
835
|
|
|
1246
|
-
let
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
836
|
+
let {items, toBlock: queriedToBlock, requestStats} = try await rpcClient.getNextPage({
|
|
837
|
+
fromBlock,
|
|
838
|
+
toBlockCeiling: toBlock,
|
|
839
|
+
partitionId,
|
|
840
|
+
registrationIndexes: selection.onEventRegistrations->Array.map(reg => reg.index),
|
|
841
|
+
addressesByContractName,
|
|
842
|
+
}) catch {
|
|
843
|
+
| exn =>
|
|
844
|
+
switch exn->parseGetNextPageRetryError {
|
|
845
|
+
| Some((attemptedToBlock, retry, requestStats)) =>
|
|
846
|
+
requestStats->Array.forEach(stat =>
|
|
847
|
+
recordRequest(~method=stat.method, ~seconds=stat.seconds)
|
|
848
|
+
)
|
|
849
|
+
throw(Source.GetItemsError(FailedGettingItems({exn, attemptedToBlock, retry})))
|
|
850
|
+
| None =>
|
|
851
|
+
throw(
|
|
852
|
+
Source.GetItemsError(
|
|
853
|
+
FailedGettingItems({
|
|
854
|
+
exn,
|
|
855
|
+
attemptedToBlock: toBlock,
|
|
856
|
+
retry: WithBackoff({
|
|
857
|
+
message: "Unexpected issue while fetching events from the RPC client. Attempt a retry.",
|
|
858
|
+
backoffMillis: switch retry {
|
|
859
|
+
| 0 => 500
|
|
860
|
+
| _ => 1000 * retry
|
|
861
|
+
},
|
|
862
|
+
}),
|
|
863
|
+
}),
|
|
864
|
+
),
|
|
865
|
+
)
|
|
1253
866
|
}
|
|
867
|
+
}
|
|
868
|
+
requestStats->Array.forEach(stat => recordRequest(~method=stat.method, ~seconds=stat.seconds))
|
|
1254
869
|
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1269
|
-
| Some(decoded) =>
|
|
1270
|
-
Some(
|
|
1271
|
-
(
|
|
1272
|
-
async () => {
|
|
870
|
+
let latestFetchedBlockInfo = await blockLoader.contents
|
|
871
|
+
->LazyLoader.get(queriedToBlock)
|
|
872
|
+
->Promise.thenResolve(parseBlockInfo)
|
|
873
|
+
|
|
874
|
+
let parsedQueueItems = await items
|
|
875
|
+
->Array.map(({log, onEventRegistrationIndex, params: decoded}: EvmRpcClient.rpcEventItem) => {
|
|
876
|
+
// `log.address` comes back already normalized to the client's casing.
|
|
877
|
+
let onEventRegistration = onEventRegistrations->Array.getUnsafe(onEventRegistrationIndex)
|
|
878
|
+
let eventConfig =
|
|
879
|
+
onEventRegistration.eventConfig->(
|
|
880
|
+
Utils.magic: Internal.eventConfig => Internal.evmEventConfig
|
|
881
|
+
)
|
|
882
|
+
(
|
|
883
|
+
async () => {
|
|
1273
884
|
let (block, transaction) = try await Promise.all2((
|
|
1274
885
|
log->getEventBlockOrThrow(~selectedBlockFields=eventConfig.selectedBlockFields),
|
|
1275
886
|
log->getEventTransactionOrThrow(
|
|
@@ -1321,16 +932,12 @@ let make = (
|
|
|
1321
932
|
params: decoded,
|
|
1322
933
|
block,
|
|
1323
934
|
transaction,
|
|
1324
|
-
srcAddress:
|
|
935
|
+
srcAddress: log.address,
|
|
1325
936
|
logIndex: log.logIndex,
|
|
1326
937
|
}->Evm.fromPayload,
|
|
1327
938
|
})
|
|
1328
|
-
}
|
|
1329
|
-
)(),
|
|
1330
|
-
)
|
|
1331
|
-
| None => None
|
|
1332
939
|
}
|
|
1333
|
-
|
|
940
|
+
)()
|
|
1334
941
|
})
|
|
1335
942
|
->Promise.all
|
|
1336
943
|
|