envio 3.0.2-svm-alpha.1 → 3.0.2-svm-alpha.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +0 -1
  2. package/evm.schema.json +7 -0
  3. package/fuel.schema.json +7 -0
  4. package/index.d.ts +0 -2
  5. package/package.json +6 -6
  6. package/src/Batch.res +4 -214
  7. package/src/Batch.res.mjs +6 -165
  8. package/src/ChainFetcher.res +4 -5
  9. package/src/ChainFetcher.res.mjs +6 -7
  10. package/src/ChainManager.res +3 -5
  11. package/src/ChainManager.res.mjs +4 -6
  12. package/src/Config.res +100 -32
  13. package/src/Config.res.mjs +102 -31
  14. package/src/Core.res +4 -8
  15. package/src/Env.res +0 -1
  16. package/src/Env.res.mjs +0 -2
  17. package/src/EventConfigBuilder.res +38 -136
  18. package/src/EventConfigBuilder.res.mjs +14 -79
  19. package/src/EventUtils.res +0 -18
  20. package/src/EventUtils.res.mjs +0 -18
  21. package/src/FetchState.res +1 -11
  22. package/src/FetchState.res.mjs +2 -16
  23. package/src/GlobalState.res +3 -8
  24. package/src/GlobalState.res.mjs +1 -13
  25. package/src/HandlerLoader.res +6 -5
  26. package/src/HandlerLoader.res.mjs +27 -9
  27. package/src/HandlerRegister.res +1 -12
  28. package/src/HandlerRegister.res.mjs +1 -6
  29. package/src/HandlerRegister.resi +1 -1
  30. package/src/Hasura.res +96 -32
  31. package/src/Hasura.res.mjs +93 -38
  32. package/src/Internal.res +10 -17
  33. package/src/Internal.res.mjs +3 -3
  34. package/src/Main.res +2 -6
  35. package/src/Main.res.mjs +24 -16
  36. package/src/Persistence.res +17 -2
  37. package/src/Persistence.res.mjs +14 -2
  38. package/src/PgStorage.res +5 -1
  39. package/src/PgStorage.res.mjs +12 -6
  40. package/src/SimulateItems.res.mjs +21 -3
  41. package/src/TestIndexer.res.mjs +23 -4
  42. package/src/bindings/ClickHouse.res +6 -2
  43. package/src/bindings/ClickHouse.res.mjs +3 -2
  44. package/src/bindings/Vitest.res +3 -0
  45. package/src/db/InternalTable.res.mjs +35 -35
  46. package/src/db/Table.res +9 -2
  47. package/src/db/Table.res.mjs +10 -7
  48. package/src/sources/EvmChain.res +32 -9
  49. package/src/sources/EvmChain.res.mjs +31 -4
  50. package/src/sources/HyperSyncClient.res +35 -79
  51. package/src/sources/HyperSyncClient.res.mjs +35 -46
  52. package/src/sources/HyperSyncSolanaClient.res +2 -0
  53. package/src/sources/HyperSyncSolanaSource.res +72 -50
  54. package/src/sources/HyperSyncSolanaSource.res.mjs +71 -57
  55. package/src/sources/HyperSyncSource.res +13 -14
  56. package/src/sources/HyperSyncSource.res.mjs +8 -5
  57. package/src/sources/RpcSource.res +11 -7
  58. package/src/sources/RpcSource.res.mjs +6 -5
  59. package/svm.schema.json +112 -29
package/README.md CHANGED
@@ -70,7 +70,6 @@ A few things already running in production:
70
70
  **Multichain indexing**
71
71
  - Index EVM, SVM, and Fuel blockchains from a single indexer
72
72
  - 70+ EVM chains with native HyperSync support, plus any EVM chain via RPC
73
- - Unordered multichain mode for maximum throughput across chains
74
73
  - Real-time indexing with reorg handling built in
75
74
 
76
75
  **Developer experience**
package/evm.schema.json CHANGED
@@ -356,6 +356,13 @@
356
356
  "format": "uint64",
357
357
  "minimum": 0
358
358
  },
359
+ "skip": {
360
+ "description": "Excludes the chain from indexing and migrations. Code generation is unaffected. For testing, prefer using a test framework instead.",
361
+ "type": [
362
+ "boolean",
363
+ "null"
364
+ ]
365
+ },
359
366
  "rpc": {
360
367
  "description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
361
368
  "anyOf": [
package/fuel.schema.json CHANGED
@@ -230,6 +230,13 @@
230
230
  "format": "uint64",
231
231
  "minimum": 0
232
232
  },
233
+ "skip": {
234
+ "description": "Excludes the chain from indexing and migrations. Code generation is unaffected. For testing, prefer using a test framework instead.",
235
+ "type": [
236
+ "boolean",
237
+ "null"
238
+ ]
239
+ },
233
240
  "start_block": {
234
241
  "description": "The block at which the indexer should start ingesting data",
235
242
  "type": "integer",
package/index.d.ts CHANGED
@@ -299,8 +299,6 @@ type IndexerConfig = {
299
299
  description?: string;
300
300
  /** Path to handlers directory for auto-loading (default: "src/handlers"). */
301
301
  handlers?: string;
302
- /** Multichain mode: ordered or unordered (default: "unordered"). */
303
- multichain?: "ordered" | "unordered";
304
302
  /** Target batch size for event processing (default: 5000). */
305
303
  fullBatchSize?: number;
306
304
  /** Whether to rollback on chain reorg (default: true). */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "envio",
3
- "version": "3.0.2-svm-alpha.1",
3
+ "version": "3.0.2-svm-alpha.2",
4
4
  "type": "module",
5
5
  "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
6
6
  "bin": "./bin.mjs",
@@ -70,10 +70,10 @@
70
70
  "tsx": "4.21.0"
71
71
  },
72
72
  "optionalDependencies": {
73
- "envio-linux-x64": "3.0.2-svm-alpha.1",
74
- "envio-linux-x64-musl": "3.0.2-svm-alpha.1",
75
- "envio-linux-arm64": "3.0.2-svm-alpha.1",
76
- "envio-darwin-x64": "3.0.2-svm-alpha.1",
77
- "envio-darwin-arm64": "3.0.2-svm-alpha.1"
73
+ "envio-linux-x64": "3.0.2-svm-alpha.2",
74
+ "envio-linux-x64-musl": "3.0.2-svm-alpha.2",
75
+ "envio-linux-arm64": "3.0.2-svm-alpha.2",
76
+ "envio-darwin-x64": "3.0.2-svm-alpha.2",
77
+ "envio-darwin-arm64": "3.0.2-svm-alpha.2"
78
78
  }
79
79
  }
package/src/Batch.res CHANGED
@@ -31,50 +31,7 @@ type t = {
31
31
  checkpointEventsProcessed: array<int>,
32
32
  }
33
33
 
34
- /**
35
- It either returnes an earliest item among all chains, or None if no chains are actively indexing
36
- */
37
- let getOrderedNextChain = (fetchStates: ChainMap.t<FetchState.t>, ~batchSizePerChain) => {
38
- let earliestChain: ref<option<FetchState.t>> = ref(None)
39
- let earliestChainTimestamp = ref(0)
40
- let chainKeys = fetchStates->ChainMap.keys
41
- for idx in 0 to chainKeys->Array.length - 1 {
42
- let chain = chainKeys->Array.getUnsafe(idx)
43
- let fetchState = fetchStates->ChainMap.get(chain)
44
- if fetchState->FetchState.isActivelyIndexing {
45
- let timestamp = fetchState->FetchState.getTimestampAt(
46
- ~index=switch batchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
47
- chain->ChainMap.Chain.toChainId,
48
- ) {
49
- | Some(batchSize) => batchSize
50
- | None => 0
51
- },
52
- )
53
- switch earliestChain.contents {
54
- | Some(earliestChain)
55
- if timestamp > earliestChainTimestamp.contents ||
56
- (timestamp === earliestChainTimestamp.contents &&
57
- chain->ChainMap.Chain.toChainId > earliestChain.chainId) => ()
58
- | _ => {
59
- earliestChain := Some(fetchState)
60
- earliestChainTimestamp := timestamp
61
- }
62
- }
63
- }
64
- }
65
- earliestChain.contents
66
- }
67
-
68
- // Save overhead of recreating the dict every time
69
- let immutableEmptyBatchSizePerChain: dict<int> = Dict.make()
70
- let hasOrderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
71
- switch fetchStates->getOrderedNextChain(~batchSizePerChain=immutableEmptyBatchSizePerChain) {
72
- | Some(fetchState) => fetchState->FetchState.hasReadyItem
73
- | None => false
74
- }
75
- }
76
-
77
- let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
34
+ let hasReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
78
35
  fetchStates
79
36
  ->ChainMap.values
80
37
  ->Array.some(fetchState => {
@@ -82,16 +39,6 @@ let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
82
39
  })
83
40
  }
84
41
 
85
- let hasMultichainReadyItem = (
86
- fetchStates: ChainMap.t<FetchState.t>,
87
- ~multichain: Config.multichain,
88
- ) => {
89
- switch multichain {
90
- | Ordered => hasOrderedReadyItem(fetchStates)
91
- | Unordered => hasUnorderedReadyItem(fetchStates)
92
- }
93
- }
94
-
95
42
  let getProgressedChainsById = {
96
43
  let getChainAfterBatchIfProgressed = (
97
44
  ~chainBeforeBatch: chainBeforeBatch,
@@ -214,154 +161,7 @@ let addReorgCheckpoints = (
214
161
  }
215
162
  }
216
163
 
217
- let prepareOrderedBatch = (
218
- ~checkpointIdBeforeBatch,
219
- ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
220
- ~batchSizeTarget,
221
- ) => {
222
- let totalBatchSize = ref(0)
223
- let isFinished = ref(false)
224
- let prevCheckpointId = ref(checkpointIdBeforeBatch)
225
- let mutBatchSizePerChain = Dict.make()
226
- let mutProgressBlockNumberPerChain = Dict.make()
227
-
228
- let fetchStates = chainsBeforeBatch->ChainMap.map(chainBeforeBatch => chainBeforeBatch.fetchState)
229
-
230
- let items = []
231
- let checkpointIds = []
232
- let checkpointChainIds = []
233
- let checkpointBlockNumbers = []
234
- let checkpointBlockHashes = []
235
- let checkpointEventsProcessed = []
236
-
237
- while totalBatchSize.contents < batchSizeTarget && !isFinished.contents {
238
- switch fetchStates->getOrderedNextChain(~batchSizePerChain=mutBatchSizePerChain) {
239
- | Some(fetchState) => {
240
- let chainBeforeBatch =
241
- chainsBeforeBatch->ChainMap.get(ChainMap.Chain.makeUnsafe(~chainId=fetchState.chainId))
242
- let itemsCountBefore = switch mutBatchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
243
- fetchState.chainId,
244
- ) {
245
- | Some(batchSize) => batchSize
246
- | None => 0
247
- }
248
-
249
- let prevBlockNumber = switch mutProgressBlockNumberPerChain->Utils.Dict.dangerouslyGetByIntNonOption(
250
- fetchState.chainId,
251
- ) {
252
- | Some(progressBlockNumber) => progressBlockNumber
253
- | None => chainBeforeBatch.progressBlockNumber
254
- }
255
-
256
- let newItemsCount = fetchState->FetchState.getReadyItemsCount(
257
- // We should get items only for a single block
258
- // Since for the ordered mode next block could be after another chain's block
259
- ~targetSize=1,
260
- ~fromItem=itemsCountBefore,
261
- )
262
-
263
- if newItemsCount > 0 {
264
- let item0 = fetchState.buffer->Array.getUnsafe(itemsCountBefore)
265
- let blockNumber = item0->Internal.getItemBlockNumber
266
-
267
- prevCheckpointId :=
268
- addReorgCheckpoints(
269
- ~chainId=fetchState.chainId,
270
- ~reorgDetection=chainBeforeBatch.reorgDetection,
271
- ~prevCheckpointId=prevCheckpointId.contents,
272
- ~fromBlockExclusive=prevBlockNumber,
273
- ~toBlockExclusive=blockNumber,
274
- ~mutCheckpointIds=checkpointIds,
275
- ~mutCheckpointChainIds=checkpointChainIds,
276
- ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
277
- ~mutCheckpointBlockHashes=checkpointBlockHashes,
278
- ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
279
- )
280
-
281
- let checkpointId = prevCheckpointId.contents->BigInt.add(1n)
282
-
283
- items
284
- ->Array.push(item0)
285
- ->ignore
286
- for idx in 1 to newItemsCount - 1 {
287
- items
288
- ->Array.push(fetchState.buffer->Belt.Array.getUnsafe(itemsCountBefore + idx))
289
- ->ignore
290
- }
291
-
292
- checkpointIds
293
- ->Array.push(checkpointId)
294
- ->ignore
295
- checkpointChainIds
296
- ->Array.push(fetchState.chainId)
297
- ->ignore
298
- checkpointBlockNumbers
299
- ->Array.push(blockNumber)
300
- ->ignore
301
- checkpointBlockHashes
302
- ->Array.push(
303
- chainBeforeBatch.reorgDetection->ReorgDetection.getHashByBlockNumber(~blockNumber),
304
- )
305
- ->ignore
306
- checkpointEventsProcessed
307
- ->Array.push(newItemsCount)
308
- ->ignore
309
-
310
- prevCheckpointId := checkpointId
311
- totalBatchSize := totalBatchSize.contents + newItemsCount
312
- mutBatchSizePerChain->Utils.Dict.setByInt(
313
- fetchState.chainId,
314
- itemsCountBefore + newItemsCount,
315
- )
316
- mutProgressBlockNumberPerChain->Utils.Dict.setByInt(fetchState.chainId, blockNumber)
317
- } else {
318
- let blockNumberAfterBatch = fetchState->FetchState.bufferBlockNumber
319
-
320
- prevCheckpointId :=
321
- addReorgCheckpoints(
322
- ~chainId=fetchState.chainId,
323
- ~reorgDetection=chainBeforeBatch.reorgDetection,
324
- ~prevCheckpointId=prevCheckpointId.contents,
325
- ~fromBlockExclusive=prevBlockNumber,
326
- ~toBlockExclusive=blockNumberAfterBatch + 1, // Make it inclusive
327
- ~mutCheckpointIds=checkpointIds,
328
- ~mutCheckpointChainIds=checkpointChainIds,
329
- ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
330
- ~mutCheckpointBlockHashes=checkpointBlockHashes,
331
- ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
332
- )
333
-
334
- // Since the chain was chosen as next
335
- // the fact that it doesn't have new items means that it reached the buffer block number
336
- mutProgressBlockNumberPerChain->Utils.Dict.setByInt(
337
- fetchState.chainId,
338
- blockNumberAfterBatch,
339
- )
340
- isFinished := true
341
- }
342
- }
343
-
344
- | None => isFinished := true
345
- }
346
- }
347
-
348
- {
349
- totalBatchSize: totalBatchSize.contents,
350
- items,
351
- progressedChainsById: getProgressedChainsById(
352
- ~chainsBeforeBatch,
353
- ~batchSizePerChain=mutBatchSizePerChain,
354
- ~progressBlockNumberPerChain=mutProgressBlockNumberPerChain,
355
- ),
356
- checkpointIds,
357
- checkpointChainIds,
358
- checkpointBlockNumbers,
359
- checkpointBlockHashes,
360
- checkpointEventsProcessed,
361
- }
362
- }
363
-
364
- let prepareUnorderedBatch = (
164
+ let prepareBatch = (
365
165
  ~checkpointIdBeforeBatch,
366
166
  ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
367
167
  ~batchSizeTarget,
@@ -454,7 +254,7 @@ let prepareUnorderedBatch = (
454
254
  }
455
255
 
456
256
  let progressBlockNumberAfterBatch =
457
- fetchState->FetchState.getUnorderedMultichainProgressBlockNumberAt(~index=chainBatchSize)
257
+ fetchState->FetchState.getProgressBlockNumberAt(~index=chainBatchSize)
458
258
 
459
259
  prevCheckpointId :=
460
260
  addReorgCheckpoints(
@@ -497,19 +297,9 @@ let prepareUnorderedBatch = (
497
297
  let make = (
498
298
  ~checkpointIdBeforeBatch,
499
299
  ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
500
- ~multichain: Config.multichain,
501
300
  ~batchSizeTarget,
502
301
  ) => {
503
- if (
504
- switch multichain {
505
- | Unordered => true
506
- | Ordered => chainsBeforeBatch->ChainMap.size === 1
507
- }
508
- ) {
509
- prepareUnorderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
510
- } else {
511
- prepareOrderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
512
- }
302
+ prepareBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
513
303
  }
514
304
 
515
305
  let findFirstEventBlockNumber = (batch: t, ~chainId) => {
package/src/Batch.res.mjs CHANGED
@@ -5,38 +5,7 @@ import * as ChainMap from "./ChainMap.res.mjs";
5
5
  import * as FetchState from "./FetchState.res.mjs";
6
6
  import * as ReorgDetection from "./ReorgDetection.res.mjs";
7
7
 
8
- function getOrderedNextChain(fetchStates, batchSizePerChain) {
9
- let earliestChain;
10
- let earliestChainTimestamp = 0;
11
- let chainKeys = ChainMap.keys(fetchStates);
12
- for (let idx = 0, idx_finish = chainKeys.length - 1; idx <= idx_finish; ++idx) {
13
- let chain = chainKeys[idx];
14
- let fetchState = ChainMap.get(fetchStates, chain);
15
- if (FetchState.isActivelyIndexing(fetchState)) {
16
- let batchSize = batchSizePerChain[chain];
17
- let timestamp = FetchState.getTimestampAt(fetchState, batchSize !== undefined ? batchSize : 0);
18
- let earliestChain$1 = earliestChain;
19
- if (!(earliestChain$1 !== undefined && (timestamp > earliestChainTimestamp || timestamp === earliestChainTimestamp && chain > earliestChain$1.chainId))) {
20
- earliestChain = fetchState;
21
- earliestChainTimestamp = timestamp;
22
- }
23
- }
24
- }
25
- return earliestChain;
26
- }
27
-
28
- let immutableEmptyBatchSizePerChain = {};
29
-
30
- function hasOrderedReadyItem(fetchStates) {
31
- let fetchState = getOrderedNextChain(fetchStates, immutableEmptyBatchSizePerChain);
32
- if (fetchState !== undefined) {
33
- return FetchState.hasReadyItem(fetchState);
34
- } else {
35
- return false;
36
- }
37
- }
38
-
39
- function hasUnorderedReadyItem(fetchStates) {
8
+ function hasReadyItem(fetchStates) {
40
9
  return ChainMap.values(fetchStates).some(fetchState => {
41
10
  if (FetchState.isActivelyIndexing(fetchState)) {
42
11
  return FetchState.hasReadyItem(fetchState);
@@ -46,14 +15,6 @@ function hasUnorderedReadyItem(fetchStates) {
46
15
  });
47
16
  }
48
17
 
49
- function hasMultichainReadyItem(fetchStates, multichain) {
50
- if (multichain === "ordered") {
51
- return hasOrderedReadyItem(fetchStates);
52
- } else {
53
- return hasUnorderedReadyItem(fetchStates);
54
- }
55
- }
56
-
57
18
  function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
58
19
  if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
59
20
  return {
@@ -109,114 +70,7 @@ function addReorgCheckpoints(prevCheckpointId, reorgDetection, fromBlockExclusiv
109
70
  return prevCheckpointId$1;
110
71
  }
111
72
 
112
- function prepareOrderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
113
- let totalBatchSize = 0;
114
- let isFinished = false;
115
- let prevCheckpointId = checkpointIdBeforeBatch;
116
- let mutBatchSizePerChain = {};
117
- let mutProgressBlockNumberPerChain = {};
118
- let fetchStates = ChainMap.map(chainsBeforeBatch, chainBeforeBatch => chainBeforeBatch.fetchState);
119
- let items = [];
120
- let checkpointIds = [];
121
- let checkpointChainIds = [];
122
- let checkpointBlockNumbers = [];
123
- let checkpointBlockHashes = [];
124
- let checkpointEventsProcessed = [];
125
- while (totalBatchSize < batchSizeTarget && !isFinished) {
126
- let fetchState = getOrderedNextChain(fetchStates, mutBatchSizePerChain);
127
- if (fetchState !== undefined) {
128
- let chainBeforeBatch = ChainMap.get(chainsBeforeBatch, ChainMap.Chain.makeUnsafe(fetchState.chainId));
129
- let batchSize = mutBatchSizePerChain[fetchState.chainId];
130
- let itemsCountBefore = batchSize !== undefined ? batchSize : 0;
131
- let progressBlockNumber = mutProgressBlockNumberPerChain[fetchState.chainId];
132
- let prevBlockNumber = progressBlockNumber !== undefined ? progressBlockNumber : chainBeforeBatch.progressBlockNumber;
133
- let newItemsCount = FetchState.getReadyItemsCount(fetchState, 1, itemsCountBefore);
134
- if (newItemsCount > 0) {
135
- let item0 = fetchState.buffer[itemsCountBefore];
136
- let blockNumber = item0.blockNumber;
137
- let chainId = fetchState.chainId;
138
- let reorgDetection = chainBeforeBatch.reorgDetection;
139
- let prevCheckpointId$1 = prevCheckpointId;
140
- let tmp;
141
- if (reorgDetection.shouldRollbackOnReorg && !Utils.Dict.isEmpty(reorgDetection.dataByBlockNumber)) {
142
- let prevCheckpointId$2 = prevCheckpointId$1;
143
- for (let blockNumber$1 = prevBlockNumber + 1, blockNumber_finish = blockNumber - 1; blockNumber$1 <= blockNumber_finish; ++blockNumber$1) {
144
- let hash = ReorgDetection.getHashByBlockNumber(reorgDetection, blockNumber$1);
145
- if (hash !== null) {
146
- let checkpointId = prevCheckpointId$2 + 1n;
147
- prevCheckpointId$2 = checkpointId;
148
- checkpointIds.push(checkpointId);
149
- checkpointChainIds.push(chainId);
150
- checkpointBlockNumbers.push(blockNumber$1);
151
- checkpointBlockHashes.push(hash);
152
- checkpointEventsProcessed.push(0);
153
- }
154
- }
155
- tmp = prevCheckpointId$2;
156
- } else {
157
- tmp = prevCheckpointId$1;
158
- }
159
- prevCheckpointId = tmp;
160
- let checkpointId$1 = prevCheckpointId + 1n;
161
- items.push(item0);
162
- for (let idx = 1, idx_finish = newItemsCount - 1; idx <= idx_finish; ++idx) {
163
- items.push(fetchState.buffer[itemsCountBefore + idx]);
164
- }
165
- checkpointIds.push(checkpointId$1);
166
- checkpointChainIds.push(fetchState.chainId);
167
- checkpointBlockNumbers.push(blockNumber);
168
- checkpointBlockHashes.push(ReorgDetection.getHashByBlockNumber(chainBeforeBatch.reorgDetection, blockNumber));
169
- checkpointEventsProcessed.push(newItemsCount);
170
- prevCheckpointId = checkpointId$1;
171
- totalBatchSize = totalBatchSize + newItemsCount;
172
- mutBatchSizePerChain[fetchState.chainId] = itemsCountBefore + newItemsCount;
173
- mutProgressBlockNumberPerChain[fetchState.chainId] = blockNumber;
174
- } else {
175
- let blockNumberAfterBatch = FetchState.bufferBlockNumber(fetchState);
176
- let chainId$1 = fetchState.chainId;
177
- let toBlockExclusive = blockNumberAfterBatch + 1;
178
- let reorgDetection$1 = chainBeforeBatch.reorgDetection;
179
- let prevCheckpointId$3 = prevCheckpointId;
180
- let tmp$1;
181
- if (reorgDetection$1.shouldRollbackOnReorg && !Utils.Dict.isEmpty(reorgDetection$1.dataByBlockNumber)) {
182
- let prevCheckpointId$4 = prevCheckpointId$3;
183
- for (let blockNumber$2 = prevBlockNumber + 1, blockNumber_finish$1 = toBlockExclusive - 1; blockNumber$2 <= blockNumber_finish$1; ++blockNumber$2) {
184
- let hash$1 = ReorgDetection.getHashByBlockNumber(reorgDetection$1, blockNumber$2);
185
- if (hash$1 !== null) {
186
- let checkpointId$2 = prevCheckpointId$4 + 1n;
187
- prevCheckpointId$4 = checkpointId$2;
188
- checkpointIds.push(checkpointId$2);
189
- checkpointChainIds.push(chainId$1);
190
- checkpointBlockNumbers.push(blockNumber$2);
191
- checkpointBlockHashes.push(hash$1);
192
- checkpointEventsProcessed.push(0);
193
- }
194
- }
195
- tmp$1 = prevCheckpointId$4;
196
- } else {
197
- tmp$1 = prevCheckpointId$3;
198
- }
199
- prevCheckpointId = tmp$1;
200
- mutProgressBlockNumberPerChain[fetchState.chainId] = blockNumberAfterBatch;
201
- isFinished = true;
202
- }
203
- } else {
204
- isFinished = true;
205
- }
206
- };
207
- return {
208
- totalBatchSize: totalBatchSize,
209
- items: items,
210
- progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutBatchSizePerChain, mutProgressBlockNumberPerChain),
211
- checkpointIds: checkpointIds,
212
- checkpointChainIds: checkpointChainIds,
213
- checkpointBlockNumbers: checkpointBlockNumbers,
214
- checkpointBlockHashes: checkpointBlockHashes,
215
- checkpointEventsProcessed: checkpointEventsProcessed
216
- };
217
- }
218
-
219
- function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
73
+ function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
220
74
  let preparedFetchStates = FetchState.sortForUnorderedBatch(ChainMap.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
221
75
  let chainIdx = 0;
222
76
  let preparedNumber = preparedFetchStates.length;
@@ -281,7 +135,7 @@ function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batch
281
135
  totalBatchSize = totalBatchSize + chainBatchSize;
282
136
  mutBatchSizePerChain[fetchState.chainId] = chainBatchSize;
283
137
  }
284
- let progressBlockNumberAfterBatch = FetchState.getUnorderedMultichainProgressBlockNumberAt(fetchState, chainBatchSize);
138
+ let progressBlockNumberAfterBatch = FetchState.getProgressBlockNumberAt(fetchState, chainBatchSize);
285
139
  let chainId$1 = fetchState.chainId;
286
140
  let toBlockExclusive = progressBlockNumberAfterBatch + 1;
287
141
  let fromBlockExclusive$1 = prevBlockNumber;
@@ -322,15 +176,7 @@ function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batch
322
176
  };
323
177
  }
324
178
 
325
- function make(checkpointIdBeforeBatch, chainsBeforeBatch, multichain, batchSizeTarget) {
326
- let tmp;
327
- tmp = multichain === "ordered" ? ChainMap.size(chainsBeforeBatch) === 1 : true;
328
- if (tmp) {
329
- return prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget);
330
- } else {
331
- return prepareOrderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget);
332
- }
333
- }
179
+ let make = prepareBatch;
334
180
 
335
181
  function findFirstEventBlockNumber(batch, chainId) {
336
182
  let idx = 0;
@@ -362,15 +208,10 @@ function findLastEventItem(batch, chainId) {
362
208
  }
363
209
 
364
210
  export {
365
- getOrderedNextChain,
366
- immutableEmptyBatchSizePerChain,
367
- hasOrderedReadyItem,
368
- hasUnorderedReadyItem,
369
- hasMultichainReadyItem,
211
+ hasReadyItem,
370
212
  getProgressedChainsById,
371
213
  addReorgCheckpoints,
372
- prepareOrderedBatch,
373
- prepareUnorderedBatch,
214
+ prepareBatch,
374
215
  make,
375
216
  findFirstEventBlockNumber,
376
217
  findLastEventItem,
@@ -51,6 +51,7 @@ let make = (
51
51
  ~reorgCheckpoints: array<Internal.reorgCheckpoint>,
52
52
  ~maxReorgDepth,
53
53
  ~knownHeight=0,
54
+ ~reducedPollingInterval=?,
54
55
  ): t => {
55
56
  // We don't need the router itself, but only validation logic,
56
57
  // since now event router is created for selection of events
@@ -210,10 +211,6 @@ let make = (
210
211
  Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
211
212
  ),
212
213
  })
213
- // Collect all event signatures from contracts
214
- let allEventSignatures =
215
- chainConfig.contracts->Array.flatMap(contract => contract.eventSignatures)
216
- // Convert rpcs to EvmChain.rpc format
217
214
  let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
218
215
  let syncConfig = rpc.syncConfig
219
216
  let ws = rpc.ws
@@ -228,7 +225,6 @@ let make = (
228
225
  ~chain,
229
226
  ~contracts=evmContracts,
230
227
  ~hyperSync=hypersync,
231
- ~allEventSignatures,
232
228
  ~rpcs=evmRpcs,
233
229
  ~lowercaseAddresses,
234
230
  )
@@ -269,6 +265,7 @@ let make = (
269
265
  ~sources,
270
266
  ~maxPartitionConcurrency=Env.maxPartitionConcurrency,
271
267
  ~isRealtime,
268
+ ~reducedPollingInterval?,
272
269
  ),
273
270
  reorgDetection: ReorgDetection.make(
274
271
  ~chainReorgCheckpoints,
@@ -329,6 +326,7 @@ let makeFromDbState = (
329
326
  ~config,
330
327
  ~registrations,
331
328
  ~targetBufferSize,
329
+ ~reducedPollingInterval=?,
332
330
  ) => {
333
331
  let chainId = chainConfig.id
334
332
  let logger = Logging.createChild(~params={"chainId": chainId})
@@ -361,6 +359,7 @@ let makeFromDbState = (
361
359
  ~isInReorgThreshold,
362
360
  ~isRealtime,
363
361
  ~knownHeight=resumedChainState.sourceBlockNumber,
362
+ ~reducedPollingInterval?,
364
363
  )
365
364
  }
366
365
 
@@ -38,7 +38,7 @@ function configAddresses(chainConfig) {
38
38
  return addresses;
39
39
  }
40
40
 
41
- function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrations, targetBufferSize, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt) {
41
+ function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrations, targetBufferSize, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, maxReorgDepth, knownHeightOpt, reducedPollingInterval) {
42
42
  let firstEventBlock = firstEventBlockOpt !== undefined ? Primitive_option.valFromOption(firstEventBlockOpt) : undefined;
43
43
  let knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
44
44
  let eventRouter = EventRouter.empty();
@@ -114,7 +114,6 @@ function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBl
114
114
  abi: contract.abi,
115
115
  events: contract.events
116
116
  }));
117
- let allEventSignatures = chainConfig.contracts.flatMap(contract => contract.eventSignatures);
118
117
  let evmRpcs = sources.rpcs.map(rpc => {
119
118
  let syncConfig = rpc.syncConfig;
120
119
  let ws = rpc.ws;
@@ -125,7 +124,7 @@ function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBl
125
124
  ws: ws
126
125
  };
127
126
  });
128
- sources$1 = EvmChain.makeSources(chain, evmContracts, sources.hypersync, allEventSignatures, evmRpcs, lowercaseAddresses);
127
+ sources$1 = EvmChain.makeSources(chain, evmContracts, sources.hypersync, evmRpcs, lowercaseAddresses);
129
128
  break;
130
129
  case "FuelSourceConfig" :
131
130
  sources$1 = [HyperFuelSource.make({
@@ -160,7 +159,7 @@ function make(chainConfig, indexingAddresses, startBlock, endBlock, firstEventBl
160
159
  return {
161
160
  logger: logger,
162
161
  fetchState: fetchState,
163
- sourceManager: SourceManager.make(sources$1, Env.maxPartitionConcurrency, isRealtime, undefined, undefined, undefined, undefined, undefined, undefined),
162
+ sourceManager: SourceManager.make(sources$1, Env.maxPartitionConcurrency, isRealtime, undefined, undefined, undefined, reducedPollingInterval, undefined, undefined),
164
163
  chainConfig: chainConfig,
165
164
  isProgressAtHead: false,
166
165
  timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
@@ -175,17 +174,17 @@ function makeFromConfig(chainConfig, config, registrations, targetBufferSize, kn
175
174
  let logger = Logging.createChild({
176
175
  chainId: chainConfig.id
177
176
  });
178
- return make(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, targetBufferSize, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight);
177
+ return make(chainConfig, configAddresses(chainConfig), chainConfig.startBlock, chainConfig.endBlock, undefined, -1, config, registrations, targetBufferSize, logger, undefined, 0, false, false, [], chainConfig.maxReorgDepth, knownHeight, undefined);
179
178
  }
180
179
 
181
- function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrations, targetBufferSize) {
180
+ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInReorgThreshold, isRealtime, config, registrations, targetBufferSize, reducedPollingInterval) {
182
181
  let chainId = chainConfig.id;
183
182
  let logger = Logging.createChild({
184
183
  chainId: chainId
185
184
  });
186
185
  Prometheus.ProgressEventsCount.set(resumedChainState.numEventsProcessed, chainId);
187
186
  let progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
188
- return make(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrations, targetBufferSize, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber);
187
+ return make(chainConfig, resumedChainState.indexingAddresses, resumedChainState.startBlock, resumedChainState.endBlock, Primitive_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrations, targetBufferSize, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, isInReorgThreshold, isRealtime, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber, reducedPollingInterval);
189
188
  }
190
189
 
191
190
  async function runContractRegistersOrThrow(itemsWithContractRegister, config) {