envio 3.0.2 → 3.1.0-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (101) hide show
  1. package/README.md +0 -1
  2. package/evm.schema.json +15 -8
  3. package/fuel.schema.json +19 -12
  4. package/index.d.ts +0 -2
  5. package/package.json +6 -7
  6. package/rescript.json +1 -1
  7. package/src/Batch.res +4 -214
  8. package/src/Batch.res.mjs +6 -165
  9. package/src/ChainFetcher.res +12 -28
  10. package/src/ChainFetcher.res.mjs +8 -17
  11. package/src/ChainManager.res +10 -9
  12. package/src/ChainManager.res.mjs +6 -10
  13. package/src/Config.res +9 -25
  14. package/src/Config.res.mjs +17 -27
  15. package/src/Core.res +7 -0
  16. package/src/Ctx.res +1 -0
  17. package/src/Env.res +0 -8
  18. package/src/Env.res.mjs +0 -6
  19. package/src/EventConfigBuilder.res +13 -123
  20. package/src/EventConfigBuilder.res.mjs +6 -73
  21. package/src/EventProcessing.res +5 -29
  22. package/src/EventProcessing.res.mjs +11 -20
  23. package/src/EventUtils.res +0 -27
  24. package/src/EventUtils.res.mjs +0 -24
  25. package/src/FetchState.res +2 -15
  26. package/src/FetchState.res.mjs +3 -18
  27. package/src/GlobalState.res +26 -39
  28. package/src/GlobalState.res.mjs +12 -40
  29. package/src/HandlerLoader.res +6 -5
  30. package/src/HandlerLoader.res.mjs +27 -9
  31. package/src/HandlerRegister.res +1 -12
  32. package/src/HandlerRegister.res.mjs +1 -6
  33. package/src/HandlerRegister.resi +1 -1
  34. package/src/Hasura.res +96 -32
  35. package/src/Hasura.res.mjs +93 -38
  36. package/src/InMemoryStore.res +205 -45
  37. package/src/InMemoryStore.res.mjs +157 -40
  38. package/src/InMemoryTable.res +165 -249
  39. package/src/InMemoryTable.res.mjs +156 -227
  40. package/src/Internal.res +10 -34
  41. package/src/Internal.res.mjs +9 -3
  42. package/src/LoadLayer.res +5 -5
  43. package/src/LoadLayer.res.mjs +5 -5
  44. package/src/LogSelection.res +15 -19
  45. package/src/LogSelection.res.mjs +5 -6
  46. package/src/Main.res +4 -6
  47. package/src/Main.res.mjs +26 -15
  48. package/src/Persistence.res +7 -132
  49. package/src/Persistence.res.mjs +1 -102
  50. package/src/PgStorage.res +57 -40
  51. package/src/PgStorage.res.mjs +60 -34
  52. package/src/ReorgDetection.res +35 -58
  53. package/src/ReorgDetection.res.mjs +21 -29
  54. package/src/SimulateItems.res.mjs +21 -3
  55. package/src/Sink.res +2 -2
  56. package/src/Sink.res.mjs +1 -1
  57. package/src/TableIndices.res +9 -2
  58. package/src/TableIndices.res.mjs +7 -1
  59. package/src/TestIndexer.res +53 -60
  60. package/src/TestIndexer.res.mjs +77 -63
  61. package/src/TestIndexerProxyStorage.res +4 -14
  62. package/src/TestIndexerProxyStorage.res.mjs +1 -5
  63. package/src/UserContext.res +2 -4
  64. package/src/UserContext.res.mjs +4 -5
  65. package/src/Utils.res +0 -2
  66. package/src/Utils.res.mjs +0 -3
  67. package/src/bindings/ClickHouse.res +45 -38
  68. package/src/bindings/ClickHouse.res.mjs +16 -17
  69. package/src/bindings/Vitest.res +3 -0
  70. package/src/db/InternalTable.res +59 -18
  71. package/src/db/InternalTable.res.mjs +82 -51
  72. package/src/db/Table.res +9 -2
  73. package/src/db/Table.res.mjs +10 -7
  74. package/src/sources/EnvioApiClient.res +15 -0
  75. package/src/sources/EnvioApiClient.res.mjs +24 -0
  76. package/src/sources/EvmChain.res +32 -10
  77. package/src/sources/EvmChain.res.mjs +31 -5
  78. package/src/sources/HyperFuelSource.res +15 -58
  79. package/src/sources/HyperFuelSource.res.mjs +20 -39
  80. package/src/sources/HyperSync.res +54 -100
  81. package/src/sources/HyperSync.res.mjs +67 -96
  82. package/src/sources/HyperSync.resi +4 -22
  83. package/src/sources/HyperSyncClient.res +70 -247
  84. package/src/sources/HyperSyncClient.res.mjs +47 -46
  85. package/src/sources/HyperSyncSource.res +94 -166
  86. package/src/sources/HyperSyncSource.res.mjs +100 -127
  87. package/src/sources/RpcSource.res +43 -22
  88. package/src/sources/RpcSource.res.mjs +50 -35
  89. package/src/sources/SimulateSource.res +1 -7
  90. package/src/sources/SimulateSource.res.mjs +1 -7
  91. package/src/sources/Source.res +10 -1
  92. package/src/sources/Source.res.mjs +3 -0
  93. package/src/sources/SourceManager.res +177 -8
  94. package/src/sources/SourceManager.res.mjs +141 -3
  95. package/src/sources/SourceManager.resi +19 -0
  96. package/src/tui/Tui.res +44 -6
  97. package/src/tui/Tui.res.mjs +56 -8
  98. package/src/tui/components/TuiData.res +3 -0
  99. package/svm.schema.json +11 -4
  100. package/src/sources/HyperSyncJsonApi.res +0 -390
  101. package/src/sources/HyperSyncJsonApi.res.mjs +0 -237
package/README.md CHANGED
@@ -70,7 +70,6 @@ A few things already running in production:
70
70
  **Multichain indexing**
71
71
  - Index EVM, SVM, and Fuel blockchains from a single indexer
72
72
  - 70+ EVM chains with native HyperSync support, plus any EVM chain via RPC
73
- - Unordered multichain mode for maximum throughput across chains
74
73
  - Real-time indexing with reorg handling built in
75
74
 
76
75
  **Developer experience**
package/evm.schema.json CHANGED
@@ -4,6 +4,10 @@
4
4
  "description": "Schema for a YAML config for an envio indexer",
5
5
  "type": "object",
6
6
  "properties": {
7
+ "name": {
8
+ "description": "Name of the project",
9
+ "type": "string"
10
+ },
7
11
  "description": {
8
12
  "description": "Description of the project",
9
13
  "type": [
@@ -11,10 +15,6 @@
11
15
  "null"
12
16
  ]
13
17
  },
14
- "name": {
15
- "description": "Name of the project",
16
- "type": "string"
17
- },
18
18
  "schema": {
19
19
  "description": "Custom path to schema.graphql file",
20
20
  "type": [
@@ -67,7 +67,7 @@
67
67
  "null"
68
68
  ],
69
69
  "items": {
70
- "$ref": "#/$defs/GlobalContract_for_ContractConfig"
70
+ "$ref": "#/$defs/GlobalContract"
71
71
  }
72
72
  },
73
73
  "chains": {
@@ -189,7 +189,7 @@
189
189
  "evm"
190
190
  ]
191
191
  },
192
- "GlobalContract_for_ContractConfig": {
192
+ "GlobalContract": {
193
193
  "type": "object",
194
194
  "properties": {
195
195
  "name": {
@@ -356,6 +356,13 @@
356
356
  "format": "uint64",
357
357
  "minimum": 0
358
358
  },
359
+ "skip": {
360
+ "description": "Excludes the chain from indexing and migrations. Code generation is unaffected. For testing, prefer using a test framework instead.",
361
+ "type": [
362
+ "boolean",
363
+ "null"
364
+ ]
365
+ },
359
366
  "rpc": {
360
367
  "description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
361
368
  "anyOf": [
@@ -418,7 +425,7 @@
418
425
  "null"
419
426
  ],
420
427
  "items": {
421
- "$ref": "#/$defs/ChainContract_for_ContractConfig"
428
+ "$ref": "#/$defs/ChainContract"
422
429
  }
423
430
  }
424
431
  },
@@ -578,7 +585,7 @@
578
585
  "url"
579
586
  ]
580
587
  },
581
- "ChainContract_for_ContractConfig": {
588
+ "ChainContract": {
582
589
  "type": "object",
583
590
  "properties": {
584
591
  "name": {
package/fuel.schema.json CHANGED
@@ -4,6 +4,10 @@
4
4
  "description": "Schema for a YAML config for an envio indexer",
5
5
  "type": "object",
6
6
  "properties": {
7
+ "name": {
8
+ "description": "Name of the project",
9
+ "type": "string"
10
+ },
7
11
  "description": {
8
12
  "description": "Description of the project",
9
13
  "type": [
@@ -11,10 +15,6 @@
11
15
  "null"
12
16
  ]
13
17
  },
14
- "name": {
15
- "description": "Name of the project",
16
- "type": "string"
17
- },
18
18
  "schema": {
19
19
  "description": "Custom path to schema.graphql file",
20
20
  "type": [
@@ -60,7 +60,7 @@
60
60
  "null"
61
61
  ],
62
62
  "items": {
63
- "$ref": "#/$defs/GlobalContract_for_ContractConfig"
63
+ "$ref": "#/$defs/GlobalContract"
64
64
  }
65
65
  },
66
66
  "chains": {
@@ -147,7 +147,7 @@
147
147
  "fuel"
148
148
  ]
149
149
  },
150
- "GlobalContract_for_ContractConfig": {
150
+ "GlobalContract": {
151
151
  "type": "object",
152
152
  "properties": {
153
153
  "name": {
@@ -183,6 +183,10 @@
183
183
  "EventConfig": {
184
184
  "type": "object",
185
185
  "properties": {
186
+ "name": {
187
+ "description": "Name of the event in the HyperIndex generated code",
188
+ "type": "string"
189
+ },
186
190
  "type": {
187
191
  "description": "Explicitly set the event type you want to index. It's derived from the event name and fallbacks to LogData.",
188
192
  "anyOf": [
@@ -194,10 +198,6 @@
194
198
  }
195
199
  ]
196
200
  },
197
- "name": {
198
- "description": "Name of the event in the HyperIndex generated code",
199
- "type": "string"
200
- },
201
201
  "logId": {
202
202
  "description": "An identifier of a logged type from ABI. Used for indexing LogData receipts. The option can be omitted when the event name matches the logged struct/enum name.",
203
203
  "type": [
@@ -230,6 +230,13 @@
230
230
  "format": "uint64",
231
231
  "minimum": 0
232
232
  },
233
+ "skip": {
234
+ "description": "Excludes the chain from indexing and migrations. Code generation is unaffected. For testing, prefer using a test framework instead.",
235
+ "type": [
236
+ "boolean",
237
+ "null"
238
+ ]
239
+ },
233
240
  "start_block": {
234
241
  "description": "The block at which the indexer should start ingesting data",
235
242
  "type": "integer",
@@ -281,7 +288,7 @@
281
288
  "null"
282
289
  ],
283
290
  "items": {
284
- "$ref": "#/$defs/ChainContract_for_ContractConfig"
291
+ "$ref": "#/$defs/ChainContract"
285
292
  }
286
293
  }
287
294
  },
@@ -304,7 +311,7 @@
304
311
  "url"
305
312
  ]
306
313
  },
307
- "ChainContract_for_ContractConfig": {
314
+ "ChainContract": {
308
315
  "type": "object",
309
316
  "properties": {
310
317
  "name": {
package/index.d.ts CHANGED
@@ -299,8 +299,6 @@ type IndexerConfig = {
299
299
  description?: string;
300
300
  /** Path to handlers directory for auto-loading (default: "src/handlers"). */
301
301
  handlers?: string;
302
- /** Multichain mode: ordered or unordered (default: "unordered"). */
303
- multichain?: "ordered" | "unordered";
304
302
  /** Target batch size for event processing (default: 5000). */
305
303
  fullBatchSize?: number;
306
304
  /** Whether to rollback on chain reorg (default: true). */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "envio",
3
- "version": "3.0.2",
3
+ "version": "3.1.0-rc.1",
4
4
  "type": "module",
5
5
  "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
6
6
  "bin": "./bin.mjs",
@@ -43,7 +43,6 @@
43
43
  "@clickhouse/client": "1.17.0",
44
44
  "@elastic/ecs-pino-format": "1.4.0",
45
45
  "@envio-dev/hyperfuel-client": "1.2.2",
46
- "@envio-dev/hypersync-client": "1.3.0",
47
46
  "@fuel-ts/crypto": "0.96.1",
48
47
  "@fuel-ts/errors": "0.96.1",
49
48
  "@fuel-ts/hasher": "0.96.1",
@@ -71,10 +70,10 @@
71
70
  "tsx": "4.21.0"
72
71
  },
73
72
  "optionalDependencies": {
74
- "envio-linux-x64": "3.0.2",
75
- "envio-linux-x64-musl": "3.0.2",
76
- "envio-linux-arm64": "3.0.2",
77
- "envio-darwin-x64": "3.0.2",
78
- "envio-darwin-arm64": "3.0.2"
73
+ "envio-linux-x64": "3.1.0-rc.1",
74
+ "envio-linux-x64-musl": "3.1.0-rc.1",
75
+ "envio-linux-arm64": "3.1.0-rc.1",
76
+ "envio-darwin-x64": "3.1.0-rc.1",
77
+ "envio-darwin-arm64": "3.1.0-rc.1"
79
78
  }
80
79
  }
package/rescript.json CHANGED
@@ -18,6 +18,6 @@
18
18
  "dependencies": ["rescript-schema", "@rescript/react"],
19
19
  "compiler-flags": ["-open RescriptSchema"],
20
20
  "warnings": {
21
- "error": "+3"
21
+ "error": "+a"
22
22
  }
23
23
  }
package/src/Batch.res CHANGED
@@ -31,50 +31,7 @@ type t = {
31
31
  checkpointEventsProcessed: array<int>,
32
32
  }
33
33
 
34
- /**
35
- It either returnes an earliest item among all chains, or None if no chains are actively indexing
36
- */
37
- let getOrderedNextChain = (fetchStates: ChainMap.t<FetchState.t>, ~batchSizePerChain) => {
38
- let earliestChain: ref<option<FetchState.t>> = ref(None)
39
- let earliestChainTimestamp = ref(0)
40
- let chainKeys = fetchStates->ChainMap.keys
41
- for idx in 0 to chainKeys->Array.length - 1 {
42
- let chain = chainKeys->Array.getUnsafe(idx)
43
- let fetchState = fetchStates->ChainMap.get(chain)
44
- if fetchState->FetchState.isActivelyIndexing {
45
- let timestamp = fetchState->FetchState.getTimestampAt(
46
- ~index=switch batchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
47
- chain->ChainMap.Chain.toChainId,
48
- ) {
49
- | Some(batchSize) => batchSize
50
- | None => 0
51
- },
52
- )
53
- switch earliestChain.contents {
54
- | Some(earliestChain)
55
- if timestamp > earliestChainTimestamp.contents ||
56
- (timestamp === earliestChainTimestamp.contents &&
57
- chain->ChainMap.Chain.toChainId > earliestChain.chainId) => ()
58
- | _ => {
59
- earliestChain := Some(fetchState)
60
- earliestChainTimestamp := timestamp
61
- }
62
- }
63
- }
64
- }
65
- earliestChain.contents
66
- }
67
-
68
- // Save overhead of recreating the dict every time
69
- let immutableEmptyBatchSizePerChain: dict<int> = Dict.make()
70
- let hasOrderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
71
- switch fetchStates->getOrderedNextChain(~batchSizePerChain=immutableEmptyBatchSizePerChain) {
72
- | Some(fetchState) => fetchState->FetchState.hasReadyItem
73
- | None => false
74
- }
75
- }
76
-
77
- let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
34
+ let hasReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
78
35
  fetchStates
79
36
  ->ChainMap.values
80
37
  ->Array.some(fetchState => {
@@ -82,16 +39,6 @@ let hasUnorderedReadyItem = (fetchStates: ChainMap.t<FetchState.t>) => {
82
39
  })
83
40
  }
84
41
 
85
- let hasMultichainReadyItem = (
86
- fetchStates: ChainMap.t<FetchState.t>,
87
- ~multichain: Config.multichain,
88
- ) => {
89
- switch multichain {
90
- | Ordered => hasOrderedReadyItem(fetchStates)
91
- | Unordered => hasUnorderedReadyItem(fetchStates)
92
- }
93
- }
94
-
95
42
  let getProgressedChainsById = {
96
43
  let getChainAfterBatchIfProgressed = (
97
44
  ~chainBeforeBatch: chainBeforeBatch,
@@ -214,154 +161,7 @@ let addReorgCheckpoints = (
214
161
  }
215
162
  }
216
163
 
217
- let prepareOrderedBatch = (
218
- ~checkpointIdBeforeBatch,
219
- ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
220
- ~batchSizeTarget,
221
- ) => {
222
- let totalBatchSize = ref(0)
223
- let isFinished = ref(false)
224
- let prevCheckpointId = ref(checkpointIdBeforeBatch)
225
- let mutBatchSizePerChain = Dict.make()
226
- let mutProgressBlockNumberPerChain = Dict.make()
227
-
228
- let fetchStates = chainsBeforeBatch->ChainMap.map(chainBeforeBatch => chainBeforeBatch.fetchState)
229
-
230
- let items = []
231
- let checkpointIds = []
232
- let checkpointChainIds = []
233
- let checkpointBlockNumbers = []
234
- let checkpointBlockHashes = []
235
- let checkpointEventsProcessed = []
236
-
237
- while totalBatchSize.contents < batchSizeTarget && !isFinished.contents {
238
- switch fetchStates->getOrderedNextChain(~batchSizePerChain=mutBatchSizePerChain) {
239
- | Some(fetchState) => {
240
- let chainBeforeBatch =
241
- chainsBeforeBatch->ChainMap.get(ChainMap.Chain.makeUnsafe(~chainId=fetchState.chainId))
242
- let itemsCountBefore = switch mutBatchSizePerChain->Utils.Dict.dangerouslyGetByIntNonOption(
243
- fetchState.chainId,
244
- ) {
245
- | Some(batchSize) => batchSize
246
- | None => 0
247
- }
248
-
249
- let prevBlockNumber = switch mutProgressBlockNumberPerChain->Utils.Dict.dangerouslyGetByIntNonOption(
250
- fetchState.chainId,
251
- ) {
252
- | Some(progressBlockNumber) => progressBlockNumber
253
- | None => chainBeforeBatch.progressBlockNumber
254
- }
255
-
256
- let newItemsCount = fetchState->FetchState.getReadyItemsCount(
257
- // We should get items only for a single block
258
- // Since for the ordered mode next block could be after another chain's block
259
- ~targetSize=1,
260
- ~fromItem=itemsCountBefore,
261
- )
262
-
263
- if newItemsCount > 0 {
264
- let item0 = fetchState.buffer->Array.getUnsafe(itemsCountBefore)
265
- let blockNumber = item0->Internal.getItemBlockNumber
266
-
267
- prevCheckpointId :=
268
- addReorgCheckpoints(
269
- ~chainId=fetchState.chainId,
270
- ~reorgDetection=chainBeforeBatch.reorgDetection,
271
- ~prevCheckpointId=prevCheckpointId.contents,
272
- ~fromBlockExclusive=prevBlockNumber,
273
- ~toBlockExclusive=blockNumber,
274
- ~mutCheckpointIds=checkpointIds,
275
- ~mutCheckpointChainIds=checkpointChainIds,
276
- ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
277
- ~mutCheckpointBlockHashes=checkpointBlockHashes,
278
- ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
279
- )
280
-
281
- let checkpointId = prevCheckpointId.contents->BigInt.add(1n)
282
-
283
- items
284
- ->Array.push(item0)
285
- ->ignore
286
- for idx in 1 to newItemsCount - 1 {
287
- items
288
- ->Array.push(fetchState.buffer->Belt.Array.getUnsafe(itemsCountBefore + idx))
289
- ->ignore
290
- }
291
-
292
- checkpointIds
293
- ->Array.push(checkpointId)
294
- ->ignore
295
- checkpointChainIds
296
- ->Array.push(fetchState.chainId)
297
- ->ignore
298
- checkpointBlockNumbers
299
- ->Array.push(blockNumber)
300
- ->ignore
301
- checkpointBlockHashes
302
- ->Array.push(
303
- chainBeforeBatch.reorgDetection->ReorgDetection.getHashByBlockNumber(~blockNumber),
304
- )
305
- ->ignore
306
- checkpointEventsProcessed
307
- ->Array.push(newItemsCount)
308
- ->ignore
309
-
310
- prevCheckpointId := checkpointId
311
- totalBatchSize := totalBatchSize.contents + newItemsCount
312
- mutBatchSizePerChain->Utils.Dict.setByInt(
313
- fetchState.chainId,
314
- itemsCountBefore + newItemsCount,
315
- )
316
- mutProgressBlockNumberPerChain->Utils.Dict.setByInt(fetchState.chainId, blockNumber)
317
- } else {
318
- let blockNumberAfterBatch = fetchState->FetchState.bufferBlockNumber
319
-
320
- prevCheckpointId :=
321
- addReorgCheckpoints(
322
- ~chainId=fetchState.chainId,
323
- ~reorgDetection=chainBeforeBatch.reorgDetection,
324
- ~prevCheckpointId=prevCheckpointId.contents,
325
- ~fromBlockExclusive=prevBlockNumber,
326
- ~toBlockExclusive=blockNumberAfterBatch + 1, // Make it inclusive
327
- ~mutCheckpointIds=checkpointIds,
328
- ~mutCheckpointChainIds=checkpointChainIds,
329
- ~mutCheckpointBlockNumbers=checkpointBlockNumbers,
330
- ~mutCheckpointBlockHashes=checkpointBlockHashes,
331
- ~mutCheckpointEventsProcessed=checkpointEventsProcessed,
332
- )
333
-
334
- // Since the chain was chosen as next
335
- // the fact that it doesn't have new items means that it reached the buffer block number
336
- mutProgressBlockNumberPerChain->Utils.Dict.setByInt(
337
- fetchState.chainId,
338
- blockNumberAfterBatch,
339
- )
340
- isFinished := true
341
- }
342
- }
343
-
344
- | None => isFinished := true
345
- }
346
- }
347
-
348
- {
349
- totalBatchSize: totalBatchSize.contents,
350
- items,
351
- progressedChainsById: getProgressedChainsById(
352
- ~chainsBeforeBatch,
353
- ~batchSizePerChain=mutBatchSizePerChain,
354
- ~progressBlockNumberPerChain=mutProgressBlockNumberPerChain,
355
- ),
356
- checkpointIds,
357
- checkpointChainIds,
358
- checkpointBlockNumbers,
359
- checkpointBlockHashes,
360
- checkpointEventsProcessed,
361
- }
362
- }
363
-
364
- let prepareUnorderedBatch = (
164
+ let prepareBatch = (
365
165
  ~checkpointIdBeforeBatch,
366
166
  ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
367
167
  ~batchSizeTarget,
@@ -454,7 +254,7 @@ let prepareUnorderedBatch = (
454
254
  }
455
255
 
456
256
  let progressBlockNumberAfterBatch =
457
- fetchState->FetchState.getUnorderedMultichainProgressBlockNumberAt(~index=chainBatchSize)
257
+ fetchState->FetchState.getProgressBlockNumberAt(~index=chainBatchSize)
458
258
 
459
259
  prevCheckpointId :=
460
260
  addReorgCheckpoints(
@@ -497,19 +297,9 @@ let prepareUnorderedBatch = (
497
297
  let make = (
498
298
  ~checkpointIdBeforeBatch,
499
299
  ~chainsBeforeBatch: ChainMap.t<chainBeforeBatch>,
500
- ~multichain: Config.multichain,
501
300
  ~batchSizeTarget,
502
301
  ) => {
503
- if (
504
- switch multichain {
505
- | Unordered => true
506
- | Ordered => chainsBeforeBatch->ChainMap.size === 1
507
- }
508
- ) {
509
- prepareUnorderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
510
- } else {
511
- prepareOrderedBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
512
- }
302
+ prepareBatch(~checkpointIdBeforeBatch, ~chainsBeforeBatch, ~batchSizeTarget)
513
303
  }
514
304
 
515
305
  let findFirstEventBlockNumber = (batch: t, ~chainId) => {
package/src/Batch.res.mjs CHANGED
@@ -5,38 +5,7 @@ import * as ChainMap from "./ChainMap.res.mjs";
5
5
  import * as FetchState from "./FetchState.res.mjs";
6
6
  import * as ReorgDetection from "./ReorgDetection.res.mjs";
7
7
 
8
- function getOrderedNextChain(fetchStates, batchSizePerChain) {
9
- let earliestChain;
10
- let earliestChainTimestamp = 0;
11
- let chainKeys = ChainMap.keys(fetchStates);
12
- for (let idx = 0, idx_finish = chainKeys.length - 1; idx <= idx_finish; ++idx) {
13
- let chain = chainKeys[idx];
14
- let fetchState = ChainMap.get(fetchStates, chain);
15
- if (FetchState.isActivelyIndexing(fetchState)) {
16
- let batchSize = batchSizePerChain[chain];
17
- let timestamp = FetchState.getTimestampAt(fetchState, batchSize !== undefined ? batchSize : 0);
18
- let earliestChain$1 = earliestChain;
19
- if (!(earliestChain$1 !== undefined && (timestamp > earliestChainTimestamp || timestamp === earliestChainTimestamp && chain > earliestChain$1.chainId))) {
20
- earliestChain = fetchState;
21
- earliestChainTimestamp = timestamp;
22
- }
23
- }
24
- }
25
- return earliestChain;
26
- }
27
-
28
- let immutableEmptyBatchSizePerChain = {};
29
-
30
- function hasOrderedReadyItem(fetchStates) {
31
- let fetchState = getOrderedNextChain(fetchStates, immutableEmptyBatchSizePerChain);
32
- if (fetchState !== undefined) {
33
- return FetchState.hasReadyItem(fetchState);
34
- } else {
35
- return false;
36
- }
37
- }
38
-
39
- function hasUnorderedReadyItem(fetchStates) {
8
+ function hasReadyItem(fetchStates) {
40
9
  return ChainMap.values(fetchStates).some(fetchState => {
41
10
  if (FetchState.isActivelyIndexing(fetchState)) {
42
11
  return FetchState.hasReadyItem(fetchState);
@@ -46,14 +15,6 @@ function hasUnorderedReadyItem(fetchStates) {
46
15
  });
47
16
  }
48
17
 
49
- function hasMultichainReadyItem(fetchStates, multichain) {
50
- if (multichain === "ordered") {
51
- return hasOrderedReadyItem(fetchStates);
52
- } else {
53
- return hasUnorderedReadyItem(fetchStates);
54
- }
55
- }
56
-
57
18
  function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, fetchStateAfterBatch, batchSize) {
58
19
  if (chainBeforeBatch.progressBlockNumber < progressBlockNumberAfterBatch) {
59
20
  return {
@@ -109,114 +70,7 @@ function addReorgCheckpoints(prevCheckpointId, reorgDetection, fromBlockExclusiv
109
70
  return prevCheckpointId$1;
110
71
  }
111
72
 
112
- function prepareOrderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
113
- let totalBatchSize = 0;
114
- let isFinished = false;
115
- let prevCheckpointId = checkpointIdBeforeBatch;
116
- let mutBatchSizePerChain = {};
117
- let mutProgressBlockNumberPerChain = {};
118
- let fetchStates = ChainMap.map(chainsBeforeBatch, chainBeforeBatch => chainBeforeBatch.fetchState);
119
- let items = [];
120
- let checkpointIds = [];
121
- let checkpointChainIds = [];
122
- let checkpointBlockNumbers = [];
123
- let checkpointBlockHashes = [];
124
- let checkpointEventsProcessed = [];
125
- while (totalBatchSize < batchSizeTarget && !isFinished) {
126
- let fetchState = getOrderedNextChain(fetchStates, mutBatchSizePerChain);
127
- if (fetchState !== undefined) {
128
- let chainBeforeBatch = ChainMap.get(chainsBeforeBatch, ChainMap.Chain.makeUnsafe(fetchState.chainId));
129
- let batchSize = mutBatchSizePerChain[fetchState.chainId];
130
- let itemsCountBefore = batchSize !== undefined ? batchSize : 0;
131
- let progressBlockNumber = mutProgressBlockNumberPerChain[fetchState.chainId];
132
- let prevBlockNumber = progressBlockNumber !== undefined ? progressBlockNumber : chainBeforeBatch.progressBlockNumber;
133
- let newItemsCount = FetchState.getReadyItemsCount(fetchState, 1, itemsCountBefore);
134
- if (newItemsCount > 0) {
135
- let item0 = fetchState.buffer[itemsCountBefore];
136
- let blockNumber = item0.blockNumber;
137
- let chainId = fetchState.chainId;
138
- let reorgDetection = chainBeforeBatch.reorgDetection;
139
- let prevCheckpointId$1 = prevCheckpointId;
140
- let tmp;
141
- if (reorgDetection.shouldRollbackOnReorg && !Utils.Dict.isEmpty(reorgDetection.dataByBlockNumber)) {
142
- let prevCheckpointId$2 = prevCheckpointId$1;
143
- for (let blockNumber$1 = prevBlockNumber + 1, blockNumber_finish = blockNumber - 1; blockNumber$1 <= blockNumber_finish; ++blockNumber$1) {
144
- let hash = ReorgDetection.getHashByBlockNumber(reorgDetection, blockNumber$1);
145
- if (hash !== null) {
146
- let checkpointId = prevCheckpointId$2 + 1n;
147
- prevCheckpointId$2 = checkpointId;
148
- checkpointIds.push(checkpointId);
149
- checkpointChainIds.push(chainId);
150
- checkpointBlockNumbers.push(blockNumber$1);
151
- checkpointBlockHashes.push(hash);
152
- checkpointEventsProcessed.push(0);
153
- }
154
- }
155
- tmp = prevCheckpointId$2;
156
- } else {
157
- tmp = prevCheckpointId$1;
158
- }
159
- prevCheckpointId = tmp;
160
- let checkpointId$1 = prevCheckpointId + 1n;
161
- items.push(item0);
162
- for (let idx = 1, idx_finish = newItemsCount - 1; idx <= idx_finish; ++idx) {
163
- items.push(fetchState.buffer[itemsCountBefore + idx]);
164
- }
165
- checkpointIds.push(checkpointId$1);
166
- checkpointChainIds.push(fetchState.chainId);
167
- checkpointBlockNumbers.push(blockNumber);
168
- checkpointBlockHashes.push(ReorgDetection.getHashByBlockNumber(chainBeforeBatch.reorgDetection, blockNumber));
169
- checkpointEventsProcessed.push(newItemsCount);
170
- prevCheckpointId = checkpointId$1;
171
- totalBatchSize = totalBatchSize + newItemsCount;
172
- mutBatchSizePerChain[fetchState.chainId] = itemsCountBefore + newItemsCount;
173
- mutProgressBlockNumberPerChain[fetchState.chainId] = blockNumber;
174
- } else {
175
- let blockNumberAfterBatch = FetchState.bufferBlockNumber(fetchState);
176
- let chainId$1 = fetchState.chainId;
177
- let toBlockExclusive = blockNumberAfterBatch + 1;
178
- let reorgDetection$1 = chainBeforeBatch.reorgDetection;
179
- let prevCheckpointId$3 = prevCheckpointId;
180
- let tmp$1;
181
- if (reorgDetection$1.shouldRollbackOnReorg && !Utils.Dict.isEmpty(reorgDetection$1.dataByBlockNumber)) {
182
- let prevCheckpointId$4 = prevCheckpointId$3;
183
- for (let blockNumber$2 = prevBlockNumber + 1, blockNumber_finish$1 = toBlockExclusive - 1; blockNumber$2 <= blockNumber_finish$1; ++blockNumber$2) {
184
- let hash$1 = ReorgDetection.getHashByBlockNumber(reorgDetection$1, blockNumber$2);
185
- if (hash$1 !== null) {
186
- let checkpointId$2 = prevCheckpointId$4 + 1n;
187
- prevCheckpointId$4 = checkpointId$2;
188
- checkpointIds.push(checkpointId$2);
189
- checkpointChainIds.push(chainId$1);
190
- checkpointBlockNumbers.push(blockNumber$2);
191
- checkpointBlockHashes.push(hash$1);
192
- checkpointEventsProcessed.push(0);
193
- }
194
- }
195
- tmp$1 = prevCheckpointId$4;
196
- } else {
197
- tmp$1 = prevCheckpointId$3;
198
- }
199
- prevCheckpointId = tmp$1;
200
- mutProgressBlockNumberPerChain[fetchState.chainId] = blockNumberAfterBatch;
201
- isFinished = true;
202
- }
203
- } else {
204
- isFinished = true;
205
- }
206
- };
207
- return {
208
- totalBatchSize: totalBatchSize,
209
- items: items,
210
- progressedChainsById: getProgressedChainsById(chainsBeforeBatch, mutBatchSizePerChain, mutProgressBlockNumberPerChain),
211
- checkpointIds: checkpointIds,
212
- checkpointChainIds: checkpointChainIds,
213
- checkpointBlockNumbers: checkpointBlockNumbers,
214
- checkpointBlockHashes: checkpointBlockHashes,
215
- checkpointEventsProcessed: checkpointEventsProcessed
216
- };
217
- }
218
-
219
- function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
73
+ function prepareBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget) {
220
74
  let preparedFetchStates = FetchState.sortForUnorderedBatch(ChainMap.values(chainsBeforeBatch).map(chainBeforeBatch => chainBeforeBatch.fetchState), batchSizeTarget);
221
75
  let chainIdx = 0;
222
76
  let preparedNumber = preparedFetchStates.length;
@@ -281,7 +135,7 @@ function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batch
281
135
  totalBatchSize = totalBatchSize + chainBatchSize;
282
136
  mutBatchSizePerChain[fetchState.chainId] = chainBatchSize;
283
137
  }
284
- let progressBlockNumberAfterBatch = FetchState.getUnorderedMultichainProgressBlockNumberAt(fetchState, chainBatchSize);
138
+ let progressBlockNumberAfterBatch = FetchState.getProgressBlockNumberAt(fetchState, chainBatchSize);
285
139
  let chainId$1 = fetchState.chainId;
286
140
  let toBlockExclusive = progressBlockNumberAfterBatch + 1;
287
141
  let fromBlockExclusive$1 = prevBlockNumber;
@@ -322,15 +176,7 @@ function prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batch
322
176
  };
323
177
  }
324
178
 
325
- function make(checkpointIdBeforeBatch, chainsBeforeBatch, multichain, batchSizeTarget) {
326
- let tmp;
327
- tmp = multichain === "ordered" ? ChainMap.size(chainsBeforeBatch) === 1 : true;
328
- if (tmp) {
329
- return prepareUnorderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget);
330
- } else {
331
- return prepareOrderedBatch(checkpointIdBeforeBatch, chainsBeforeBatch, batchSizeTarget);
332
- }
333
- }
179
+ let make = prepareBatch;
334
180
 
335
181
  function findFirstEventBlockNumber(batch, chainId) {
336
182
  let idx = 0;
@@ -362,15 +208,10 @@ function findLastEventItem(batch, chainId) {
362
208
  }
363
209
 
364
210
  export {
365
- getOrderedNextChain,
366
- immutableEmptyBatchSizePerChain,
367
- hasOrderedReadyItem,
368
- hasUnorderedReadyItem,
369
- hasMultichainReadyItem,
211
+ hasReadyItem,
370
212
  getProgressedChainsById,
371
213
  addReorgCheckpoints,
372
- prepareOrderedBatch,
373
- prepareUnorderedBatch,
214
+ prepareBatch,
374
215
  make,
375
216
  findFirstEventBlockNumber,
376
217
  findLastEventItem,