envio 3.0.0-alpha.9 → 3.0.0-main-alpha15-build-test

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/README.md +1 -1
  2. package/evm.schema.json +37 -119
  3. package/fuel.schema.json +18 -0
  4. package/index.d.ts +95 -5
  5. package/package.json +25 -23
  6. package/src/Batch.res +4 -1
  7. package/src/Batch.res.mjs +3 -2
  8. package/src/ChainFetcher.res +63 -15
  9. package/src/ChainFetcher.res.mjs +65 -24
  10. package/src/ChainManager.res +12 -22
  11. package/src/ChainManager.res.mjs +17 -20
  12. package/src/Config.gen.ts +19 -0
  13. package/src/Config.res +405 -9
  14. package/src/Config.res.mjs +395 -16
  15. package/src/Ctx.res +1 -1
  16. package/src/Env.res +1 -1
  17. package/src/Env.res.mjs +4 -4
  18. package/src/Envio.gen.ts +0 -6
  19. package/src/Envio.res +13 -5
  20. package/src/EvmTypes.gen.ts +6 -0
  21. package/src/EvmTypes.res +1 -0
  22. package/src/FetchState.res +1226 -636
  23. package/src/FetchState.res.mjs +1111 -615
  24. package/src/GlobalState.res +22 -14
  25. package/src/GlobalState.res.mjs +62 -56
  26. package/src/HandlerLoader.res +2 -2
  27. package/src/HandlerLoader.res.mjs +3 -3
  28. package/src/HandlerRegister.res +319 -0
  29. package/src/HandlerRegister.res.mjs +299 -0
  30. package/src/{EventRegister.resi → HandlerRegister.resi} +9 -10
  31. package/src/Hasura.res +3 -3
  32. package/src/Hasura.res.mjs +3 -3
  33. package/src/InMemoryStore.res +1 -1
  34. package/src/InMemoryStore.res.mjs +3 -3
  35. package/src/Internal.gen.ts +4 -0
  36. package/src/Internal.res +3 -1
  37. package/src/Main.res +15 -2
  38. package/src/Main.res.mjs +18 -3
  39. package/src/Persistence.res +2 -2
  40. package/src/Persistence.res.mjs +3 -3
  41. package/src/PgStorage.res +29 -2
  42. package/src/PgStorage.res.mjs +32 -2
  43. package/src/Prometheus.res +22 -0
  44. package/src/Prometheus.res.mjs +55 -30
  45. package/src/TestIndexer.res +189 -14
  46. package/src/TestIndexer.res.mjs +238 -106
  47. package/src/UserContext.res +118 -61
  48. package/src/UserContext.res.mjs +70 -27
  49. package/src/Utils.res +56 -1
  50. package/src/Utils.res.mjs +65 -6
  51. package/src/bindings/EventSource.res +8 -1
  52. package/src/bindings/EventSource.res.mjs +8 -1
  53. package/src/bindings/Vitest.res +15 -7
  54. package/src/bindings/WebSocket.res +27 -0
  55. package/src/bindings/WebSocket.res.mjs +2 -0
  56. package/src/db/InternalTable.res +6 -67
  57. package/src/db/InternalTable.res.mjs +33 -82
  58. package/src/db/Table.res +21 -8
  59. package/src/db/Table.res.mjs +20 -10
  60. package/src/sources/EvmChain.res +16 -12
  61. package/src/sources/EvmChain.res.mjs +13 -10
  62. package/src/sources/HyperFuelSource.res +5 -1
  63. package/src/sources/HyperFuelSource.res.mjs +5 -1
  64. package/src/sources/HyperSync.res +12 -3
  65. package/src/sources/HyperSync.res.mjs +9 -7
  66. package/src/sources/HyperSync.resi +4 -0
  67. package/src/sources/HyperSyncClient.res +1 -1
  68. package/src/sources/HyperSyncHeightStream.res +20 -7
  69. package/src/sources/HyperSyncHeightStream.res.mjs +17 -11
  70. package/src/sources/HyperSyncSource.res +26 -16
  71. package/src/sources/HyperSyncSource.res.mjs +12 -15
  72. package/src/sources/Rpc.res +72 -48
  73. package/src/sources/Rpc.res.mjs +84 -34
  74. package/src/sources/RpcSource.res +342 -166
  75. package/src/sources/RpcSource.res.mjs +556 -270
  76. package/src/sources/RpcWebSocketHeightStream.res +177 -0
  77. package/src/sources/RpcWebSocketHeightStream.res.mjs +196 -0
  78. package/src/sources/SourceManager.res +2 -10
  79. package/src/sources/SourceManager.res.mjs +9 -13
  80. package/src/sources/Svm.res +15 -2
  81. package/src/sources/Svm.res.mjs +7 -1
  82. package/src/tui/Tui.res +17 -35
  83. package/src/tui/Tui.res.mjs +28 -32
  84. package/src/tui/components/CustomHooks.res +10 -2
  85. package/src/tui/components/CustomHooks.res.mjs +20 -3
  86. package/svm.schema.json +112 -0
  87. package/src/EventRegister.res +0 -241
  88. package/src/EventRegister.res.mjs +0 -240
  89. package/src/bindings/Ethers.res +0 -164
  90. package/src/bindings/Ethers.res.mjs +0 -78
  91. package/src/bindings/RescriptMocha.res +0 -123
  92. package/src/bindings/RescriptMocha.res.mjs +0 -18
package/README.md CHANGED
@@ -12,7 +12,7 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
12
12
  ## Key Features
13
13
 
14
14
  - **[Indexer auto-generation](https://docs.envio.dev/docs/HyperIndex/contract-import)** – Generate Indexers directly from smart contract addresses
15
- - **High performance** – Historical backfills at over 10,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
15
+ - **High performance** – Historical backfills at over 25,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
16
16
  - **Local development** – Full-featured local environment with Docker
17
17
  - **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-, SVM-, or Fuel-compatible blockchain
18
18
  - **Real-time indexing** – Instantly track blockchain events
package/evm.schema.json CHANGED
@@ -73,17 +73,6 @@
73
73
  "$ref": "#/$defs/Chain"
74
74
  }
75
75
  },
76
- "multichain": {
77
- "description": "Multichain mode: 'ordered' processes events across chains in order, 'unordered' processes chain events in order, but non-deterministically relatively to other chains (default: unordered)",
78
- "anyOf": [
79
- {
80
- "$ref": "#/$defs/Multichain"
81
- },
82
- {
83
- "type": "null"
84
- }
85
- ]
86
- },
87
76
  "rollback_on_reorg": {
88
77
  "description": "A flag to indicate if the indexer should rollback to the last known valid block on a reorg. This currently incurs a performance hit on historical sync and is recommended to turn this off while developing (default: true)",
89
78
  "type": [
@@ -307,17 +296,6 @@
307
296
  "format": "uint64",
308
297
  "minimum": 0
309
298
  },
310
- "rpc_config": {
311
- "description": "RPC configuration for utilizing as the chain's data-source. Typically optional for chains with HyperSync support, which is highly recommended. HyperSync dramatically enhances performance, providing up to a 1000x speed boost over traditional RPC.",
312
- "anyOf": [
313
- {
314
- "$ref": "#/$defs/RpcConfig"
315
- },
316
- {
317
- "type": "null"
318
- }
319
- ]
320
- },
321
299
  "rpc": {
322
300
  "description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
323
301
  "anyOf": [
@@ -346,7 +324,17 @@
346
324
  "integer",
347
325
  "null"
348
326
  ],
349
- "format": "int32"
327
+ "format": "uint32",
328
+ "minimum": 0
329
+ },
330
+ "block_lag": {
331
+ "description": "The number of blocks behind the chain head that the indexer should lag. Useful for avoiding reorg issues by indexing slightly behind the tip.",
332
+ "type": [
333
+ "integer",
334
+ "null"
335
+ ],
336
+ "format": "uint32",
337
+ "minimum": 0
350
338
  },
351
339
  "start_block": {
352
340
  "description": "The block at which the indexer should start ingesting data",
@@ -380,91 +368,6 @@
380
368
  "start_block"
381
369
  ]
382
370
  },
383
- "RpcConfig": {
384
- "type": "object",
385
- "properties": {
386
- "url": {
387
- "description": "URL of the RPC endpoint. Can be a single URL or an array of URLs. If multiple URLs are provided, the first one will be used as the primary RPC endpoint and the rest will be used as fallbacks.",
388
- "anyOf": [
389
- {
390
- "type": "string"
391
- },
392
- {
393
- "type": "array",
394
- "items": {
395
- "type": "string"
396
- }
397
- }
398
- ]
399
- },
400
- "initial_block_interval": {
401
- "description": "The starting interval in range of blocks per query",
402
- "type": [
403
- "integer",
404
- "null"
405
- ],
406
- "format": "uint32",
407
- "minimum": 0
408
- },
409
- "backoff_multiplicative": {
410
- "description": "After an RPC error, how much to scale back the number of blocks requested at once",
411
- "type": [
412
- "number",
413
- "null"
414
- ],
415
- "format": "double"
416
- },
417
- "acceleration_additive": {
418
- "description": "Without RPC errors or timeouts, how much to increase the number of blocks requested by for the next batch",
419
- "type": [
420
- "integer",
421
- "null"
422
- ],
423
- "format": "uint32",
424
- "minimum": 0
425
- },
426
- "interval_ceiling": {
427
- "description": "Do not further increase the block interval past this limit",
428
- "type": [
429
- "integer",
430
- "null"
431
- ],
432
- "format": "uint32",
433
- "minimum": 0
434
- },
435
- "backoff_millis": {
436
- "description": "After an error, how long to wait before retrying",
437
- "type": [
438
- "integer",
439
- "null"
440
- ],
441
- "format": "uint32",
442
- "minimum": 0
443
- },
444
- "fallback_stall_timeout": {
445
- "description": "If a fallback RPC is provided, the amount of time in ms to wait before kicking off the next provider",
446
- "type": [
447
- "integer",
448
- "null"
449
- ],
450
- "format": "uint32",
451
- "minimum": 0
452
- },
453
- "query_timeout_millis": {
454
- "description": "How long to wait before cancelling an RPC request",
455
- "type": [
456
- "integer",
457
- "null"
458
- ],
459
- "format": "uint32",
460
- "minimum": 0
461
- }
462
- },
463
- "additionalProperties": false,
464
- "required": [
465
- "url"
466
- ]
467
- },
468
371
  "RpcSelection": {
469
372
  "anyOf": [
470
373
  {
@@ -489,8 +392,22 @@
489
392
  "type": "string"
490
393
  },
491
394
  "for": {
492
- "description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback.",
493
- "$ref": "#/$defs/For"
395
+ "description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback. If not specified, defaults to \"fallback\" when HyperSync is available for the chain, or \"sync\" otherwise.",
396
+ "anyOf": [
397
+ {
398
+ "$ref": "#/$defs/For"
399
+ },
400
+ {
401
+ "type": "null"
402
+ }
403
+ ]
404
+ },
405
+ "ws": {
406
+ "description": "Optional WebSocket endpoint URL (wss:// or ws://) for real-time block header notifications via eth_subscribe(\"newHeads\"). Provides lower latency than HTTP polling for detecting new blocks.",
407
+ "type": [
408
+ "string",
409
+ "null"
410
+ ]
494
411
  },
495
412
  "initial_block_interval": {
496
413
  "description": "The starting interval in range of blocks per query",
@@ -553,12 +470,20 @@
553
470
  ],
554
471
  "format": "uint32",
555
472
  "minimum": 0
473
+ },
474
+ "polling_interval": {
475
+ "description": "How frequently (in milliseconds) to check for new blocks in realtime. Default is 1000ms. Note: Setting this higher than block time does not reduce RPC usage as every block is still fetched to check for reorgs.",
476
+ "type": [
477
+ "integer",
478
+ "null"
479
+ ],
480
+ "format": "uint32",
481
+ "minimum": 0
556
482
  }
557
483
  },
558
484
  "additionalProperties": false,
559
485
  "required": [
560
- "url",
561
- "for"
486
+ "url"
562
487
  ]
563
488
  },
564
489
  "For": {
@@ -671,13 +596,6 @@
671
596
  }
672
597
  ]
673
598
  },
674
- "Multichain": {
675
- "type": "string",
676
- "enum": [
677
- "ordered",
678
- "unordered"
679
- ]
680
- },
681
599
  "AddressFormat": {
682
600
  "type": "string",
683
601
  "enum": [
package/fuel.schema.json CHANGED
@@ -196,6 +196,24 @@
196
196
  }
197
197
  ]
198
198
  },
199
+ "max_reorg_depth": {
200
+ "description": "The number of blocks from the head that the indexer should account for in case of reorgs.",
201
+ "type": [
202
+ "integer",
203
+ "null"
204
+ ],
205
+ "format": "uint32",
206
+ "minimum": 0
207
+ },
208
+ "block_lag": {
209
+ "description": "The number of blocks behind the chain head that the indexer should lag. Useful for avoiding reorg issues by indexing slightly behind the tip.",
210
+ "type": [
211
+ "integer",
212
+ "null"
213
+ ],
214
+ "format": "uint32",
215
+ "minimum": 0
216
+ },
199
217
  "contracts": {
200
218
  "description": "All the contracts that should be indexed on the given chain",
201
219
  "type": [
package/index.d.ts CHANGED
@@ -18,6 +18,36 @@ export type { EffectCaller, Address } from "./src/Types.ts";
18
18
  /** Utility type to expand/flatten complex types for better IDE display. */
19
19
  export type Prettify<T> = { [K in keyof T]: T[K] } & {};
20
20
 
21
+ /**
22
+ * Operator for filtering entity fields in getWhere queries.
23
+ * Only fields with `@index` in the schema can be queried at runtime.
24
+ */
25
+ export type WhereOperator<T> = {
26
+ /** Matches entities where the field equals the given value. */
27
+ readonly _eq?: T;
28
+ /** Matches entities where the field is greater than the given value. */
29
+ readonly _gt?: T;
30
+ /** Matches entities where the field is less than the given value. */
31
+ readonly _lt?: T;
32
+ /** Matches entities where the field is greater than or equal to the given value. */
33
+ readonly _gte?: T;
34
+ /** Matches entities where the field is less than or equal to the given value. */
35
+ readonly _lte?: T;
36
+ /** Matches entities where the field equals any of the given values. */
37
+ readonly _in?: readonly T[];
38
+ };
39
+
40
+ /**
41
+ * Constructs a getWhere filter type from an entity type.
42
+ * Each field can be filtered using {@link WhereOperator} (`_eq`, `_gt`, `_lt`, `_gte`, `_lte`, `_in`).
43
+ *
44
+ * Note: only fields with `@index` in the schema can be queried at runtime.
45
+ * Attempting to filter on a non-indexed field will throw a descriptive error.
46
+ */
47
+ export type GetWhereFilter<E> = {
48
+ [K in keyof E]?: WhereOperator<E[K]>;
49
+ };
50
+
21
51
  import type {
22
52
  effect as Effect,
23
53
  effectArgs as EffectArgs,
@@ -234,6 +264,8 @@ type EvmChainConfig<Id extends number = number> = {
234
264
  readonly endBlock?: number;
235
265
  /** Number of blocks to keep for reorg handling (default: 200). */
236
266
  readonly maxReorgDepth?: number;
267
+ /** Number of blocks behind the chain head to lag (default: 0). */
268
+ readonly blockLag?: number;
237
269
  };
238
270
 
239
271
  /** EVM chain value (for runtime Indexer). */
@@ -287,6 +319,8 @@ type FuelChainConfig<Id extends number = number> = {
287
319
  readonly endBlock?: number;
288
320
  /** Number of blocks to keep for reorg handling (default: 200). */
289
321
  readonly maxReorgDepth?: number;
322
+ /** Number of blocks behind the chain head to lag (default: 0). */
323
+ readonly blockLag?: number;
290
324
  };
291
325
 
292
326
  /** Fuel chain value (for runtime Indexer). */
@@ -320,6 +354,8 @@ type SvmChainConfig<Id extends number = number> = {
320
354
  readonly endBlock?: number;
321
355
  /** Number of blocks to keep for reorg handling (default: 200). */
322
356
  readonly maxReorgDepth?: number;
357
+ /** Number of blocks behind the chain head to lag (default: 0). */
358
+ readonly blockLag?: number;
323
359
  };
324
360
 
325
361
  /** SVM chain value (for runtime Indexer). */
@@ -349,6 +385,7 @@ type IndexerConfigTypes = {
349
385
  contracts?: Record<string, {}>;
350
386
  };
351
387
  svm?: { chains: Record<string, { id: number }> };
388
+ entities?: Record<string, object>;
352
389
  };
353
390
 
354
391
  // Helper: Check if ecosystem is configured in a given config
@@ -497,12 +534,55 @@ export type TestIndexerChainConfig = {
497
534
  endBlock: number;
498
535
  };
499
536
 
500
- /** Progress returned after processing blocks with the test indexer. */
501
- export type TestIndexerProcessResult = {
502
- /** Changes happened during the processing. */
503
- changes: unknown[];
537
+ /** Entity change value containing sets and/or deleted IDs. */
538
+ type EntityChangeValue<Entity> = {
539
+ /** Entities that were created or updated. */
540
+ readonly sets?: readonly Entity[];
541
+ /** IDs of entities that were deleted. */
542
+ readonly deleted?: readonly string[];
504
543
  };
505
544
 
545
+ /** A dynamic contract address registration. */
546
+ type AddressRegistration = {
547
+ /** The contract address. */
548
+ readonly address: Address;
549
+ /** The contract name. */
550
+ readonly contract: string;
551
+ };
552
+
553
+ /** Extract entities from config. */
554
+ type ConfigEntities<Config extends IndexerConfigTypes> =
555
+ Config["entities"] extends Record<string, object> ? Config["entities"] : {};
556
+
557
+ /** Entity operations available on test indexer for direct entity manipulation. */
558
+ type EntityOps<Entity> = {
559
+ /** Get an entity by ID. Returns undefined if not found. */
560
+ readonly get: (id: string) => Promise<Entity | undefined>;
561
+ /** Set (create or update) an entity. */
562
+ readonly set: (entity: Entity) => void;
563
+ };
564
+
565
+ /** A single change representing entity modifications at a specific block. */
566
+ type EntityChange<Config extends IndexerConfigTypes> = {
567
+ /** The block where the changes occurred. */
568
+ readonly block: number;
569
+ /** The block hash (if available). */
570
+ readonly blockHash?: string;
571
+ /** The chain ID. */
572
+ readonly chainId: number;
573
+ /** Number of events processed in this block. */
574
+ readonly eventsProcessed: number;
575
+ /** Dynamic contract address registrations for this block. */
576
+ readonly addresses?: {
577
+ readonly sets?: readonly AddressRegistration[];
578
+ };
579
+ } & {
580
+ readonly [K in keyof ConfigEntities<Config>]?: EntityChangeValue<
581
+ ConfigEntities<Config>[K]
582
+ >;
583
+ };
584
+
585
+
506
586
  // Helper to extract chain IDs from config for test indexer
507
587
  type TestIndexerChainIds<Config extends IndexerConfigTypes> =
508
588
  HasEvm<Config> extends true
@@ -541,5 +621,15 @@ export type TestIndexerFromConfig<Config extends IndexerConfigTypes> = {
541
621
  /** Process blocks for the specified chains and return progress with checkpoints and changes. */
542
622
  process: (
543
623
  config: Prettify<TestIndexerProcessConfig<Config>>
544
- ) => Promise<TestIndexerProcessResult>;
624
+ ) => Promise<{
625
+ /** Changes happened during the processing. */
626
+ readonly changes: readonly EntityChange<Config>[];
627
+ }>;
628
+ } & (EcosystemCount<Config> extends 1
629
+ ? SingleEcosystemChains<Config>
630
+ : MultiEcosystemChains<Config>) & {
631
+ /** Entity operations for direct manipulation outside of handlers. */
632
+ readonly [K in keyof ConfigEntities<Config>]: EntityOps<
633
+ ConfigEntities<Config>[K]
634
+ >;
545
635
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "envio",
3
- "version": "v3.0.0-alpha.9",
3
+ "version": "3.0.0-main-alpha15-build-test",
4
4
  "type": "module",
5
5
  "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
6
6
  "bin": "./bin.js",
@@ -10,30 +10,35 @@
10
10
  "type": "git",
11
11
  "url": "git+https://github.com/enviodev/hyperindex.git"
12
12
  },
13
+ "author": "envio contributors <about@envio.dev>",
14
+ "license": "GPL-3.0",
15
+ "bugs": {
16
+ "url": "https://github.com/enviodev/hyperindex/issues"
17
+ },
18
+ "homepage": "https://envio.dev",
13
19
  "keywords": [
14
20
  "blockchain",
15
21
  "indexer",
16
22
  "ethereum",
17
23
  "evm",
18
24
  "fuel",
25
+ "solana",
19
26
  "data",
20
27
  "dapp"
21
28
  ],
22
- "author": "envio contributors <about@envio.dev>",
23
- "license": "GPL-3.0",
24
- "bugs": {
25
- "url": "https://github.com/enviodev/hyperindex/issues"
26
- },
27
- "homepage": "https://envio.dev",
28
29
  "engines": {
29
30
  "node": ">=22.0.0"
30
31
  },
31
- "optionalDependencies": {
32
- "envio-linux-x64": "v3.0.0-alpha.9",
33
- "envio-linux-arm64": "v3.0.0-alpha.9",
34
- "envio-darwin-x64": "v3.0.0-alpha.9",
35
- "envio-darwin-arm64": "v3.0.0-alpha.9"
36
- },
32
+ "files": [
33
+ "bin.js",
34
+ "evm.schema.json",
35
+ "fuel.schema.json",
36
+ "svm.schema.json",
37
+ "rescript.json",
38
+ "index.d.ts",
39
+ "index.js",
40
+ "src"
41
+ ],
37
42
  "dependencies": {
38
43
  "@clickhouse/client": "1.12.1",
39
44
  "@elastic/ecs-pino-format": "1.4.0",
@@ -59,13 +64,10 @@
59
64
  "postgres": "3.4.8",
60
65
  "tsx": "4.21.0"
61
66
  },
62
- "files": [
63
- "bin.js",
64
- "evm.schema.json",
65
- "fuel.schema.json",
66
- "rescript.json",
67
- "index.d.ts",
68
- "index.js",
69
- "src"
70
- ]
71
- }
67
+ "optionalDependencies": {
68
+ "envio-linux-x64": "3.0.0-main-alpha15-build-test",
69
+ "envio-linux-arm64": "3.0.0-main-alpha15-build-test",
70
+ "envio-darwin-x64": "3.0.0-main-alpha15-build-test",
71
+ "envio-darwin-arm64": "3.0.0-main-alpha15-build-test"
72
+ }
73
+ }
package/src/Batch.res CHANGED
@@ -6,6 +6,7 @@ open Utils.UnsafeIntOperators
6
6
  type chainAfterBatch = {
7
7
  batchSize: int,
8
8
  progressBlockNumber: int,
9
+ sourceBlockNumber: int,
9
10
  totalEventsProcessed: int,
10
11
  fetchState: FetchState.t,
11
12
  isProgressAtHeadWhenBatchCreated: bool,
@@ -17,6 +18,7 @@ type chainBeforeBatch = {
17
18
  progressBlockNumber: int,
18
19
  sourceBlockNumber: int,
19
20
  totalEventsProcessed: int,
21
+ chainConfig: Config.chain,
20
22
  }
21
23
 
22
24
  type t = {
@@ -108,10 +110,11 @@ let getProgressedChainsById = {
108
110
  {
109
111
  batchSize,
110
112
  progressBlockNumber: progressBlockNumberAfterBatch,
113
+ sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
111
114
  totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
112
115
  fetchState: fetchStateAfterBatch,
113
116
  isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >=
114
- chainBeforeBatch.sourceBlockNumber,
117
+ chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag,
115
118
  }: chainAfterBatch
116
119
  ),
117
120
  )
package/src/Batch.res.mjs CHANGED
@@ -62,9 +62,10 @@ function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAft
62
62
  return {
63
63
  batchSize: batchSize,
64
64
  progressBlockNumber: progressBlockNumberAfterBatch,
65
+ sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
65
66
  totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
66
67
  fetchState: fetchStateAfterBatch,
67
- isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber
68
+ isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag
68
69
  };
69
70
  }
70
71
 
@@ -80,7 +81,7 @@ function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressB
80
81
  var progressedChain;
81
82
  if (batchSize !== undefined) {
82
83
  var leftItems = fetchState.buffer.slice(batchSize);
83
- progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, undefined, undefined, leftItems, undefined, undefined), batchSize);
84
+ progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, undefined, leftItems, undefined, undefined), batchSize);
84
85
  } else {
85
86
  progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, 0);
86
87
  }
@@ -15,7 +15,6 @@ type t = {
15
15
  isProgressAtHead: bool,
16
16
  timestampCaughtUpToHeadOrEndblock: option<Js.Date.t>,
17
17
  committedProgressBlockNumber: int,
18
- firstEventBlockNumber: option<int>,
19
18
  numEventsProcessed: int,
20
19
  numBatchesFetched: int,
21
20
  reorgDetection: ReorgDetection.t,
@@ -28,10 +27,10 @@ let make = (
28
27
  ~dynamicContracts: array<Internal.indexingContract>,
29
28
  ~startBlock,
30
29
  ~endBlock,
31
- ~firstEventBlockNumber,
30
+ ~firstEventBlock=None,
32
31
  ~progressBlockNumber,
33
32
  ~config: Config.t,
34
- ~registrations: EventRegister.registrations,
33
+ ~registrations: HandlerRegister.registrations,
35
34
  ~targetBufferSize,
36
35
  ~logger,
37
36
  ~timestampCaughtUpToHeadOrEndblock,
@@ -149,7 +148,7 @@ let make = (
149
148
  registrations.onBlockByChainId->Utils.Dict.dangerouslyGetNonOption(chainConfig.id->Int.toString)
150
149
  switch onBlockConfigs {
151
150
  | Some(onBlockConfigs) =>
152
- // TODO: Move it to the EventRegister module
151
+ // TODO: Move it to the HandlerRegister module
153
152
  // so the error is thrown with better stack trace
154
153
  onBlockConfigs->Array.forEach(onBlockConfig => {
155
154
  if onBlockConfig.startBlock->Option.getWithDefault(startBlock) < startBlock {
@@ -183,9 +182,10 @@ let make = (
183
182
  // FIXME: Shouldn't set with full history
184
183
  ~blockLag=Pervasives.max(
185
184
  !config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth,
186
- Env.indexingBlockLag->Option.getWithDefault(0),
185
+ chainConfig.blockLag,
187
186
  ),
188
187
  ~onBlockConfigs?,
188
+ ~firstEventBlock,
189
189
  )
190
190
 
191
191
  let chainReorgCheckpoints = reorgCheckpoints->Array.keepMapU(reorgCheckpoint => {
@@ -196,11 +196,54 @@ let make = (
196
196
  }
197
197
  })
198
198
 
199
+ // Create sources lazily here - this is where API token validation happens
200
+ let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
201
+ let lowercaseAddresses = config.lowercaseAddresses
202
+ let sources = switch chainConfig.sourceConfig {
203
+ | Config.EvmSourceConfig({hypersync, rpcs}) =>
204
+ // Build Internal.evmContractConfig from contracts for EvmChain.makeSources
205
+ let evmContracts: array<Internal.evmContractConfig> = chainConfig.contracts->Array.map((
206
+ contract
207
+ ): Internal.evmContractConfig => {
208
+ name: contract.name,
209
+ abi: contract.abi,
210
+ events: contract.events->(
211
+ Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
212
+ ),
213
+ })
214
+ // Collect all event signatures from contracts
215
+ let allEventSignatures =
216
+ chainConfig.contracts->Array.flatMap(contract => contract.eventSignatures)
217
+ // Convert rpcs to EvmChain.rpc format
218
+ let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
219
+ let syncConfig = rpc.syncConfig
220
+ let ws = rpc.ws
221
+ {
222
+ url: rpc.url,
223
+ sourceFor: rpc.sourceFor,
224
+ ?syncConfig,
225
+ ?ws,
226
+ }
227
+ })
228
+ EvmChain.makeSources(
229
+ ~chain,
230
+ ~contracts=evmContracts,
231
+ ~hyperSync=hypersync,
232
+ ~allEventSignatures,
233
+ ~rpcs=evmRpcs,
234
+ ~lowercaseAddresses,
235
+ )
236
+ | Config.FuelSourceConfig({hypersync}) => [HyperFuelSource.make({chain, endpointUrl: hypersync})]
237
+ | Config.SvmSourceConfig({rpc}) => [Svm.makeRPCSource(~chain, ~rpc)]
238
+ // For tests: use ready-to-use sources directly
239
+ | Config.CustomSources(sources) => sources
240
+ }
241
+
199
242
  {
200
243
  logger,
201
244
  chainConfig,
202
245
  sourceManager: SourceManager.make(
203
- ~sources=chainConfig.sources,
246
+ ~sources,
204
247
  ~maxPartitionConcurrency=Env.maxPartitionConcurrency,
205
248
  ),
206
249
  reorgDetection: ReorgDetection.make(
@@ -215,7 +258,6 @@ let make = (
215
258
  ),
216
259
  isProgressAtHead: false,
217
260
  fetchState,
218
- firstEventBlockNumber,
219
261
  committedProgressBlockNumber: progressBlockNumber,
220
262
  timestampCaughtUpToHeadOrEndblock,
221
263
  numEventsProcessed,
@@ -223,7 +265,13 @@ let make = (
223
265
  }
224
266
  }
225
267
 
226
- let makeFromConfig = (chainConfig: Config.chain, ~config, ~registrations, ~targetBufferSize) => {
268
+ let makeFromConfig = (
269
+ chainConfig: Config.chain,
270
+ ~config,
271
+ ~registrations,
272
+ ~targetBufferSize,
273
+ ~knownHeight,
274
+ ) => {
227
275
  let logger = Logging.createChild(~params={"chainId": chainConfig.id})
228
276
 
229
277
  make(
@@ -234,7 +282,6 @@ let makeFromConfig = (chainConfig: Config.chain, ~config, ~registrations, ~targe
234
282
  ~endBlock=chainConfig.endBlock,
235
283
  ~reorgCheckpoints=[],
236
284
  ~maxReorgDepth=chainConfig.maxReorgDepth,
237
- ~firstEventBlockNumber=None,
238
285
  ~progressBlockNumber=-1,
239
286
  ~timestampCaughtUpToHeadOrEndblock=None,
240
287
  ~numEventsProcessed=0,
@@ -243,6 +290,7 @@ let makeFromConfig = (chainConfig: Config.chain, ~config, ~registrations, ~targe
243
290
  ~logger,
244
291
  ~dynamicContracts=[],
245
292
  ~isInReorgThreshold=false,
293
+ ~knownHeight,
246
294
  )
247
295
  }
248
296
 
@@ -278,7 +326,7 @@ let makeFromDbState = async (
278
326
  ~registrations,
279
327
  ~reorgCheckpoints,
280
328
  ~maxReorgDepth=resumedChainState.maxReorgDepth,
281
- ~firstEventBlockNumber=resumedChainState.firstEventBlockNumber,
329
+ ~firstEventBlock=resumedChainState.firstEventBlockNumber,
282
330
  ~progressBlockNumber,
283
331
  ~timestampCaughtUpToHeadOrEndblock=Env.updateSyncTimeOnRestart
284
332
  ? None
@@ -405,12 +453,12 @@ let handleQueryResult = (
405
453
  | _ => chainFetcher.fetchState->FetchState.registerDynamicContracts(newItemsWithDcs)
406
454
  }
407
455
 
408
- fs
409
- ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
410
- ->Result.map(fs => {
456
+ {
411
457
  ...chainFetcher,
412
- fetchState: fs->FetchState.updateKnownHeight(~knownHeight),
413
- })
458
+ fetchState: fs
459
+ ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
460
+ ->FetchState.updateKnownHeight(~knownHeight),
461
+ }
414
462
  }
415
463
 
416
464
  /**