envio 3.0.0-alpha.9 → 3.0.0-main-alpha15-build-test
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/evm.schema.json +37 -119
- package/fuel.schema.json +18 -0
- package/index.d.ts +95 -5
- package/package.json +25 -23
- package/src/Batch.res +4 -1
- package/src/Batch.res.mjs +3 -2
- package/src/ChainFetcher.res +63 -15
- package/src/ChainFetcher.res.mjs +65 -24
- package/src/ChainManager.res +12 -22
- package/src/ChainManager.res.mjs +17 -20
- package/src/Config.gen.ts +19 -0
- package/src/Config.res +405 -9
- package/src/Config.res.mjs +395 -16
- package/src/Ctx.res +1 -1
- package/src/Env.res +1 -1
- package/src/Env.res.mjs +4 -4
- package/src/Envio.gen.ts +0 -6
- package/src/Envio.res +13 -5
- package/src/EvmTypes.gen.ts +6 -0
- package/src/EvmTypes.res +1 -0
- package/src/FetchState.res +1226 -636
- package/src/FetchState.res.mjs +1111 -615
- package/src/GlobalState.res +22 -14
- package/src/GlobalState.res.mjs +62 -56
- package/src/HandlerLoader.res +2 -2
- package/src/HandlerLoader.res.mjs +3 -3
- package/src/HandlerRegister.res +319 -0
- package/src/HandlerRegister.res.mjs +299 -0
- package/src/{EventRegister.resi → HandlerRegister.resi} +9 -10
- package/src/Hasura.res +3 -3
- package/src/Hasura.res.mjs +3 -3
- package/src/InMemoryStore.res +1 -1
- package/src/InMemoryStore.res.mjs +3 -3
- package/src/Internal.gen.ts +4 -0
- package/src/Internal.res +3 -1
- package/src/Main.res +15 -2
- package/src/Main.res.mjs +18 -3
- package/src/Persistence.res +2 -2
- package/src/Persistence.res.mjs +3 -3
- package/src/PgStorage.res +29 -2
- package/src/PgStorage.res.mjs +32 -2
- package/src/Prometheus.res +22 -0
- package/src/Prometheus.res.mjs +55 -30
- package/src/TestIndexer.res +189 -14
- package/src/TestIndexer.res.mjs +238 -106
- package/src/UserContext.res +118 -61
- package/src/UserContext.res.mjs +70 -27
- package/src/Utils.res +56 -1
- package/src/Utils.res.mjs +65 -6
- package/src/bindings/EventSource.res +8 -1
- package/src/bindings/EventSource.res.mjs +8 -1
- package/src/bindings/Vitest.res +15 -7
- package/src/bindings/WebSocket.res +27 -0
- package/src/bindings/WebSocket.res.mjs +2 -0
- package/src/db/InternalTable.res +6 -67
- package/src/db/InternalTable.res.mjs +33 -82
- package/src/db/Table.res +21 -8
- package/src/db/Table.res.mjs +20 -10
- package/src/sources/EvmChain.res +16 -12
- package/src/sources/EvmChain.res.mjs +13 -10
- package/src/sources/HyperFuelSource.res +5 -1
- package/src/sources/HyperFuelSource.res.mjs +5 -1
- package/src/sources/HyperSync.res +12 -3
- package/src/sources/HyperSync.res.mjs +9 -7
- package/src/sources/HyperSync.resi +4 -0
- package/src/sources/HyperSyncClient.res +1 -1
- package/src/sources/HyperSyncHeightStream.res +20 -7
- package/src/sources/HyperSyncHeightStream.res.mjs +17 -11
- package/src/sources/HyperSyncSource.res +26 -16
- package/src/sources/HyperSyncSource.res.mjs +12 -15
- package/src/sources/Rpc.res +72 -48
- package/src/sources/Rpc.res.mjs +84 -34
- package/src/sources/RpcSource.res +342 -166
- package/src/sources/RpcSource.res.mjs +556 -270
- package/src/sources/RpcWebSocketHeightStream.res +177 -0
- package/src/sources/RpcWebSocketHeightStream.res.mjs +196 -0
- package/src/sources/SourceManager.res +2 -10
- package/src/sources/SourceManager.res.mjs +9 -13
- package/src/sources/Svm.res +15 -2
- package/src/sources/Svm.res.mjs +7 -1
- package/src/tui/Tui.res +17 -35
- package/src/tui/Tui.res.mjs +28 -32
- package/src/tui/components/CustomHooks.res +10 -2
- package/src/tui/components/CustomHooks.res.mjs +20 -3
- package/svm.schema.json +112 -0
- package/src/EventRegister.res +0 -241
- package/src/EventRegister.res.mjs +0 -240
- package/src/bindings/Ethers.res +0 -164
- package/src/bindings/Ethers.res.mjs +0 -78
- package/src/bindings/RescriptMocha.res +0 -123
- package/src/bindings/RescriptMocha.res.mjs +0 -18
package/README.md
CHANGED
|
@@ -12,7 +12,7 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
|
|
|
12
12
|
## Key Features
|
|
13
13
|
|
|
14
14
|
- **[Indexer auto-generation](https://docs.envio.dev/docs/HyperIndex/contract-import)** – Generate Indexers directly from smart contract addresses
|
|
15
|
-
- **High performance** – Historical backfills at over
|
|
15
|
+
- **High performance** – Historical backfills at over 25,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
|
|
16
16
|
- **Local development** – Full-featured local environment with Docker
|
|
17
17
|
- **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-, SVM-, or Fuel-compatible blockchain
|
|
18
18
|
- **Real-time indexing** – Instantly track blockchain events
|
package/evm.schema.json
CHANGED
|
@@ -73,17 +73,6 @@
|
|
|
73
73
|
"$ref": "#/$defs/Chain"
|
|
74
74
|
}
|
|
75
75
|
},
|
|
76
|
-
"multichain": {
|
|
77
|
-
"description": "Multichain mode: 'ordered' processes events across chains in order, 'unordered' processes chain events in order, but non-deterministically relatively to other chains (default: unordered)",
|
|
78
|
-
"anyOf": [
|
|
79
|
-
{
|
|
80
|
-
"$ref": "#/$defs/Multichain"
|
|
81
|
-
},
|
|
82
|
-
{
|
|
83
|
-
"type": "null"
|
|
84
|
-
}
|
|
85
|
-
]
|
|
86
|
-
},
|
|
87
76
|
"rollback_on_reorg": {
|
|
88
77
|
"description": "A flag to indicate if the indexer should rollback to the last known valid block on a reorg. This currently incurs a performance hit on historical sync and is recommended to turn this off while developing (default: true)",
|
|
89
78
|
"type": [
|
|
@@ -307,17 +296,6 @@
|
|
|
307
296
|
"format": "uint64",
|
|
308
297
|
"minimum": 0
|
|
309
298
|
},
|
|
310
|
-
"rpc_config": {
|
|
311
|
-
"description": "RPC configuration for utilizing as the chain's data-source. Typically optional for chains with HyperSync support, which is highly recommended. HyperSync dramatically enhances performance, providing up to a 1000x speed boost over traditional RPC.",
|
|
312
|
-
"anyOf": [
|
|
313
|
-
{
|
|
314
|
-
"$ref": "#/$defs/RpcConfig"
|
|
315
|
-
},
|
|
316
|
-
{
|
|
317
|
-
"type": "null"
|
|
318
|
-
}
|
|
319
|
-
]
|
|
320
|
-
},
|
|
321
299
|
"rpc": {
|
|
322
300
|
"description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
|
|
323
301
|
"anyOf": [
|
|
@@ -346,7 +324,17 @@
|
|
|
346
324
|
"integer",
|
|
347
325
|
"null"
|
|
348
326
|
],
|
|
349
|
-
"format": "
|
|
327
|
+
"format": "uint32",
|
|
328
|
+
"minimum": 0
|
|
329
|
+
},
|
|
330
|
+
"block_lag": {
|
|
331
|
+
"description": "The number of blocks behind the chain head that the indexer should lag. Useful for avoiding reorg issues by indexing slightly behind the tip.",
|
|
332
|
+
"type": [
|
|
333
|
+
"integer",
|
|
334
|
+
"null"
|
|
335
|
+
],
|
|
336
|
+
"format": "uint32",
|
|
337
|
+
"minimum": 0
|
|
350
338
|
},
|
|
351
339
|
"start_block": {
|
|
352
340
|
"description": "The block at which the indexer should start ingesting data",
|
|
@@ -380,91 +368,6 @@
|
|
|
380
368
|
"start_block"
|
|
381
369
|
]
|
|
382
370
|
},
|
|
383
|
-
"RpcConfig": {
|
|
384
|
-
"type": "object",
|
|
385
|
-
"properties": {
|
|
386
|
-
"url": {
|
|
387
|
-
"description": "URL of the RPC endpoint. Can be a single URL or an array of URLs. If multiple URLs are provided, the first one will be used as the primary RPC endpoint and the rest will be used as fallbacks.",
|
|
388
|
-
"anyOf": [
|
|
389
|
-
{
|
|
390
|
-
"type": "string"
|
|
391
|
-
},
|
|
392
|
-
{
|
|
393
|
-
"type": "array",
|
|
394
|
-
"items": {
|
|
395
|
-
"type": "string"
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
]
|
|
399
|
-
},
|
|
400
|
-
"initial_block_interval": {
|
|
401
|
-
"description": "The starting interval in range of blocks per query",
|
|
402
|
-
"type": [
|
|
403
|
-
"integer",
|
|
404
|
-
"null"
|
|
405
|
-
],
|
|
406
|
-
"format": "uint32",
|
|
407
|
-
"minimum": 0
|
|
408
|
-
},
|
|
409
|
-
"backoff_multiplicative": {
|
|
410
|
-
"description": "After an RPC error, how much to scale back the number of blocks requested at once",
|
|
411
|
-
"type": [
|
|
412
|
-
"number",
|
|
413
|
-
"null"
|
|
414
|
-
],
|
|
415
|
-
"format": "double"
|
|
416
|
-
},
|
|
417
|
-
"acceleration_additive": {
|
|
418
|
-
"description": "Without RPC errors or timeouts, how much to increase the number of blocks requested by for the next batch",
|
|
419
|
-
"type": [
|
|
420
|
-
"integer",
|
|
421
|
-
"null"
|
|
422
|
-
],
|
|
423
|
-
"format": "uint32",
|
|
424
|
-
"minimum": 0
|
|
425
|
-
},
|
|
426
|
-
"interval_ceiling": {
|
|
427
|
-
"description": "Do not further increase the block interval past this limit",
|
|
428
|
-
"type": [
|
|
429
|
-
"integer",
|
|
430
|
-
"null"
|
|
431
|
-
],
|
|
432
|
-
"format": "uint32",
|
|
433
|
-
"minimum": 0
|
|
434
|
-
},
|
|
435
|
-
"backoff_millis": {
|
|
436
|
-
"description": "After an error, how long to wait before retrying",
|
|
437
|
-
"type": [
|
|
438
|
-
"integer",
|
|
439
|
-
"null"
|
|
440
|
-
],
|
|
441
|
-
"format": "uint32",
|
|
442
|
-
"minimum": 0
|
|
443
|
-
},
|
|
444
|
-
"fallback_stall_timeout": {
|
|
445
|
-
"description": "If a fallback RPC is provided, the amount of time in ms to wait before kicking off the next provider",
|
|
446
|
-
"type": [
|
|
447
|
-
"integer",
|
|
448
|
-
"null"
|
|
449
|
-
],
|
|
450
|
-
"format": "uint32",
|
|
451
|
-
"minimum": 0
|
|
452
|
-
},
|
|
453
|
-
"query_timeout_millis": {
|
|
454
|
-
"description": "How long to wait before cancelling an RPC request",
|
|
455
|
-
"type": [
|
|
456
|
-
"integer",
|
|
457
|
-
"null"
|
|
458
|
-
],
|
|
459
|
-
"format": "uint32",
|
|
460
|
-
"minimum": 0
|
|
461
|
-
}
|
|
462
|
-
},
|
|
463
|
-
"additionalProperties": false,
|
|
464
|
-
"required": [
|
|
465
|
-
"url"
|
|
466
|
-
]
|
|
467
|
-
},
|
|
468
371
|
"RpcSelection": {
|
|
469
372
|
"anyOf": [
|
|
470
373
|
{
|
|
@@ -489,8 +392,22 @@
|
|
|
489
392
|
"type": "string"
|
|
490
393
|
},
|
|
491
394
|
"for": {
|
|
492
|
-
"description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback.",
|
|
493
|
-
"
|
|
395
|
+
"description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback. If not specified, defaults to \"fallback\" when HyperSync is available for the chain, or \"sync\" otherwise.",
|
|
396
|
+
"anyOf": [
|
|
397
|
+
{
|
|
398
|
+
"$ref": "#/$defs/For"
|
|
399
|
+
},
|
|
400
|
+
{
|
|
401
|
+
"type": "null"
|
|
402
|
+
}
|
|
403
|
+
]
|
|
404
|
+
},
|
|
405
|
+
"ws": {
|
|
406
|
+
"description": "Optional WebSocket endpoint URL (wss:// or ws://) for real-time block header notifications via eth_subscribe(\"newHeads\"). Provides lower latency than HTTP polling for detecting new blocks.",
|
|
407
|
+
"type": [
|
|
408
|
+
"string",
|
|
409
|
+
"null"
|
|
410
|
+
]
|
|
494
411
|
},
|
|
495
412
|
"initial_block_interval": {
|
|
496
413
|
"description": "The starting interval in range of blocks per query",
|
|
@@ -553,12 +470,20 @@
|
|
|
553
470
|
],
|
|
554
471
|
"format": "uint32",
|
|
555
472
|
"minimum": 0
|
|
473
|
+
},
|
|
474
|
+
"polling_interval": {
|
|
475
|
+
"description": "How frequently (in milliseconds) to check for new blocks in realtime. Default is 1000ms. Note: Setting this higher than block time does not reduce RPC usage as every block is still fetched to check for reorgs.",
|
|
476
|
+
"type": [
|
|
477
|
+
"integer",
|
|
478
|
+
"null"
|
|
479
|
+
],
|
|
480
|
+
"format": "uint32",
|
|
481
|
+
"minimum": 0
|
|
556
482
|
}
|
|
557
483
|
},
|
|
558
484
|
"additionalProperties": false,
|
|
559
485
|
"required": [
|
|
560
|
-
"url"
|
|
561
|
-
"for"
|
|
486
|
+
"url"
|
|
562
487
|
]
|
|
563
488
|
},
|
|
564
489
|
"For": {
|
|
@@ -671,13 +596,6 @@
|
|
|
671
596
|
}
|
|
672
597
|
]
|
|
673
598
|
},
|
|
674
|
-
"Multichain": {
|
|
675
|
-
"type": "string",
|
|
676
|
-
"enum": [
|
|
677
|
-
"ordered",
|
|
678
|
-
"unordered"
|
|
679
|
-
]
|
|
680
|
-
},
|
|
681
599
|
"AddressFormat": {
|
|
682
600
|
"type": "string",
|
|
683
601
|
"enum": [
|
package/fuel.schema.json
CHANGED
|
@@ -196,6 +196,24 @@
|
|
|
196
196
|
}
|
|
197
197
|
]
|
|
198
198
|
},
|
|
199
|
+
"max_reorg_depth": {
|
|
200
|
+
"description": "The number of blocks from the head that the indexer should account for in case of reorgs.",
|
|
201
|
+
"type": [
|
|
202
|
+
"integer",
|
|
203
|
+
"null"
|
|
204
|
+
],
|
|
205
|
+
"format": "uint32",
|
|
206
|
+
"minimum": 0
|
|
207
|
+
},
|
|
208
|
+
"block_lag": {
|
|
209
|
+
"description": "The number of blocks behind the chain head that the indexer should lag. Useful for avoiding reorg issues by indexing slightly behind the tip.",
|
|
210
|
+
"type": [
|
|
211
|
+
"integer",
|
|
212
|
+
"null"
|
|
213
|
+
],
|
|
214
|
+
"format": "uint32",
|
|
215
|
+
"minimum": 0
|
|
216
|
+
},
|
|
199
217
|
"contracts": {
|
|
200
218
|
"description": "All the contracts that should be indexed on the given chain",
|
|
201
219
|
"type": [
|
package/index.d.ts
CHANGED
|
@@ -18,6 +18,36 @@ export type { EffectCaller, Address } from "./src/Types.ts";
|
|
|
18
18
|
/** Utility type to expand/flatten complex types for better IDE display. */
|
|
19
19
|
export type Prettify<T> = { [K in keyof T]: T[K] } & {};
|
|
20
20
|
|
|
21
|
+
/**
|
|
22
|
+
* Operator for filtering entity fields in getWhere queries.
|
|
23
|
+
* Only fields with `@index` in the schema can be queried at runtime.
|
|
24
|
+
*/
|
|
25
|
+
export type WhereOperator<T> = {
|
|
26
|
+
/** Matches entities where the field equals the given value. */
|
|
27
|
+
readonly _eq?: T;
|
|
28
|
+
/** Matches entities where the field is greater than the given value. */
|
|
29
|
+
readonly _gt?: T;
|
|
30
|
+
/** Matches entities where the field is less than the given value. */
|
|
31
|
+
readonly _lt?: T;
|
|
32
|
+
/** Matches entities where the field is greater than or equal to the given value. */
|
|
33
|
+
readonly _gte?: T;
|
|
34
|
+
/** Matches entities where the field is less than or equal to the given value. */
|
|
35
|
+
readonly _lte?: T;
|
|
36
|
+
/** Matches entities where the field equals any of the given values. */
|
|
37
|
+
readonly _in?: readonly T[];
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Constructs a getWhere filter type from an entity type.
|
|
42
|
+
* Each field can be filtered using {@link WhereOperator} (`_eq`, `_gt`, `_lt`, `_gte`, `_lte`, `_in`).
|
|
43
|
+
*
|
|
44
|
+
* Note: only fields with `@index` in the schema can be queried at runtime.
|
|
45
|
+
* Attempting to filter on a non-indexed field will throw a descriptive error.
|
|
46
|
+
*/
|
|
47
|
+
export type GetWhereFilter<E> = {
|
|
48
|
+
[K in keyof E]?: WhereOperator<E[K]>;
|
|
49
|
+
};
|
|
50
|
+
|
|
21
51
|
import type {
|
|
22
52
|
effect as Effect,
|
|
23
53
|
effectArgs as EffectArgs,
|
|
@@ -234,6 +264,8 @@ type EvmChainConfig<Id extends number = number> = {
|
|
|
234
264
|
readonly endBlock?: number;
|
|
235
265
|
/** Number of blocks to keep for reorg handling (default: 200). */
|
|
236
266
|
readonly maxReorgDepth?: number;
|
|
267
|
+
/** Number of blocks behind the chain head to lag (default: 0). */
|
|
268
|
+
readonly blockLag?: number;
|
|
237
269
|
};
|
|
238
270
|
|
|
239
271
|
/** EVM chain value (for runtime Indexer). */
|
|
@@ -287,6 +319,8 @@ type FuelChainConfig<Id extends number = number> = {
|
|
|
287
319
|
readonly endBlock?: number;
|
|
288
320
|
/** Number of blocks to keep for reorg handling (default: 200). */
|
|
289
321
|
readonly maxReorgDepth?: number;
|
|
322
|
+
/** Number of blocks behind the chain head to lag (default: 0). */
|
|
323
|
+
readonly blockLag?: number;
|
|
290
324
|
};
|
|
291
325
|
|
|
292
326
|
/** Fuel chain value (for runtime Indexer). */
|
|
@@ -320,6 +354,8 @@ type SvmChainConfig<Id extends number = number> = {
|
|
|
320
354
|
readonly endBlock?: number;
|
|
321
355
|
/** Number of blocks to keep for reorg handling (default: 200). */
|
|
322
356
|
readonly maxReorgDepth?: number;
|
|
357
|
+
/** Number of blocks behind the chain head to lag (default: 0). */
|
|
358
|
+
readonly blockLag?: number;
|
|
323
359
|
};
|
|
324
360
|
|
|
325
361
|
/** SVM chain value (for runtime Indexer). */
|
|
@@ -349,6 +385,7 @@ type IndexerConfigTypes = {
|
|
|
349
385
|
contracts?: Record<string, {}>;
|
|
350
386
|
};
|
|
351
387
|
svm?: { chains: Record<string, { id: number }> };
|
|
388
|
+
entities?: Record<string, object>;
|
|
352
389
|
};
|
|
353
390
|
|
|
354
391
|
// Helper: Check if ecosystem is configured in a given config
|
|
@@ -497,12 +534,55 @@ export type TestIndexerChainConfig = {
|
|
|
497
534
|
endBlock: number;
|
|
498
535
|
};
|
|
499
536
|
|
|
500
|
-
/**
|
|
501
|
-
|
|
502
|
-
/**
|
|
503
|
-
|
|
537
|
+
/** Entity change value containing sets and/or deleted IDs. */
|
|
538
|
+
type EntityChangeValue<Entity> = {
|
|
539
|
+
/** Entities that were created or updated. */
|
|
540
|
+
readonly sets?: readonly Entity[];
|
|
541
|
+
/** IDs of entities that were deleted. */
|
|
542
|
+
readonly deleted?: readonly string[];
|
|
504
543
|
};
|
|
505
544
|
|
|
545
|
+
/** A dynamic contract address registration. */
|
|
546
|
+
type AddressRegistration = {
|
|
547
|
+
/** The contract address. */
|
|
548
|
+
readonly address: Address;
|
|
549
|
+
/** The contract name. */
|
|
550
|
+
readonly contract: string;
|
|
551
|
+
};
|
|
552
|
+
|
|
553
|
+
/** Extract entities from config. */
|
|
554
|
+
type ConfigEntities<Config extends IndexerConfigTypes> =
|
|
555
|
+
Config["entities"] extends Record<string, object> ? Config["entities"] : {};
|
|
556
|
+
|
|
557
|
+
/** Entity operations available on test indexer for direct entity manipulation. */
|
|
558
|
+
type EntityOps<Entity> = {
|
|
559
|
+
/** Get an entity by ID. Returns undefined if not found. */
|
|
560
|
+
readonly get: (id: string) => Promise<Entity | undefined>;
|
|
561
|
+
/** Set (create or update) an entity. */
|
|
562
|
+
readonly set: (entity: Entity) => void;
|
|
563
|
+
};
|
|
564
|
+
|
|
565
|
+
/** A single change representing entity modifications at a specific block. */
|
|
566
|
+
type EntityChange<Config extends IndexerConfigTypes> = {
|
|
567
|
+
/** The block where the changes occurred. */
|
|
568
|
+
readonly block: number;
|
|
569
|
+
/** The block hash (if available). */
|
|
570
|
+
readonly blockHash?: string;
|
|
571
|
+
/** The chain ID. */
|
|
572
|
+
readonly chainId: number;
|
|
573
|
+
/** Number of events processed in this block. */
|
|
574
|
+
readonly eventsProcessed: number;
|
|
575
|
+
/** Dynamic contract address registrations for this block. */
|
|
576
|
+
readonly addresses?: {
|
|
577
|
+
readonly sets?: readonly AddressRegistration[];
|
|
578
|
+
};
|
|
579
|
+
} & {
|
|
580
|
+
readonly [K in keyof ConfigEntities<Config>]?: EntityChangeValue<
|
|
581
|
+
ConfigEntities<Config>[K]
|
|
582
|
+
>;
|
|
583
|
+
};
|
|
584
|
+
|
|
585
|
+
|
|
506
586
|
// Helper to extract chain IDs from config for test indexer
|
|
507
587
|
type TestIndexerChainIds<Config extends IndexerConfigTypes> =
|
|
508
588
|
HasEvm<Config> extends true
|
|
@@ -541,5 +621,15 @@ export type TestIndexerFromConfig<Config extends IndexerConfigTypes> = {
|
|
|
541
621
|
/** Process blocks for the specified chains and return progress with checkpoints and changes. */
|
|
542
622
|
process: (
|
|
543
623
|
config: Prettify<TestIndexerProcessConfig<Config>>
|
|
544
|
-
) => Promise<
|
|
624
|
+
) => Promise<{
|
|
625
|
+
/** Changes happened during the processing. */
|
|
626
|
+
readonly changes: readonly EntityChange<Config>[];
|
|
627
|
+
}>;
|
|
628
|
+
} & (EcosystemCount<Config> extends 1
|
|
629
|
+
? SingleEcosystemChains<Config>
|
|
630
|
+
: MultiEcosystemChains<Config>) & {
|
|
631
|
+
/** Entity operations for direct manipulation outside of handlers. */
|
|
632
|
+
readonly [K in keyof ConfigEntities<Config>]: EntityOps<
|
|
633
|
+
ConfigEntities<Config>[K]
|
|
634
|
+
>;
|
|
545
635
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "envio",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "3.0.0-main-alpha15-build-test",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
|
|
6
6
|
"bin": "./bin.js",
|
|
@@ -10,30 +10,35 @@
|
|
|
10
10
|
"type": "git",
|
|
11
11
|
"url": "git+https://github.com/enviodev/hyperindex.git"
|
|
12
12
|
},
|
|
13
|
+
"author": "envio contributors <about@envio.dev>",
|
|
14
|
+
"license": "GPL-3.0",
|
|
15
|
+
"bugs": {
|
|
16
|
+
"url": "https://github.com/enviodev/hyperindex/issues"
|
|
17
|
+
},
|
|
18
|
+
"homepage": "https://envio.dev",
|
|
13
19
|
"keywords": [
|
|
14
20
|
"blockchain",
|
|
15
21
|
"indexer",
|
|
16
22
|
"ethereum",
|
|
17
23
|
"evm",
|
|
18
24
|
"fuel",
|
|
25
|
+
"solana",
|
|
19
26
|
"data",
|
|
20
27
|
"dapp"
|
|
21
28
|
],
|
|
22
|
-
"author": "envio contributors <about@envio.dev>",
|
|
23
|
-
"license": "GPL-3.0",
|
|
24
|
-
"bugs": {
|
|
25
|
-
"url": "https://github.com/enviodev/hyperindex/issues"
|
|
26
|
-
},
|
|
27
|
-
"homepage": "https://envio.dev",
|
|
28
29
|
"engines": {
|
|
29
30
|
"node": ">=22.0.0"
|
|
30
31
|
},
|
|
31
|
-
"
|
|
32
|
-
"
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
|
|
32
|
+
"files": [
|
|
33
|
+
"bin.js",
|
|
34
|
+
"evm.schema.json",
|
|
35
|
+
"fuel.schema.json",
|
|
36
|
+
"svm.schema.json",
|
|
37
|
+
"rescript.json",
|
|
38
|
+
"index.d.ts",
|
|
39
|
+
"index.js",
|
|
40
|
+
"src"
|
|
41
|
+
],
|
|
37
42
|
"dependencies": {
|
|
38
43
|
"@clickhouse/client": "1.12.1",
|
|
39
44
|
"@elastic/ecs-pino-format": "1.4.0",
|
|
@@ -59,13 +64,10 @@
|
|
|
59
64
|
"postgres": "3.4.8",
|
|
60
65
|
"tsx": "4.21.0"
|
|
61
66
|
},
|
|
62
|
-
"
|
|
63
|
-
"
|
|
64
|
-
"
|
|
65
|
-
"
|
|
66
|
-
"
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
"src"
|
|
70
|
-
]
|
|
71
|
-
}
|
|
67
|
+
"optionalDependencies": {
|
|
68
|
+
"envio-linux-x64": "3.0.0-main-alpha15-build-test",
|
|
69
|
+
"envio-linux-arm64": "3.0.0-main-alpha15-build-test",
|
|
70
|
+
"envio-darwin-x64": "3.0.0-main-alpha15-build-test",
|
|
71
|
+
"envio-darwin-arm64": "3.0.0-main-alpha15-build-test"
|
|
72
|
+
}
|
|
73
|
+
}
|
package/src/Batch.res
CHANGED
|
@@ -6,6 +6,7 @@ open Utils.UnsafeIntOperators
|
|
|
6
6
|
type chainAfterBatch = {
|
|
7
7
|
batchSize: int,
|
|
8
8
|
progressBlockNumber: int,
|
|
9
|
+
sourceBlockNumber: int,
|
|
9
10
|
totalEventsProcessed: int,
|
|
10
11
|
fetchState: FetchState.t,
|
|
11
12
|
isProgressAtHeadWhenBatchCreated: bool,
|
|
@@ -17,6 +18,7 @@ type chainBeforeBatch = {
|
|
|
17
18
|
progressBlockNumber: int,
|
|
18
19
|
sourceBlockNumber: int,
|
|
19
20
|
totalEventsProcessed: int,
|
|
21
|
+
chainConfig: Config.chain,
|
|
20
22
|
}
|
|
21
23
|
|
|
22
24
|
type t = {
|
|
@@ -108,10 +110,11 @@ let getProgressedChainsById = {
|
|
|
108
110
|
{
|
|
109
111
|
batchSize,
|
|
110
112
|
progressBlockNumber: progressBlockNumberAfterBatch,
|
|
113
|
+
sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
|
|
111
114
|
totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
|
|
112
115
|
fetchState: fetchStateAfterBatch,
|
|
113
116
|
isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >=
|
|
114
|
-
chainBeforeBatch.sourceBlockNumber,
|
|
117
|
+
chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag,
|
|
115
118
|
}: chainAfterBatch
|
|
116
119
|
),
|
|
117
120
|
)
|
package/src/Batch.res.mjs
CHANGED
|
@@ -62,9 +62,10 @@ function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAft
|
|
|
62
62
|
return {
|
|
63
63
|
batchSize: batchSize,
|
|
64
64
|
progressBlockNumber: progressBlockNumberAfterBatch,
|
|
65
|
+
sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
|
|
65
66
|
totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
|
|
66
67
|
fetchState: fetchStateAfterBatch,
|
|
67
|
-
isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber
|
|
68
|
+
isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag
|
|
68
69
|
};
|
|
69
70
|
}
|
|
70
71
|
|
|
@@ -80,7 +81,7 @@ function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressB
|
|
|
80
81
|
var progressedChain;
|
|
81
82
|
if (batchSize !== undefined) {
|
|
82
83
|
var leftItems = fetchState.buffer.slice(batchSize);
|
|
83
|
-
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, undefined,
|
|
84
|
+
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, undefined, leftItems, undefined, undefined), batchSize);
|
|
84
85
|
} else {
|
|
85
86
|
progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, 0);
|
|
86
87
|
}
|
package/src/ChainFetcher.res
CHANGED
|
@@ -15,7 +15,6 @@ type t = {
|
|
|
15
15
|
isProgressAtHead: bool,
|
|
16
16
|
timestampCaughtUpToHeadOrEndblock: option<Js.Date.t>,
|
|
17
17
|
committedProgressBlockNumber: int,
|
|
18
|
-
firstEventBlockNumber: option<int>,
|
|
19
18
|
numEventsProcessed: int,
|
|
20
19
|
numBatchesFetched: int,
|
|
21
20
|
reorgDetection: ReorgDetection.t,
|
|
@@ -28,10 +27,10 @@ let make = (
|
|
|
28
27
|
~dynamicContracts: array<Internal.indexingContract>,
|
|
29
28
|
~startBlock,
|
|
30
29
|
~endBlock,
|
|
31
|
-
~
|
|
30
|
+
~firstEventBlock=None,
|
|
32
31
|
~progressBlockNumber,
|
|
33
32
|
~config: Config.t,
|
|
34
|
-
~registrations:
|
|
33
|
+
~registrations: HandlerRegister.registrations,
|
|
35
34
|
~targetBufferSize,
|
|
36
35
|
~logger,
|
|
37
36
|
~timestampCaughtUpToHeadOrEndblock,
|
|
@@ -149,7 +148,7 @@ let make = (
|
|
|
149
148
|
registrations.onBlockByChainId->Utils.Dict.dangerouslyGetNonOption(chainConfig.id->Int.toString)
|
|
150
149
|
switch onBlockConfigs {
|
|
151
150
|
| Some(onBlockConfigs) =>
|
|
152
|
-
// TODO: Move it to the
|
|
151
|
+
// TODO: Move it to the HandlerRegister module
|
|
153
152
|
// so the error is thrown with better stack trace
|
|
154
153
|
onBlockConfigs->Array.forEach(onBlockConfig => {
|
|
155
154
|
if onBlockConfig.startBlock->Option.getWithDefault(startBlock) < startBlock {
|
|
@@ -183,9 +182,10 @@ let make = (
|
|
|
183
182
|
// FIXME: Shouldn't set with full history
|
|
184
183
|
~blockLag=Pervasives.max(
|
|
185
184
|
!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth,
|
|
186
|
-
|
|
185
|
+
chainConfig.blockLag,
|
|
187
186
|
),
|
|
188
187
|
~onBlockConfigs?,
|
|
188
|
+
~firstEventBlock,
|
|
189
189
|
)
|
|
190
190
|
|
|
191
191
|
let chainReorgCheckpoints = reorgCheckpoints->Array.keepMapU(reorgCheckpoint => {
|
|
@@ -196,11 +196,54 @@ let make = (
|
|
|
196
196
|
}
|
|
197
197
|
})
|
|
198
198
|
|
|
199
|
+
// Create sources lazily here - this is where API token validation happens
|
|
200
|
+
let chain = ChainMap.Chain.makeUnsafe(~chainId=chainConfig.id)
|
|
201
|
+
let lowercaseAddresses = config.lowercaseAddresses
|
|
202
|
+
let sources = switch chainConfig.sourceConfig {
|
|
203
|
+
| Config.EvmSourceConfig({hypersync, rpcs}) =>
|
|
204
|
+
// Build Internal.evmContractConfig from contracts for EvmChain.makeSources
|
|
205
|
+
let evmContracts: array<Internal.evmContractConfig> = chainConfig.contracts->Array.map((
|
|
206
|
+
contract
|
|
207
|
+
): Internal.evmContractConfig => {
|
|
208
|
+
name: contract.name,
|
|
209
|
+
abi: contract.abi,
|
|
210
|
+
events: contract.events->(
|
|
211
|
+
Utils.magic: array<Internal.eventConfig> => array<Internal.evmEventConfig>
|
|
212
|
+
),
|
|
213
|
+
})
|
|
214
|
+
// Collect all event signatures from contracts
|
|
215
|
+
let allEventSignatures =
|
|
216
|
+
chainConfig.contracts->Array.flatMap(contract => contract.eventSignatures)
|
|
217
|
+
// Convert rpcs to EvmChain.rpc format
|
|
218
|
+
let evmRpcs: array<EvmChain.rpc> = rpcs->Array.map((rpc): EvmChain.rpc => {
|
|
219
|
+
let syncConfig = rpc.syncConfig
|
|
220
|
+
let ws = rpc.ws
|
|
221
|
+
{
|
|
222
|
+
url: rpc.url,
|
|
223
|
+
sourceFor: rpc.sourceFor,
|
|
224
|
+
?syncConfig,
|
|
225
|
+
?ws,
|
|
226
|
+
}
|
|
227
|
+
})
|
|
228
|
+
EvmChain.makeSources(
|
|
229
|
+
~chain,
|
|
230
|
+
~contracts=evmContracts,
|
|
231
|
+
~hyperSync=hypersync,
|
|
232
|
+
~allEventSignatures,
|
|
233
|
+
~rpcs=evmRpcs,
|
|
234
|
+
~lowercaseAddresses,
|
|
235
|
+
)
|
|
236
|
+
| Config.FuelSourceConfig({hypersync}) => [HyperFuelSource.make({chain, endpointUrl: hypersync})]
|
|
237
|
+
| Config.SvmSourceConfig({rpc}) => [Svm.makeRPCSource(~chain, ~rpc)]
|
|
238
|
+
// For tests: use ready-to-use sources directly
|
|
239
|
+
| Config.CustomSources(sources) => sources
|
|
240
|
+
}
|
|
241
|
+
|
|
199
242
|
{
|
|
200
243
|
logger,
|
|
201
244
|
chainConfig,
|
|
202
245
|
sourceManager: SourceManager.make(
|
|
203
|
-
~sources
|
|
246
|
+
~sources,
|
|
204
247
|
~maxPartitionConcurrency=Env.maxPartitionConcurrency,
|
|
205
248
|
),
|
|
206
249
|
reorgDetection: ReorgDetection.make(
|
|
@@ -215,7 +258,6 @@ let make = (
|
|
|
215
258
|
),
|
|
216
259
|
isProgressAtHead: false,
|
|
217
260
|
fetchState,
|
|
218
|
-
firstEventBlockNumber,
|
|
219
261
|
committedProgressBlockNumber: progressBlockNumber,
|
|
220
262
|
timestampCaughtUpToHeadOrEndblock,
|
|
221
263
|
numEventsProcessed,
|
|
@@ -223,7 +265,13 @@ let make = (
|
|
|
223
265
|
}
|
|
224
266
|
}
|
|
225
267
|
|
|
226
|
-
let makeFromConfig = (
|
|
268
|
+
let makeFromConfig = (
|
|
269
|
+
chainConfig: Config.chain,
|
|
270
|
+
~config,
|
|
271
|
+
~registrations,
|
|
272
|
+
~targetBufferSize,
|
|
273
|
+
~knownHeight,
|
|
274
|
+
) => {
|
|
227
275
|
let logger = Logging.createChild(~params={"chainId": chainConfig.id})
|
|
228
276
|
|
|
229
277
|
make(
|
|
@@ -234,7 +282,6 @@ let makeFromConfig = (chainConfig: Config.chain, ~config, ~registrations, ~targe
|
|
|
234
282
|
~endBlock=chainConfig.endBlock,
|
|
235
283
|
~reorgCheckpoints=[],
|
|
236
284
|
~maxReorgDepth=chainConfig.maxReorgDepth,
|
|
237
|
-
~firstEventBlockNumber=None,
|
|
238
285
|
~progressBlockNumber=-1,
|
|
239
286
|
~timestampCaughtUpToHeadOrEndblock=None,
|
|
240
287
|
~numEventsProcessed=0,
|
|
@@ -243,6 +290,7 @@ let makeFromConfig = (chainConfig: Config.chain, ~config, ~registrations, ~targe
|
|
|
243
290
|
~logger,
|
|
244
291
|
~dynamicContracts=[],
|
|
245
292
|
~isInReorgThreshold=false,
|
|
293
|
+
~knownHeight,
|
|
246
294
|
)
|
|
247
295
|
}
|
|
248
296
|
|
|
@@ -278,7 +326,7 @@ let makeFromDbState = async (
|
|
|
278
326
|
~registrations,
|
|
279
327
|
~reorgCheckpoints,
|
|
280
328
|
~maxReorgDepth=resumedChainState.maxReorgDepth,
|
|
281
|
-
~
|
|
329
|
+
~firstEventBlock=resumedChainState.firstEventBlockNumber,
|
|
282
330
|
~progressBlockNumber,
|
|
283
331
|
~timestampCaughtUpToHeadOrEndblock=Env.updateSyncTimeOnRestart
|
|
284
332
|
? None
|
|
@@ -405,12 +453,12 @@ let handleQueryResult = (
|
|
|
405
453
|
| _ => chainFetcher.fetchState->FetchState.registerDynamicContracts(newItemsWithDcs)
|
|
406
454
|
}
|
|
407
455
|
|
|
408
|
-
|
|
409
|
-
->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
|
|
410
|
-
->Result.map(fs => {
|
|
456
|
+
{
|
|
411
457
|
...chainFetcher,
|
|
412
|
-
fetchState: fs
|
|
413
|
-
|
|
458
|
+
fetchState: fs
|
|
459
|
+
->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
|
|
460
|
+
->FetchState.updateKnownHeight(~knownHeight),
|
|
461
|
+
}
|
|
414
462
|
}
|
|
415
463
|
|
|
416
464
|
/**
|