envio 3.15.0-subgraph → 3.16.0-subgraph

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/README.md +40 -34
  2. package/evm.schema.json +3 -3
  3. package/fuel.schema.json +2 -2
  4. package/index.d.ts +3 -3
  5. package/licenses/README.md +1 -1
  6. package/package.json +8 -8
  7. package/src/Batch.res.mjs +5 -5
  8. package/src/BatchProcessing.res +1 -0
  9. package/src/BatchProcessing.res.mjs +1 -0
  10. package/src/ChainFetching.res +1 -6
  11. package/src/ChainFetching.res.mjs +1 -1
  12. package/src/ChainState.res +48 -7
  13. package/src/ChainState.res.mjs +21 -8
  14. package/src/Clock.res +57 -0
  15. package/src/Clock.res.mjs +72 -0
  16. package/src/Config.res +16 -4
  17. package/src/Config.res.mjs +5 -3
  18. package/src/Core.res +4 -40
  19. package/src/CrossChainState.res +10 -6
  20. package/src/CrossChainState.res.mjs +1 -1
  21. package/src/EffectState.res +10 -2
  22. package/src/EffectState.res.mjs +8 -4
  23. package/src/EffectState.resi +3 -1
  24. package/src/EntityId.res +2 -7
  25. package/src/EntityId.res.mjs +1 -8
  26. package/src/EntityTables.res +4 -1
  27. package/src/EntityTables.res.mjs +1 -1
  28. package/src/Env.res +4 -0
  29. package/src/Env.res.mjs +5 -0
  30. package/src/FetchState.res +157 -69
  31. package/src/FetchState.res.mjs +112 -47
  32. package/src/FinalizeBackfill.res +2 -4
  33. package/src/FinalizeBackfill.res.mjs +3 -4
  34. package/src/InMemoryTable.res +17 -16
  35. package/src/InMemoryTable.res.mjs +14 -15
  36. package/src/IndexerState.res +28 -4
  37. package/src/IndexerState.res.mjs +19 -6
  38. package/src/IndexerState.resi +4 -0
  39. package/src/Internal.res +0 -49
  40. package/src/Internal.res.mjs +1 -80
  41. package/src/LoadLayer.res +17 -10
  42. package/src/LoadLayer.res.mjs +11 -8
  43. package/src/Main.res.mjs +1 -1
  44. package/src/Persistence.res +3 -23
  45. package/src/Persistence.res.mjs +3 -3
  46. package/src/PreloadLookahead.res +42 -24
  47. package/src/PreloadLookahead.res.mjs +51 -30
  48. package/src/Rollback.res +0 -1
  49. package/src/Rollback.res.mjs +1 -1
  50. package/src/SafeCheckpointTracking.res +4 -4
  51. package/src/SafeCheckpointTracking.res.mjs +1 -1
  52. package/src/StagedRows.res +106 -0
  53. package/src/StagedRows.res.mjs +102 -0
  54. package/src/Staging.res +32 -14
  55. package/src/Staging.res.mjs +17 -20
  56. package/src/Storage.res +278 -973
  57. package/src/Storage.res.mjs +218 -720
  58. package/src/Supervisor.res +75 -56
  59. package/src/Supervisor.res.mjs +55 -26
  60. package/src/SyncWrites.res +24 -18
  61. package/src/SyncWrites.res.mjs +11 -11
  62. package/src/TestIndexer.res +22 -27
  63. package/src/TestIndexer.res.mjs +20 -17
  64. package/src/Throttler.res +15 -8
  65. package/src/Throttler.res.mjs +9 -6
  66. package/src/Throttler.resi +1 -1
  67. package/src/UserContext.res +13 -4
  68. package/src/UserContext.res.mjs +5 -5
  69. package/src/Utils.res +0 -6
  70. package/src/Utils.res.mjs +0 -5
  71. package/src/Writing.res +0 -1
  72. package/src/Writing.res.mjs +1 -1
  73. package/src/bindings/ClickHouse.res +0 -322
  74. package/src/bindings/ClickHouse.res.mjs +0 -338
  75. package/src/bindings/StorageClient.res +107 -174
  76. package/src/bindings/StorageClient.res.mjs +19 -8
  77. package/src/bindings/Vitest.res +0 -26
  78. package/src/bindings/Vitest.res.mjs +0 -3
  79. package/src/db/Table.res +24 -150
  80. package/src/db/Table.res.mjs +38 -272
  81. package/src/sources/ChainSources.res +2 -0
  82. package/src/sources/ChainSources.res.mjs +2 -2
  83. package/src/sources/EvmChain.res +3 -0
  84. package/src/sources/EvmChain.res.mjs +4 -2
  85. package/src/sources/EvmHyperSyncSource.res +2 -2
  86. package/src/sources/EvmHyperSyncSource.res.mjs +1 -1
  87. package/src/sources/EvmRpcClient.res +13 -3
  88. package/src/sources/EvmRpcClient.res.mjs +4 -3
  89. package/src/sources/EvmRpcWs.res +4 -3
  90. package/src/sources/EvmRpcWs.res.mjs +2 -2
  91. package/src/sources/HeightFeed.res +31 -10
  92. package/src/sources/HeightFeed.res.mjs +10 -6
  93. package/src/sources/HeightStream.res +32 -17
  94. package/src/sources/HeightStream.res.mjs +10 -10
  95. package/src/sources/HyperSyncSSE.res +4 -6
  96. package/src/sources/HyperSyncSSE.res.mjs +2 -2
  97. package/src/sources/RpcSource.res +10 -1
  98. package/src/sources/RpcSource.res.mjs +4 -3
  99. package/src/sources/Source.res +3 -0
  100. package/src/sources/SourceManager.res +85 -104
  101. package/src/sources/SourceManager.res.mjs +18 -37
  102. package/src/sources/SourceManager.resi +3 -5
  103. package/src/sources/StartBlockResolver.res +1 -0
  104. package/src/sources/StartBlockResolver.res.mjs +4 -3
  105. package/src/sources/SvmHyperSyncSource.res +2 -2
  106. package/src/sources/SvmHyperSyncSource.res.mjs +1 -1
  107. package/src/subgraph/graph-ts.ts +87 -28
  108. package/src/subgraph/runtime.ts +0 -1
  109. package/src/subgraph/scope.ts +0 -1
  110. package/svm.schema.json +3 -3
  111. package/src/PgWriting.res +0 -36
  112. package/src/PgWriting.res.mjs +0 -34
  113. package/src/db/IndexDefinition.res +0 -47
  114. package/src/db/IndexDefinition.res.mjs +0 -60
package/README.md CHANGED
@@ -1,6 +1,6 @@
1
1
  <div align="center">
2
- <h1>HyperIndex: Ultra-Fast Multichain Indexer</h1>
3
- <p><strong>The fastest independently benchmarked multichain blockchain indexer.</strong></p>
2
+ <h1>HyperIndex</h1>
3
+ <p><strong>Build custom blockchain APIs and analytics. Real-time, with enterprise SLAs.</strong></p>
4
4
  <p>
5
5
  <a href="https://github.com/enviodev/hyperindex/releases"><img src="https://img.shields.io/github/release/enviodev/hyperindex.svg" alt="GitHub release" /></a>
6
6
  <a href="https://github.com/enviodev/hyperindex/issues"><img src="https://img.shields.io/github/issues/enviodev/hyperindex.svg" alt="GitHub issues" /></a>
@@ -10,18 +10,20 @@
10
10
  </p>
11
11
  <p>
12
12
  <a href="https://docs.envio.dev">Documentation</a> ·
13
- <a href="https://envio.dev">Hosted Service</a> ·
13
+ <a href="https://envio.dev">Envio Cloud</a> ·
14
+ <a href="https://github.com/enviodev/open-indexer-benchmark">Benchmark</a> ·
15
+ <a href="https://docs.envio.dev/docs/HyperIndex/quickstart-with-ai">Build with AI</a> ·
14
16
  <a href="https://discord.gg/DhfFhzuJQh">Discord</a> ·
15
- <a href="https://docs.envio.dev/blog/best-blockchain-indexers-2026">Benchmarks</a>
17
+ <a href="https://envio.dev/pricing">Contact sales</a>
16
18
  </p>
17
19
  </div>
18
20
 
19
21
  ---
20
22
  ## What is HyperIndex?
21
23
 
22
- HyperIndex is Envio's full-featured blockchain indexing framework. It transforms onchain events into structured, queryable databases with GraphQL APIs, built for developers who care about performance and a clean local experience.
24
+ HyperIndex is Envio's blockchain indexing framework. Choose the contracts and events to track, write TypeScript handlers that shape the data, and HyperIndex keeps it up to date across EVM chains, Solana and Fuel, handles chain reorgs, and serves the result as a GraphQL API or direct Postgres/ClickHouse SQL.
23
25
 
24
- Powered by [HyperSync](https://docs.envio.dev/docs/HyperSync/overview), Envio's proprietary data engine, HyperIndex delivers up to 2000x faster data access than traditional RPC endpoints. That's the difference between syncing in minutes instead of days.
26
+ It reads chain data through [HyperSync](https://docs.envio.dev/docs/HyperSync/overview), Envio's blockchain data API, which is up to 2,000x faster than JSON-RPC. Run it yourself with Docker, or deploy to [Envio Cloud](https://docs.envio.dev/docs/HyperIndex/hosted-service). Cloud starts with a free development plan and scales with your project up to Dedicated infrastructure with a 99.99% uptime SLA.
25
27
 
26
28
  ![Envio Sync](sync.gif)
27
29
 
@@ -31,12 +33,9 @@ Powered by [HyperSync](https://docs.envio.dev/docs/HyperSync/overview), Envio's
31
33
 
32
34
  Most blockchain indexers are bottlenecked by RPC. HyperIndex isn't. HyperSync, the data engine underneath, is a purpose-built, Rust-based query layer that retrieves multiple blocks per round trip, cuts out the overhead, and makes historical backfills genuinely fast.
33
35
 
34
- Independent benchmarks run by Sentio confirm it:
36
+ The [open indexer benchmark](https://github.com/enviodev/open-indexer-benchmark) tracks current speed results across scenarios, and HyperIndex passes every reliability check. Every number there comes from code in the repository, so you can run it yourself.
35
37
 
36
- - **Uniswap V2 Factory** (May 2025): HyperIndex completed in 1 minute. 15x faster than the nearest competitor (Subsquid), 143x faster than The Graph, 158x faster than Ponder
37
- - **LBTC Token with RPC calls** (April 2025): HyperIndex completed in 3 minutes vs 3 hours 9 minutes for The Graph
38
-
39
- [View full benchmark results →](https://docs.envio.dev/docs/HyperIndex/benchmarks)
38
+ [How the benchmarks work →](https://docs.envio.dev/docs/HyperIndex/benchmarks)
40
39
 
41
40
  ---
42
41
 
@@ -44,15 +43,14 @@ Independent benchmarks run by Sentio confirm it:
44
43
 
45
44
  HyperIndex and HyperSync are the data layer for DeFi dashboards, protocol analytics, block explorers, stablecoin monitors, liquidation trackers, oracle comparisons, NFT explorers, and any application that needs fast, structured onchain data.
46
45
 
47
- A few things already running in production:
46
+ A few apps built with them, from the showcase:
48
47
 
49
- - [v4.xyz](https://v4.xyz): the hub for Uniswap V4 data and analytics, indexing across 10 chains in real-time
50
- - [Stable Volume](https://www.stablevolume.com/): real-time stablecoin transaction monitoring across 10+ chains
48
+ - [v4.xyz](https://v4.xyz): the hub for Uniswap V4 data and analytics, indexed in real time across chains
49
+ - [Stable Volume](https://www.stablevolume.com/): real-time stablecoin transaction monitoring across chains
51
50
  - [Stable Radar](https://stable-radar.com): real-time USDC transaction dashboard across multiple chains
52
51
  - [Liqo](https://liqo.xyz): multichain liquidation tracking for DeFi lending protocols
53
- - [Safe Stats](https://safe-stats.vercel.app/): real-time analytics for Safe multisig activity across all chains
54
52
  - [Oracle Wars](https://oraclewars.xyz/): real-time oracle price comparison across multiple oracles
55
- - [Chain Density](https://chaindensity.xyz/): transaction and event density analysis for any address across 70+ chains
53
+ - [Chain Density](https://chaindensity.xyz/): transaction and event density analysis for any address across chains
56
54
  - [LogTUI](https://www.npmjs.com/package/logtui): terminal UI for monitoring blockchain events in real-time
57
55
 
58
56
  [See the full showcase →](https://docs.envio.dev/showcase)
@@ -62,34 +60,33 @@ A few things already running in production:
62
60
  ## Key Features
63
61
 
64
62
  **Performance**
65
- - Historical backfills at 25,000+ events per second
66
- - Up to 2000x faster than traditional RPC via HyperSync (enabled by default, no config required)
67
- - Sync times reduced from days to minutes
63
+ - Historical backfills at 100,000+ events per second ([benchmarking](https://docs.envio.dev/docs/HyperIndex/benchmarking))
64
+ - Up to 2,000x faster than JSON-RPC via HyperSync (on by default, no config required)
68
65
  - Fallback RPC support for reliability without touching your indexer code
69
66
 
70
67
  **Multichain indexing**
71
68
  - Index EVM, SVM, and Fuel blockchains from a single indexer
72
- - 79+ EVM chains with first-class HyperSync support, plus any EVM chain via RPC
69
+ - Every chain on the [HyperSync network list](https://docs.envio.dev/docs/HyperSync/hypersync-supported-networks), plus any EVM chain via RPC
73
70
  - Real-time indexing with reorg handling built in
74
71
 
75
72
  **Developer experience**
76
73
  - Auto-generate an indexer directly from a smart contract address or ABI, no manual setup required
77
- - Write handlers in TypeScript, JavaScript, or ReScript
74
+ - Write handlers in TypeScript or ReScript, and plain JavaScript handler files load too
78
75
  - Full local development environment with Docker
79
76
  - GraphQL API generated automatically from your schema
80
77
  - Wildcard topic indexing: index by event signatures across any contract, not just specified addresses
81
- - Factory contract support for 1M+ dynamically registered contracts
78
+ - Factory contract support for dynamically registered contracts
82
79
  - Onchain and off-chain data integration
83
80
  - External API actions triggered by blockchain events
84
81
  - Detailed logging and error reporting
85
82
 
86
83
  **Deployment**
87
- - Managed [hosted service](https://docs.envio.dev/docs/HyperIndex/hosted-service) with static endpoints, built-in alerts, and production-ready infrastructure
84
+ - [Envio Cloud](https://docs.envio.dev/docs/HyperIndex/hosted-service), managed hosting with static endpoints and built-in alerts, from a free development plan up to Dedicated deployments with an SLA of up to 99.99% uptime
88
85
  - Self-hosted via Docker
89
86
  - No vendor lock-in. Switch between HyperSync and RPC at any time
90
87
 
91
88
  **Agentic development**
92
- - HyperIndex is the default indexing framework for AI-assisted and agentic workflows via Envio's hosted service CLI (`envio-cloud`) and Claude skills
89
+ - Build and deploy with an AI agent using the Envio Cloud CLI (`envio-cloud`) and the [HyperIndex skills](https://github.com/enviodev/hyperindex/tree/main/packages/cli/templates/static/shared/.claude/skills) that `envio init` adds to every project
93
90
  - An agent can scaffold, configure, and deploy a production-ready indexer without touching a config file. [400,000 events indexed in ~20 seconds](https://docs.envio.dev/blog/agentic-blockchain-indexing-envio-hyperindex)
94
91
  - Powerful built-in testing framework purpose-built for AI-driven development. Agents can write, run, and iterate on handler tests in a tight feedback loop, catching regressions before deploy
95
92
 
@@ -97,7 +94,7 @@ A few things already running in production:
97
94
 
98
95
  ## Getting started
99
96
 
100
- **Requirements**: Node.js, Docker (only needed for local development)
97
+ **Requirements**: Node.js 22.15 or newer, and Docker for local development. pnpm is recommended.
101
98
 
102
99
  ```bash
103
100
  pnpx envio init
@@ -119,7 +116,7 @@ From there, three files define your indexer:
119
116
 
120
117
  HyperSync is the data engine that makes HyperIndex fast. It's active by default for all supported networks, no configuration needed.
121
118
 
122
- Instead of making individual RPC calls per block, HyperSync retrieves multiple data points per round trip with advanced filtering. The result: sync speeds up to 2000x faster than standard RPC, dramatically lower infrastructure costs, and no rate limit headaches on supported networks.
119
+ Instead of making individual RPC calls per block, HyperSync retrieves multiple data points per round trip with advanced filtering. The result is sync speeds up to 2,000x faster than JSON-RPC, lower infrastructure costs, and no rate limits to work around on supported networks.
123
120
 
124
121
  HyperSync can also be used directly for custom data pipelines in Python, Rust, Node.js, and Go, independent of HyperIndex.
125
122
 
@@ -129,19 +126,17 @@ HyperSync can also be used directly for custom data pipelines in Python, Rust, N
129
126
 
130
127
  ## Supported networks
131
128
 
132
- HyperIndex supports any EVM-compatible L1, L2, or L3, plus Fuel and Solana (beta). 79+ chains have first-class HyperSync support for maximum speed. For any EVM chain not on the HyperSync list, RPC-based indexing works out of the box. Solana indexing runs on HyperSync for Solana, with instruction-level handlers and IDL-aware decoding.
129
+ HyperIndex supports any EVM-compatible L1, L2, or L3, plus Fuel and Solana. Chains on the [HyperSync network list](https://docs.envio.dev/docs/HyperSync/hypersync-supported-networks) get first-class HyperSync support. For any EVM chain not on the HyperSync list, RPC-based indexing works out of the box. Solana indexing runs on HyperSync for Solana, with instruction-level handlers and IDL-aware decoding.
133
130
 
134
131
  [Full network list →](https://docs.envio.dev/docs/HyperIndex/supported-networks)
135
132
 
136
133
  ---
137
134
 
138
- ## Migrating from The Graph, Ponder, or Subsquid
139
-
140
- HyperIndex has a dedicated migration guide that covers config conversion, schema mapping, and query differences in 3 steps. Envio also offers white-glove migration support. Reach out on Discord and the team will help you get set up.
135
+ ## Migrating an existing indexer
141
136
 
142
- Teams migrating from The Graph can access 2 months of free hosting and full migration support.
137
+ The migration guides cover config conversion, schema mapping and query differences, and you can [migrate with an AI agent](https://docs.envio.dev/docs/HyperIndex/migrate-with-ai). If you run subgraphs, Envio can also migrate and host them for you with Subgraph Hosting, while your applications keep querying a subgraph-compatible endpoint. For hands-on help, see [services](https://envio.dev/pricing/services) or ask on Discord.
143
138
 
144
- [Migration guide →](https://docs.envio.dev/docs/HyperIndex/migration-guide)
139
+ [Migration guides →](https://docs.envio.dev/docs/HyperIndex/migration-guide)
145
140
 
146
141
  ---
147
142
 
@@ -155,16 +150,27 @@ Key sections:
155
150
  - [Multichain Indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)
156
151
  - [Wildcard Indexing](https://docs.envio.dev/docs/HyperIndex/wildcard-indexing)
157
152
  - [Reorg Support](https://docs.envio.dev/docs/HyperIndex/reorgs-support)
158
- - [Hosted Service](https://docs.envio.dev/docs/HyperIndex/hosted-service)
153
+ - [Envio Cloud](https://docs.envio.dev/docs/HyperIndex/hosted-service)
159
154
  - [HyperSync as Data Source](https://docs.envio.dev/docs/HyperIndex/hypersync)
160
155
  - [Migration Guide](https://docs.envio.dev/docs/HyperIndex/migration-guide)
161
156
 
162
157
  ---
163
158
 
159
+ ## Build with an AI agent
160
+
161
+ HyperIndex works with Claude Code, Cursor, Codex and other coding agents.
162
+
163
+ - **Docs for agents.** Point your agent at [llms.txt](https://docs.envio.dev/llms.txt), or the full text at [llms-full.txt](https://docs.envio.dev/llms-full.txt).
164
+ - **Docs MCP server.** Add `https://docs.envio.dev/mcp` to your agent for `docs_search` and `docs_fetch`. See the [MCP server guide](https://docs.envio.dev/docs/HyperIndex/mcp-server).
165
+ - **Skills.** `pnpx envio init` adds the [HyperIndex skills](https://github.com/enviodev/hyperindex/tree/main/packages/cli/templates/static/shared/.claude/skills) to every new project. After upgrading `envio`, run `pnpm envio skills update` to refresh them.
166
+ - **Quickstart.** Follow [Quickstart with AI](https://docs.envio.dev/docs/HyperIndex/quickstart-with-ai) to go from a contract address to a deployed GraphQL API.
167
+
168
+ ---
169
+
164
170
  ## Community and support
165
171
 
166
172
  - Follow us on [X](https://twitter.com/envio_indexer)
167
- - Join the [Discord](https://discord.gg/DhfFhzuJQh), fastest way to get help
173
+ - Join the [Discord](https://discord.gg/DhfFhzuJQh) or [Telegram](https://t.me/+BeS5ihVUFONjNGFk), the fastest ways to get help
168
174
  - Open an issue on [GitHub](https://github.com/enviodev/hyperindex/issues/new/choose)
169
175
  - Browse [common issues](https://docs.envio.dev/docs/common-issues) for quick troubleshooting
170
176
 
package/evm.schema.json CHANGED
@@ -289,7 +289,7 @@
289
289
  "type": "string"
290
290
  },
291
291
  "name": {
292
- "description": "Name of the event in the HyperIndex generated code. When ommitted, the event field will be used. Should be unique per contract",
292
+ "description": "Name of the event in the generated types. When ommitted, the event field will be used. Should be unique per contract",
293
293
  "type": [
294
294
  "string",
295
295
  "null"
@@ -421,7 +421,7 @@
421
421
  ]
422
422
  },
423
423
  "rpc": {
424
- "description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
424
+ "description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to 2000x faster than traditional RPC.",
425
425
  "anyOf": [
426
426
  {
427
427
  "$ref": "#/$defs/RpcSelection"
@@ -461,7 +461,7 @@
461
461
  "minimum": 0
462
462
  },
463
463
  "block_throttle": {
464
- "description": "The minimum number of new blocks the chain head must advance by before the indexer fetches again once it has caught up. Useful on chains with very fast blocks, where fetching every block is wasteful. Unset, 0 and 1 all fetch on every new block.",
464
+ "description": "Once caught up, wait for this many new blocks before fetching them. Delays new events by up to as many blocks.",
465
465
  "type": [
466
466
  "integer",
467
467
  "null"
package/fuel.schema.json CHANGED
@@ -241,7 +241,7 @@
241
241
  "type": "object",
242
242
  "properties": {
243
243
  "name": {
244
- "description": "Name of the event in the HyperIndex generated code",
244
+ "description": "Name of the event in the generated types",
245
245
  "type": "string"
246
246
  },
247
247
  "type": {
@@ -337,7 +337,7 @@
337
337
  "minimum": 0
338
338
  },
339
339
  "block_throttle": {
340
- "description": "The minimum number of new blocks the chain head must advance by before the indexer fetches again once it has caught up. Useful on chains with very fast blocks, where fetching every block is wasteful. Unset, 0 and 1 all fetch on every new block.",
340
+ "description": "Once caught up, wait for this many new blocks before fetching them. Delays new events by up to as many blocks.",
341
341
  "type": [
342
342
  "integer",
343
343
  "null"
package/index.d.ts CHANGED
@@ -398,7 +398,7 @@ type EvmChainConfig<Id extends number = number> = {
398
398
  readonly maxReorgDepth?: number;
399
399
  /** Number of blocks behind the chain head to lag (default: 0). */
400
400
  readonly blockLag?: number;
401
- /** Minimum number of new blocks before fetching again at the head (default: 0, every block). */
401
+ /** Once caught up, wait for this many new blocks before fetching them. Delays new events (default: 0). */
402
402
  readonly blockThrottle?: number;
403
403
  };
404
404
 
@@ -456,7 +456,7 @@ type FuelChainConfig<Id extends number = number> = {
456
456
  readonly maxReorgDepth?: number;
457
457
  /** Number of blocks behind the chain head to lag (default: 0). */
458
458
  readonly blockLag?: number;
459
- /** Minimum number of new blocks before fetching again at the head (default: 0, every block). */
459
+ /** Once caught up, wait for this many new blocks before fetching them. Delays new events (default: 0). */
460
460
  readonly blockThrottle?: number;
461
461
  };
462
462
 
@@ -494,7 +494,7 @@ type SvmChainConfig<Id extends number = number> = {
494
494
  readonly maxReorgDepth?: number;
495
495
  /** Number of blocks behind the chain head to lag (default: 0). */
496
496
  readonly blockLag?: number;
497
- /** Minimum number of new blocks before fetching again at the head (default: 0, every block). */
497
+ /** Once caught up, wait for this many new slots before fetching them. Delays new instructions (default: 0). */
498
498
  readonly blockThrottle?: number;
499
499
  };
500
500
 
@@ -2,7 +2,7 @@
2
2
 
3
3
  ## Overview
4
4
 
5
- Envio's licensing reflects open source ethos but is not OSI recognized. Developers can use Envio's services without vendor lock-in, either by self-hosting or specifying an RPC URL. The generated code is open and public and so is this repository that generates the generated code.
5
+ Envio's licensing reflects open source ethos but is not OSI recognized. Developers can use Envio's services without vendor lock-in, either by self-hosting or specifying an RPC URL. The code in this repository is public.
6
6
 
7
7
  Our licenses allows self-hosting but restricts third-party hosting with Envio's hosted service. Envio may consider open-sourcing in the future but prioritizes stakeholder interests and market traction. Please reach out to us on discord if you would like to explore hosting indexers as a third-party.
8
8
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "envio",
3
- "version": "3.15.0-subgraph",
3
+ "version": "3.16.0-subgraph",
4
4
  "type": "module",
5
5
  "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
6
6
  "bin": "./bin.mjs",
@@ -43,17 +43,17 @@
43
43
  "dependencies": {
44
44
  "bignumber.js": "9.3.1",
45
45
  "eventsource": "4.1.0",
46
- "express": "4.19.2",
47
- "@rescript/runtime": "12.2.0",
46
+ "express": "4.22.3",
47
+ "@rescript/runtime": "12.3.1",
48
48
  "rescript-schema": "9.5.1",
49
49
  "dotenv": "16.4.5",
50
50
  "yaml": "2.8.1"
51
51
  },
52
52
  "optionalDependencies": {
53
- "envio-linux-x64": "3.15.0-subgraph",
54
- "envio-linux-x64-musl": "3.15.0-subgraph",
55
- "envio-linux-arm64": "3.15.0-subgraph",
56
- "envio-darwin-x64": "3.15.0-subgraph",
57
- "envio-darwin-arm64": "3.15.0-subgraph"
53
+ "envio-linux-x64": "3.16.0-subgraph",
54
+ "envio-linux-x64-musl": "3.16.0-subgraph",
55
+ "envio-linux-arm64": "3.16.0-subgraph",
56
+ "envio-darwin-x64": "3.16.0-subgraph",
57
+ "envio-darwin-arm64": "3.16.0-subgraph"
58
58
  }
59
59
  }
package/src/Batch.res.mjs CHANGED
@@ -109,9 +109,9 @@ function make(sequence, frontier, chainsBeforeBatch, batchSizeTarget, history) {
109
109
  chainEventsProcessed = chainEventsProcessed + 1;
110
110
  }
111
111
  if (blockNumber !== prevBlockNumber) {
112
- let chainId = fetchState.chainId;
113
- let fromBlockExclusive = prevBlockNumber;
114
112
  let scannedHashes = chainBeforeBatch.scannedHashes;
113
+ let fromBlockExclusive = prevBlockNumber;
114
+ let chainId = fetchState.chainId;
115
115
  if (chainBeforeBatch.shouldRollbackOnReorg) {
116
116
  let blockNumbers = scannedHashes.blockNumbers;
117
117
  let length = blockNumbers.length;
@@ -154,10 +154,10 @@ function make(sequence, frontier, chainsBeforeBatch, batchSizeTarget, history) {
154
154
  };
155
155
  }
156
156
  let progressBlockNumberAfterBatch = FetchState.getProgressBlockNumberAt(fetchState, chainBatchSize);
157
- let chainId$1 = fetchState.chainId;
158
- let toBlockExclusive = progressBlockNumberAfterBatch + 1;
159
- let fromBlockExclusive$1 = prevBlockNumber;
160
157
  let scannedHashes$1 = chainBeforeBatch.scannedHashes;
158
+ let fromBlockExclusive$1 = prevBlockNumber;
159
+ let toBlockExclusive = progressBlockNumberAfterBatch + 1;
160
+ let chainId$1 = fetchState.chainId;
161
161
  if (chainBeforeBatch.shouldRollbackOnReorg) {
162
162
  let blockNumbers$1 = scannedHashes$1.blockNumbers;
163
163
  let length$1 = blockNumbers$1.length;
@@ -51,6 +51,7 @@ let rec startProcessing = async (
51
51
  }
52
52
  }
53
53
  state->IndexerState.endProcessing
54
+ state->IndexerState.reportArrivalAtHead
54
55
 
55
56
  // A reorg detected mid-batch parks the rollback until the loop is idle.
56
57
  // Hand off now that no batch is in flight.
@@ -108,6 +108,7 @@ async function startProcessing(state, runtime, scheduleFetch, scheduleRollback)
108
108
  }
109
109
  };
110
110
  IndexerState.endProcessing(state);
111
+ IndexerState.reportArrivalAtHead(state);
111
112
  if (IndexerState.isResolvingReorg(state)) {
112
113
  return scheduleRollback();
113
114
  } else if (!IndexerState.isStopped(state) && !Object.values(IndexerState.chainStates(state)).every(ChainState.isFetchingAtHead)) {
@@ -767,12 +767,7 @@ let fetchChain = async (
767
767
  try {
768
768
  await chainState->ChainState.dispatch(
769
769
  ~waitForNewBlock=(~knownHeight) =>
770
- sourceManager->SourceManager.waitForNewBlock(
771
- ~knownHeight,
772
- ~isRealtime,
773
- ~reducedPolling,
774
- ~blockThrottle=(chainState->ChainState.chainConfig).blockThrottle,
775
- ),
770
+ sourceManager->SourceManager.waitForNewBlock(~knownHeight, ~isRealtime, ~reducedPolling),
776
771
  ~onNewBlock=(~knownHeight) =>
777
772
  finishWaitingForNewBlock(
778
773
  state,
@@ -544,7 +544,7 @@ async function fetchChain(state, chainId, runtime, action, stateId, scheduleFetc
544
544
  let exn = Primitive_exceptions.internalToException(raw_exn);
545
545
  return IndexerState.errorExit(state, ErrorHandling.make(exn, IndexerState.logger(state), undefined));
546
546
  }
547
- }, knownHeight => SourceManager.waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling, ChainState.chainConfig(chainState).blockThrottle), knownHeight => finishWaitingForNewBlock(state, chainId, knownHeight, stateId, scheduleFetch, scheduleProcessing), action, stateId);
547
+ }, knownHeight => SourceManager.waitForNewBlock(sourceManager, knownHeight, isRealtime, reducedPolling), knownHeight => finishWaitingForNewBlock(state, chainId, knownHeight, stateId, scheduleFetch, scheduleProcessing), action, stateId);
548
548
  } catch (raw_exn) {
549
549
  let exn = Primitive_exceptions.internalToException(raw_exn);
550
550
  return IndexerState.errorExit(state, ErrorHandling.make(exn, IndexerState.logger(state), IndexerState.unexpectedErrorMsg));
@@ -320,11 +320,16 @@ let makeInternal = (
320
320
  // FIXME: Shouldn't set with full history
321
321
  // The resumed depth, not the config one: the pre-threshold lag must match
322
322
  // the window reorg detection compares, or a reduced config depth would let
323
- // fetching enter the stored rollback window without history.
323
+ // fetching enter the stored rollback window without history. One block
324
+ // more than the window, so the last pre-threshold page ends below it and
325
+ // the window's first block is read by a page that knows it is reorgable.
324
326
  ~blockLag=Pervasives.max(
325
- !config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : maxReorgDepth,
327
+ !config.shouldRollbackOnReorg || isInReorgThreshold || maxReorgDepth == 0
328
+ ? 0
329
+ : maxReorgDepth + 1,
326
330
  chainConfig.blockLag,
327
331
  ),
332
+ ~blockThrottle=chainConfig.blockThrottle,
328
333
  ~onBlockRegistrations,
329
334
  ~firstEventBlock,
330
335
  ~deferCreationBlockEvents=config.subgraph->Option.isSome,
@@ -362,6 +367,7 @@ let makeInternal = (
362
367
  ~blockStore,
363
368
  ~transactionStore,
364
369
  ~logger,
370
+ ~maxAddressesPerRequest=config.maxAddrInPartition,
365
371
  )
366
372
 
367
373
  // Seed the stored reorg checkpoints (hash-only rows) so detection resumes
@@ -852,10 +858,10 @@ let takeFinished = (cs: t) =>
852
858
  }
853
859
  }
854
860
 
855
- let getHighestBlockBelowThreshold = (cs: t): int => {
856
- let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.maxReorgDepth
857
- highestBlockBelowThreshold < 0 ? 0 : highestBlockBelowThreshold
858
- }
861
+ // The block reorg detection compares from is itself in the window, so the
862
+ // highest block below it is the one before, where catch-up stops.
863
+ let getHighestBlockBelowThreshold = (cs: t): int =>
864
+ Pervasives.max(cs.fetchState.knownHeight - cs.maxReorgDepth - 1, 0)
859
865
 
860
866
  // True once the fetch frontier has reached the head/endBlock for this chain.
861
867
  let isFetchingAtHead = (cs: t) => cs.fetchState->FetchState.isFetchingAtHead
@@ -1053,6 +1059,34 @@ let materializePageItems = async (
1053
1059
  ))
1054
1060
  }
1055
1061
 
1062
+ // Near the head: each new block in as few queries as possible, of a shape that
1063
+ // stays the same while contracts keep being registered. Once every partition is
1064
+ // close enough, the chain's dynamic contracts leave the server-side address
1065
+ // filter for the rest of the run.
1066
+ //
1067
+ // Judged only when the chain learns a newer head. An RPC response repeats the
1068
+ // height its query went out with, which can be hours old by the end of a
1069
+ // backfill: judged against it, every block produced since would be fetched
1070
+ // address-free.
1071
+ let switchDynamicContractsToClientSideFilteringNearHead = (cs: t) =>
1072
+ if (
1073
+ !cs.fetchState.clientFilterPolicy.allDynamicContracts &&
1074
+ cs.fetchState->FetchState.isWithinHeadClientFilteringDistance
1075
+ ) {
1076
+ cs.fetchState =
1077
+ cs.fetchState->FetchState.switchDynamicContractsToClientSideFiltering(
1078
+ ~addressStore=cs.addressStore,
1079
+ )
1080
+ if cs.onEventRegistrations->Array.some(registration => registration.contractRegister !== None) {
1081
+ cs.logger->Logger.trace(
1082
+ "Near the head: dynamic contracts are fetched without their addresses for the rest of the run, and filtered on our side.",
1083
+ ~params={
1084
+ "contractNames": cs.fetchState.optimizedPartitions.dynamicContracts->Utils.Set.toArray,
1085
+ },
1086
+ )
1087
+ }
1088
+ }
1089
+
1056
1090
  let handleQueryResult = (
1057
1091
  cs: t,
1058
1092
  ~query: FetchState.query,
@@ -1076,6 +1110,7 @@ let handleQueryResult = (
1076
1110
  )
1077
1111
  }
1078
1112
 
1113
+ let previousKnownHeight = cs.fetchState.knownHeight
1079
1114
  cs.fetchState =
1080
1115
  fs
1081
1116
  ->FetchState.handleQueryResult(~query, ~latestFetchedBlock, ~newItems)
@@ -1083,6 +1118,9 @@ let handleQueryResult = (
1083
1118
 
1084
1119
  // The query is no longer in flight, so release its reservation.
1085
1120
  cs.pendingBudget = Pervasives.max(0., cs.pendingBudget -. query.itemsEst->Int.toFloat)
1121
+ if cs.fetchState.knownHeight > previousKnownHeight {
1122
+ cs->switchDynamicContractsToClientSideFilteringNearHead
1123
+ }
1086
1124
  }
1087
1125
 
1088
1126
  // Run reorg detection against a fetch response by merging its block-store page
@@ -1117,8 +1155,11 @@ let prepareReorg = (cs: t, ~eventsProcessedDiff) => {
1117
1155
  cs->resetPendingQueries
1118
1156
  }
1119
1157
 
1120
- let updateKnownHeight = (cs: t, ~knownHeight) =>
1158
+ // A chain waiting at its head goes address-free before querying the new block.
1159
+ let updateKnownHeight = (cs: t, ~knownHeight) => {
1121
1160
  cs.fetchState = cs.fetchState->FetchState.updateKnownHeight(~knownHeight)
1161
+ cs->switchDynamicContractsToClientSideFilteringNearHead
1162
+ }
1122
1163
 
1123
1164
  // In auto-exit mode, pin the endBlock to the earliest observed event block.
1124
1165
  let setEndBlockToFirstEvent = (cs: t, ~blockNumber) =>
@@ -186,7 +186,7 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
186
186
  tmp = undefined;
187
187
  break;
188
188
  }
189
- let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, config.maxAddrInPartition, chainConfig.id, notice => logFetchStateNotice(logger$1, notice), (config.batchSize << 1), knownHeight, progressBlockNumber, onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : maxReorgDepth, chainConfig.blockLag), Primitive_option.some(firstEventBlock), Primitive_option.some(tmp), isResumed, Stdlib_Option.isSome(config.subgraph));
189
+ let fetchState = FetchState.make(startBlock, endBlock, onEventRegistrations, addressStore, addressRows, config.maxAddrInPartition, chainConfig.id, notice => logFetchStateNotice(logger$1, notice), (config.batchSize << 1), knownHeight, progressBlockNumber, onBlockRegistrations, Primitive_int.max(!config.shouldRollbackOnReorg || isInReorgThreshold || maxReorgDepth === 0 ? 0 : maxReorgDepth + 1 | 0, chainConfig.blockLag), chainConfig.blockThrottle, Primitive_option.some(firstEventBlock), Primitive_option.some(tmp), isResumed, Stdlib_Option.isSome(config.subgraph));
190
190
  let chainReorgCheckpoints = Stdlib_Array.filterMap(reorgCheckpoints, reorgCheckpoint => {
191
191
  if (reorgCheckpoint.chain_id === chainConfig.id) {
192
192
  return reorgCheckpoint;
@@ -194,7 +194,7 @@ function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints, isInR
194
194
  });
195
195
  let blockStore = BlockStore.make(config.ecosystem.name);
196
196
  let transactionStore = TransactionStore.make(config.ecosystem.name);
197
- let sources = ChainSources.make(chainConfig, onEventRegistrations, onBlockRegistrations, addressStore, lowercaseAddresses, blockStore, transactionStore, logger$1);
197
+ let sources = ChainSources.make(chainConfig, onEventRegistrations, onBlockRegistrations, addressStore, lowercaseAddresses, blockStore, transactionStore, logger$1, config.maxAddrInPartition);
198
198
  if (Utils.$$Array.notEmpty(chainReorgCheckpoints)) {
199
199
  let seedPage = BlockStore.fromJs(chainReorgCheckpoints.map(cp => ({
200
200
  blockNumber: cp.block_number,
@@ -568,12 +568,7 @@ function takeFinished(cs) {
568
568
  }
569
569
 
570
570
  function getHighestBlockBelowThreshold(cs) {
571
- let highestBlockBelowThreshold = cs.fetchState.knownHeight - cs.maxReorgDepth | 0;
572
- if (highestBlockBelowThreshold < 0) {
573
- return 0;
574
- } else {
575
- return highestBlockBelowThreshold;
576
- }
571
+ return Primitive_int.max((cs.fetchState.knownHeight - cs.maxReorgDepth | 0) - 1 | 0, 0);
577
572
  }
578
573
 
579
574
  function isFetchingAtHead(cs) {
@@ -721,10 +716,27 @@ async function materializePageItems(items, transactionStore, blockStore, shouldC
721
716
  ]);
722
717
  }
723
718
 
719
+ function switchDynamicContractsToClientSideFilteringNearHead(cs) {
720
+ if (!cs.fetchState.clientFilterPolicy.allDynamicContracts && FetchState.isWithinHeadClientFilteringDistance(cs.fetchState)) {
721
+ cs.fetchState = FetchState.switchDynamicContractsToClientSideFiltering(cs.fetchState, cs.addressStore);
722
+ if (cs.onEventRegistrations.some(registration => registration.contractRegister !== undefined)) {
723
+ return Logger.trace(cs.logger, "Near the head: dynamic contracts are fetched without their addresses for the rest of the run, and filtered on our side.", {
724
+ contractNames: Array.from(cs.fetchState.optimizedPartitions.dynamicContracts)
725
+ });
726
+ } else {
727
+ return;
728
+ }
729
+ }
730
+ }
731
+
724
732
  function handleQueryResult(cs, query, newItems, newRegistrations, latestFetchedBlock, knownHeight) {
725
733
  let fs = newRegistrations.length !== 0 ? FetchState.registerDynamicContracts(cs.fetchState, cs.addressStore, notice => logFetchStateNotice(cs.logger, notice), Primitive_int.max(knownHeight, latestFetchedBlock), newRegistrations) : cs.fetchState;
734
+ let previousKnownHeight = cs.fetchState.knownHeight;
726
735
  cs.fetchState = FetchState.updateKnownHeight(FetchState.handleQueryResult(fs, query, latestFetchedBlock, newItems), knownHeight);
727
736
  cs.pendingBudget = Primitive_float.max(0, cs.pendingBudget - query.itemsEst);
737
+ if (cs.fetchState.knownHeight > previousKnownHeight) {
738
+ return switchDynamicContractsToClientSideFilteringNearHead(cs);
739
+ }
728
740
  }
729
741
 
730
742
  function registerReorgGuard(cs, blockStore, knownHeight) {
@@ -757,6 +769,7 @@ function prepareReorg(cs, eventsProcessedDiff) {
757
769
 
758
770
  function updateKnownHeight(cs, knownHeight) {
759
771
  cs.fetchState = FetchState.updateKnownHeight(cs.fetchState, knownHeight);
772
+ switchDynamicContractsToClientSideFilteringNearHead(cs);
760
773
  }
761
774
 
762
775
  function setEndBlockToFirstEvent(cs, blockNumber) {
package/src/Clock.res ADDED
@@ -0,0 +1,57 @@
1
+ // What the source layer reads time from and arms its timers on. A test can hand
2
+ // it a clock that only moves when the test moves it, which takes the scheduler
3
+ // out of every race between a timer and the step a test is taking. The label
4
+ // names a timer in that clock's timeline; the real clock ignores it.
5
+ type timer
6
+
7
+ type t = {
8
+ now: unit => float,
9
+ setTimeout: (~label: string, unit => unit, int) => timer,
10
+ clearTimeout: timer => unit,
11
+ random: unit => float,
12
+ }
13
+
14
+ // Test-only. Set while a test drives a run on virtual time, so code that reaches
15
+ // past the clock it was given fails where it does, instead of arming a timer
16
+ // the test can neither see nor move.
17
+ let refuseReal = ref(false)
18
+
19
+ let refused = what =>
20
+ JsError.throwWithMessage(
21
+ `Clock.real ${what} on virtual time. Use the clock the code was given, so advanceTime can see and move it.`,
22
+ )
23
+
24
+ // Monotonic: every reading is only ever compared with another, and a wall
25
+ // clock corrected mid-run would make a connection look like it served for
26
+ // minutes, or a benched source look recovered.
27
+ let real: t = {
28
+ now: () => refuseReal.contents ? refused("read the time") : Performance.now(),
29
+ // A zero delay is a turn of the event loop on either clock.
30
+ setTimeout: (~label, callback, millis) =>
31
+ refuseReal.contents && millis > 0
32
+ ? refused(`armed "${label}"`)
33
+ : setTimeout(callback, millis)->(Utils.magic: timeoutId => timer),
34
+ clearTimeout: timer => clearTimeout(timer->(Utils.magic: timer => timeoutId)),
35
+ random: Math.random,
36
+ }
37
+
38
+ let delay = (clock: t, ~label, millis) =>
39
+ Promise.make((resolve, _reject) => clock.setTimeout(~label, () => resolve(), millis)->ignore)
40
+
41
+ // Half the delay fixed, half spread across it. Every indexer pointed at one
42
+ // provider loses its stream, or gives up on a quiet chain, in the same instant
43
+ // that provider blinks, and putting them all back on the same schedule is how a
44
+ // blip becomes a stampede.
45
+ let jitter = (clock: t, delay) =>
46
+ delay / 2 + (clock.random() *. (delay / 2)->Int.toFloat)->Float.toInt
47
+
48
+ // Clearing a pending timer and forgetting it are one action. A ref left holding
49
+ // a spent timer reads as one still pending, and the next clear looks like it
50
+ // did something.
51
+ let clearRef = (clock: t, timer: ref<option<timer>>) => {
52
+ switch timer.contents {
53
+ | Some(timer) => clock.clearTimeout(timer)
54
+ | None => ()
55
+ }
56
+ timer := None
57
+ }