envio 2.32.6 → 3.0.0-alpha-main-clickhouse-experiment

package/README.md

@@ -12,13 +12,13 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
 ## Key Features
 
 - **[Indexer auto-generation](https://docs.envio.dev/docs/HyperIndex/contract-import)** – Generate Indexers directly from smart contract addresses
-- **High performance** – Historical backfills at over 10,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
+- **High performance** – Historical backfills at over 25,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
 - **Local development** – Full-featured local environment with Docker
-- **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-compatible blockchain and Fuel (simultaneously)
+- **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-, SVM-, or Fuel-compatible blockchain
 - **Real-time indexing** – Instantly track blockchain events
 - **[Reorg support](https://docs.envio.dev/docs/HyperIndex/reorgs-support)** – Graceful handling of blockchain reorganizations
 - **GraphQL API** – Easy-to-query indexed data
-- **Flexible language support** – JavaScript, TypeScript, and ReScript
+- **Flexible language support** – TypeScript, JavaScript, and ReScript
 - **Factory contract support** – Index data from 1M+ dynamically registered contracts seamlessly
 - **On-chain & off-chain data integration** – Easily combine multiple data sources
 - **[Self-hosted & managed options](https://docs.envio.dev/docs/HyperIndex/hosted-service)** – Run your own setup or use HyperIndex hosted services

package/bin.mjs

@@ -0,0 +1,49 @@
+#!/usr/bin/env node
+//@ts-check
+
+import { spawnSync } from "child_process";
+import { createRequire } from "module";
+
+const require = createRequire(import.meta.url);
+
+/**
+ * Returns the executable path for envio located inside node_modules.
+ * The naming convention is envio-${os}-${arch}
+ * @see https://nodejs.org/api/os.html#osarch
+ * @see https://nodejs.org/api/os.html#osplatform
+ * @example "x/xx/node_modules/envio-darwin-arm64"
+ */
+function getExePath() {
+  const pkg = `envio-${process.platform}-${process.arch}`;
+  const bin = "bin/envio";
+
+  try {
+    return require.resolve(`${pkg}/${bin}`);
+  } catch {}
+
+  throw new Error(
+    `Couldn't find envio binary package "${pkg}".\n` +
+      `Checked: require.resolve("${pkg}/${bin}")\n` +
+      `If you're using pnpm, yarn, or npm with --omit=optional, ensure optional ` +
+      `dependencies are installed:\n` +
+      `  npm install ${pkg}\n`
+  );
+}
+
+/**
+ * Runs `envio` with args using nodejs spawn
+ */
+function runEnvio() {
+  const args = process.argv.slice(2);
+  const exePath = getExePath();
+
+  const processResult = spawnSync(exePath, args, { stdio: "inherit" });
+
+  if (processResult.error) {
+    console.error(`Failed to run envio binary at ${exePath}: ${processResult.error.message}`);
+    process.exit(1);
+  }
+  process.exit(processResult.status ?? 1);
+}
+
+runEnvio();

package/evm.schema.json

@@ -15,17 +15,6 @@
       "description": "Name of the project",
       "type": "string"
     },
-    "ecosystem": {
-      "description": "Ecosystem of the project.",
-      "anyOf": [
-        {
-          "$ref": "#/$defs/EcosystemTag"
-        },
-        {
-          "type": "null"
-        }
-      ]
-    },
     "schema": {
       "description": "Custom path to schema.graphql file",
       "type": [
@@ -40,41 +29,61 @@
         "null"
       ]
     },
-    "contracts": {
-      "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
+    "handlers": {
+      "description": "Optional relative path to handlers directory for auto-loading. Defaults to 'src/handlers' if not specified.",
       "type": [
-        "array",
+        "string",
         "null"
-      ],
-      "items": {
-        "$ref": "#/$defs/GlobalContract_for_ContractConfig"
-      }
-    },
-    "networks": {
-      "description": "Configuration of the blockchain networks that the project is deployed on.",
-      "type": "array",
-      "items": {
-        "$ref": "#/$defs/Network"
-      }
+      ]
     },
-    "unordered_multichain_mode": {
-      "description": "A flag to indicate if the indexer should use a single queue for all chains or a queue per chain (default: false)",
+    "full_batch_size": {
+      "description": "Target number of events to be processed per batch. Set it to smaller number if you have many Effect API calls which are slow to resolve and can't be batched. (Default: 5000)",
       "type": [
-        "boolean",
+        "integer",
         "null"
+      ],
+      "format": "uint64",
+      "minimum": 0
+    },
+    "storage": {
+      "description": "Storage backend for the indexer. Options: 'postgres' (default) or 'clickhouse'.",
+      "anyOf": [
+        {
+          "$ref": "#/$defs/Storage"
+        },
+        {
+          "type": "null"
+        }
       ]
     },
-    "event_decoder": {
-      "description": "The event decoder to use for the indexer (default: hypersync-client)",
+    "ecosystem": {
+      "description": "Ecosystem of the project.",
       "anyOf": [
         {
-          "$ref": "#/$defs/EventDecoder"
+          "$ref": "#/$defs/EcosystemTag"
         },
         {
           "type": "null"
         }
       ]
     },
+    "contracts": {
+      "description": "Global contract definitions that must contain all definitions except addresses. You can share a single handler/abi/event definitions for contracts across multiple chains.",
+      "type": [
+        "array",
+        "null"
+      ],
+      "items": {
+        "$ref": "#/$defs/GlobalContract_for_ContractConfig"
+      }
+    },
+    "chains": {
+      "description": "Configuration of the blockchain chains that the project is deployed on.",
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/Chain"
+      }
+    },
     "rollback_on_reorg": {
       "description": "A flag to indicate if the indexer should rollback to the last known valid block on a reorg. This currently incurs a performance hit on historical sync and is recommended to turn this off while developing (default: true)",
       "type": [
@@ -107,13 +116,6 @@
         "null"
       ]
     },
-    "preload_handlers": {
-      "description": "Makes handlers run twice to enable preload optimisations. Removes handlerWithLoader API, since it's not needed. (recommended, default: false)",
-      "type": [
-        "boolean",
-        "null"
-      ]
-    },
     "address_format": {
       "description": "Address format for Ethereum addresses: 'checksum' or 'lowercase' (default: checksum)",
       "anyOf": [
@@ -129,9 +131,24 @@
   "additionalProperties": false,
   "required": [
     "name",
-    "networks"
+    "chains"
   ],
   "$defs": {
+    "Storage": {
+      "description": "Storage backend option",
+      "oneOf": [
+        {
+          "description": "Use PostgreSQL as the storage backend (default)",
+          "type": "string",
+          "const": "postgres"
+        },
+        {
+          "description": "Use ClickHouse as the storage backend",
+          "type": "string",
+          "const": "clickhouse"
+        }
+      ]
+    },
     "EcosystemTag": {
       "type": "string",
       "enum": [
@@ -153,8 +170,11 @@
           ]
         },
         "handler": {
-          "description": "The relative path to a file where handlers are registered for the given contract",
-          "type": "string"
+          "description": "Optional relative path to a file where handlers are registered for the given contract. If not provided, handlers can be auto-loaded from src directory.",
+          "type": [
+            "string",
+            "null"
+          ]
         },
         "events": {
           "description": "A list of events that should be indexed on this contract",
@@ -167,7 +187,6 @@
       "additionalProperties": false,
       "required": [
         "name",
-        "handler",
         "events"
       ]
     },
@@ -253,11 +272,10 @@
         "root",
         "status",
         "yParity",
-        "chainId",
         "accessList",
         "maxFeePerBlobGas",
         "blobVersionedHashes",
-        "kind",
+        "type",
         "l1Fee",
         "l1GasPrice",
         "l1GasUsed",
@@ -295,31 +313,20 @@
         "mixHash"
       ]
     },
-    "Network": {
+    "Chain": {
       "type": "object",
       "properties": {
         "id": {
-          "description": "The public blockchain network ID.",
+          "description": "The public blockchain chain ID.",
           "type": "integer",
           "format": "uint64",
           "minimum": 0
         },
-        "rpc_config": {
-          "description": "RPC configuration for utilizing as the network's data-source. Typically optional for chains with HyperSync support, which is highly recommended. HyperSync dramatically enhances performance, providing up to a 1000x speed boost over traditional RPC.",
-          "anyOf": [
-            {
-              "$ref": "#/$defs/RpcConfig"
-            },
-            {
-              "type": "null"
-            }
-          ]
-        },
         "rpc": {
-          "description": "RPC configuration for your indexer. If not specified otherwise, for networks supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
+          "description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
           "anyOf": [
             {
-              "$ref": "#/$defs/NetworkRpc"
+              "$ref": "#/$defs/RpcSelection"
             },
             {
               "type": "null"
@@ -337,80 +344,17 @@
             }
           ]
         },
-        "confirmed_block_threshold": {
+        "max_reorg_depth": {
           "description": "The number of blocks from the head that the indexer should account for in case of reorgs.",
           "type": [
             "integer",
             "null"
           ],
-          "format": "int32"
-        },
-        "start_block": {
-          "description": "The block at which the indexer should start ingesting data",
-          "type": "integer",
-          "format": "uint64",
-          "minimum": 0
-        },
-        "end_block": {
-          "description": "The block at which the indexer should terminate.",
-          "type": [
-            "integer",
-            "null"
-          ],
-          "format": "uint64",
-          "minimum": 0
-        },
-        "contracts": {
-          "description": "All the contracts that should be indexed on the given network",
-          "type": "array",
-          "items": {
-            "$ref": "#/$defs/NetworkContract_for_ContractConfig"
-          }
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "start_block",
-        "contracts"
-      ]
-    },
-    "RpcConfig": {
-      "type": "object",
-      "properties": {
-        "url": {
-          "description": "URL of the RPC endpoint. Can be a single URL or an array of URLs. If multiple URLs are provided, the first one will be used as the primary RPC endpoint and the rest will be used as fallbacks.",
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            }
-          ]
-        },
-        "initial_block_interval": {
-          "description": "The starting interval in range of blocks per query",
-          "type": [
-            "integer",
-            "null"
-          ],
           "format": "uint32",
           "minimum": 0
         },
-        "backoff_multiplicative": {
-          "description": "After an RPC error, how much to scale back the number of blocks requested at once",
-          "type": [
-            "number",
-            "null"
-          ],
-          "format": "double"
-        },
-        "acceleration_additive": {
-          "description": "Without RPC errors or timeouts, how much to increase the number of blocks requested by for the next batch",
+        "block_lag": {
+          "description": "The number of blocks behind the chain head that the indexer should lag. Useful for avoiding reorg issues by indexing slightly behind the tip.",
           "type": [
             "integer",
             "null"
@@ -418,49 +362,39 @@
           "format": "uint32",
           "minimum": 0
         },
-        "interval_ceiling": {
-          "description": "Do not further increase the block interval past this limit",
-          "type": [
-            "integer",
-            "null"
-          ],
-          "format": "uint32",
-          "minimum": 0
-        },
-        "backoff_millis": {
-          "description": "After an error, how long to wait before retrying",
-          "type": [
-            "integer",
-            "null"
-          ],
-          "format": "uint32",
+        "start_block": {
+          "description": "The block at which the indexer should start ingesting data",
+          "type": "integer",
+          "format": "uint64",
           "minimum": 0
         },
-        "fallback_stall_timeout": {
-          "description": "If a fallback RPC is provided, the amount of time in ms to wait before kicking off the next provider",
+        "end_block": {
+          "description": "The block at which the indexer should terminate.",
           "type": [
             "integer",
             "null"
           ],
-          "format": "uint32",
+          "format": "uint64",
           "minimum": 0
         },
-        "query_timeout_millis": {
-          "description": "How long to wait before cancelling an RPC request",
+        "contracts": {
+          "description": "All the contracts that should be indexed on the given chain",
           "type": [
-            "integer",
+            "array",
             "null"
           ],
-          "format": "uint32",
-          "minimum": 0
+          "items": {
+            "$ref": "#/$defs/ChainContract_for_ContractConfig"
+          }
         }
       },
       "additionalProperties": false,
       "required": [
-        "url"
+        "id",
+        "start_block"
       ]
     },
-    "NetworkRpc": {
+    "RpcSelection": {
       "anyOf": [
         {
           "type": "string"
@@ -484,8 +418,22 @@
           "type": "string"
         },
         "for": {
-          "description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback.",
-          "$ref": "#/$defs/For"
+          "description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback. If not specified, defaults to \"fallback\" when HyperSync is available for the chain, or \"sync\" otherwise.",
+          "anyOf": [
+            {
+              "$ref": "#/$defs/For"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "ws": {
+          "description": "Optional WebSocket endpoint URL (wss:// or ws://) for real-time block header notifications via eth_subscribe(\"newHeads\"). Provides lower latency than HTTP polling for detecting new blocks.",
+          "type": [
+            "string",
+            "null"
+          ]
         },
         "initial_block_interval": {
           "description": "The starting interval in range of blocks per query",
@@ -548,12 +496,20 @@
           ],
           "format": "uint32",
           "minimum": 0
+        },
+        "polling_interval": {
+          "description": "How frequently (in milliseconds) to check for new blocks in realtime. Default is 1000ms. Note: Setting this higher than block time does not reduce RPC usage as every block is still fetched to check for reorgs.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint32",
+          "minimum": 0
         }
       },
       "additionalProperties": false,
       "required": [
-        "url",
-        "for"
+        "url"
       ]
     },
     "For": {
@@ -567,6 +523,11 @@
           "description": "Use RPC as a backup for the main data-source. Currently, it acts as a fallback when real-time indexing stalls, with potential for more cases in the future.",
           "type": "string",
           "const": "fallback"
+        },
+        {
+          "description": "Use RPC for real-time indexing only. HyperSync will be used for historical sync, then automatically switch to this RPC once synced for lower latency.",
+          "type": "string",
+          "const": "live"
         }
       ]
     },
@@ -583,7 +544,7 @@
         "url"
       ]
     },
-    "NetworkContract_for_ContractConfig": {
+    "ChainContract_for_ContractConfig": {
       "type": "object",
       "properties": {
         "name": {
@@ -595,7 +556,7 @@
           "$ref": "#/$defs/Addresses"
         },
         "start_block": {
-          "description": "The block at which the indexer should start ingesting data for this specific contract. If not specified, uses the network start_block. Can be greater than the network start_block for more specific indexing.",
+          "description": "The block at which the indexer should start ingesting data for this specific contract. If not specified, uses the chain start_block. Can be greater than the chain start_block for more specific indexing.",
           "type": [
             "integer",
             "null"
@@ -611,8 +572,11 @@
           ]
         },
         "handler": {
-          "description": "The relative path to a file where handlers are registered for the given contract",
-          "type": "string"
+          "description": "Optional relative path to a file where handlers are registered for the given contract. If not provided, handlers can be auto-loaded from src directory.",
+          "type": [
+            "string",
+            "null"
+          ]
         },
         "events": {
           "description": "A list of events that should be indexed on this contract",
@@ -658,13 +622,6 @@
         }
       ]
     },
-    "EventDecoder": {
-      "type": "string",
-      "enum": [
-        "viem",
-        "hypersync-client"
-      ]
-    },
     "AddressFormat": {
       "type": "string",
       "enum": [