envio 3.0.0-alpha.9 → 3.0.0-main-alpha16-build-test

package/README.md

@@ -12,7 +12,7 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
 ## Key Features
 
 - **[Indexer auto-generation](https://docs.envio.dev/docs/HyperIndex/contract-import)** – Generate Indexers directly from smart contract addresses
-- **High performance** – Historical backfills at over 10,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
+- **High performance** – Historical backfills at over 25,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
 - **Local development** – Full-featured local environment with Docker
 - **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-, SVM-, or Fuel-compatible blockchain
 - **Real-time indexing** – Instantly track blockchain events

package/{bin.js → bin.mjs}

@@ -26,14 +26,20 @@ function getExePath() {
     extension = ".exe";
   }
 
+  const pkg = `envio-${os}-${arch}`;
+  const bin = `bin/envio${extension}`;
+
   try {
-    // Since the bin will be located inside `node_modules`, we can simply call require.resolve
-    return require.resolve(`envio-${os}-${arch}/bin/envio${extension}`);
-  } catch (e) {
-    throw new Error(
-      `Couldn't find envio binary inside node_modules for ${os}-${arch}`
-    );
-  }
+    return require.resolve(`${pkg}/${bin}`);
+  } catch {}
+
+  throw new Error(
+    `Couldn't find envio binary package "${pkg}".\n` +
+      `Checked: require.resolve("${pkg}/${bin}")\n` +
+      `If you're using pnpm, yarn, or npm with --omit=optional, ensure optional ` +
+      `dependencies are installed:\n` +
+      `  npm install ${pkg}\n`
+  );
 }
 
 /**
@@ -41,8 +47,15 @@ function getExePath() {
  */
 function runEnvio() {
   const args = process.argv.slice(2);
-  const processResult = spawnSync(getExePath(), args, { stdio: "inherit" });
-  process.exit(processResult.status ?? 0);
+  const exePath = getExePath();
+
+  const processResult = spawnSync(exePath, args, { stdio: "inherit" });
+
+  if (processResult.error) {
+    console.error(`Failed to run envio binary at ${exePath}: ${processResult.error.message}`);
+    process.exit(1);
+  }
+  process.exit(processResult.status ?? 1);
 }
 
 runEnvio();

package/evm.schema.json

@@ -73,17 +73,6 @@
         "$ref": "#/$defs/Chain"
       }
     },
-    "multichain": {
-      "description": "Multichain mode: 'ordered' processes events across chains in order, 'unordered' processes chain events in order, but non-deterministically relatively to other chains (default: unordered)",
-      "anyOf": [
-        {
-          "$ref": "#/$defs/Multichain"
-        },
-        {
-          "type": "null"
-        }
-      ]
-    },
     "rollback_on_reorg": {
       "description": "A flag to indicate if the indexer should rollback to the last known valid block on a reorg. This currently incurs a performance hit on historical sync and is recommended to turn this off while developing (default: true)",
       "type": [
@@ -307,17 +296,6 @@
           "format": "uint64",
           "minimum": 0
         },
-        "rpc_config": {
-          "description": "RPC configuration for utilizing as the chain's data-source. Typically optional for chains with HyperSync support, which is highly recommended. HyperSync dramatically enhances performance, providing up to a 1000x speed boost over traditional RPC.",
-          "anyOf": [
-            {
-              "$ref": "#/$defs/RpcConfig"
-            },
-            {
-              "type": "null"
-            }
-          ]
-        },
         "rpc": {
           "description": "RPC configuration for your indexer. If not specified otherwise, for chains supported by HyperSync, RPC serves as a fallback for added reliability. For others, it acts as the primary data-source. HyperSync offers significant performance improvements, up to a 1000x faster than traditional RPC.",
           "anyOf": [
@@ -346,7 +324,17 @@
             "integer",
             "null"
           ],
-          "format": "int32"
+          "format": "uint32",
+          "minimum": 0
+        },
+        "block_lag": {
+          "description": "The number of blocks behind the chain head that the indexer should lag. Useful for avoiding reorg issues by indexing slightly behind the tip.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint32",
+          "minimum": 0
         },
         "start_block": {
           "description": "The block at which the indexer should start ingesting data",
@@ -380,91 +368,6 @@
         "start_block"
       ]
     },
-    "RpcConfig": {
-      "type": "object",
-      "properties": {
-        "url": {
-          "description": "URL of the RPC endpoint. Can be a single URL or an array of URLs. If multiple URLs are provided, the first one will be used as the primary RPC endpoint and the rest will be used as fallbacks.",
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            }
-          ]
-        },
-        "initial_block_interval": {
-          "description": "The starting interval in range of blocks per query",
-          "type": [
-            "integer",
-            "null"
-          ],
-          "format": "uint32",
-          "minimum": 0
-        },
-        "backoff_multiplicative": {
-          "description": "After an RPC error, how much to scale back the number of blocks requested at once",
-          "type": [
-            "number",
-            "null"
-          ],
-          "format": "double"
-        },
-        "acceleration_additive": {
-          "description": "Without RPC errors or timeouts, how much to increase the number of blocks requested by for the next batch",
-          "type": [
-            "integer",
-            "null"
-          ],
-          "format": "uint32",
-          "minimum": 0
-        },
-        "interval_ceiling": {
-          "description": "Do not further increase the block interval past this limit",
-          "type": [
-            "integer",
-            "null"
-          ],
-          "format": "uint32",
-          "minimum": 0
-        },
-        "backoff_millis": {
-          "description": "After an error, how long to wait before retrying",
-          "type": [
-            "integer",
-            "null"
-          ],
-          "format": "uint32",
-          "minimum": 0
-        },
-        "fallback_stall_timeout": {
-          "description": "If a fallback RPC is provided, the amount of time in ms to wait before kicking off the next provider",
-          "type": [
-            "integer",
-            "null"
-          ],
-          "format": "uint32",
-          "minimum": 0
-        },
-        "query_timeout_millis": {
-          "description": "How long to wait before cancelling an RPC request",
-          "type": [
-            "integer",
-            "null"
-          ],
-          "format": "uint32",
-          "minimum": 0
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "url"
-      ]
-    },
     "RpcSelection": {
       "anyOf": [
         {
@@ -489,8 +392,22 @@
           "type": "string"
         },
         "for": {
-          "description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback.",
-          "$ref": "#/$defs/For"
+          "description": "Determines if this RPC is for historical sync, real-time chain indexing, or as a fallback. If not specified, defaults to \"fallback\" when HyperSync is available for the chain, or \"sync\" otherwise.",
+          "anyOf": [
+            {
+              "$ref": "#/$defs/For"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "ws": {
+          "description": "Optional WebSocket endpoint URL (wss:// or ws://) for real-time block header notifications via eth_subscribe(\"newHeads\"). Provides lower latency than HTTP polling for detecting new blocks.",
+          "type": [
+            "string",
+            "null"
+          ]
         },
         "initial_block_interval": {
           "description": "The starting interval in range of blocks per query",
@@ -553,12 +470,20 @@
           ],
           "format": "uint32",
           "minimum": 0
+        },
+        "polling_interval": {
+          "description": "How frequently (in milliseconds) to check for new blocks in realtime. Default is 1000ms. Note: Setting this higher than block time does not reduce RPC usage as every block is still fetched to check for reorgs.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint32",
+          "minimum": 0
         }
       },
       "additionalProperties": false,
       "required": [
-        "url",
-        "for"
+        "url"
       ]
     },
     "For": {
@@ -671,13 +596,6 @@
         }
       ]
     },
-    "Multichain": {
-      "type": "string",
-      "enum": [
-        "ordered",
-        "unordered"
-      ]
-    },
     "AddressFormat": {
       "type": "string",
       "enum": [

package/fuel.schema.json

@@ -196,6 +196,24 @@
             }
           ]
         },
+        "max_reorg_depth": {
+          "description": "The number of blocks from the head that the indexer should account for in case of reorgs.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint32",
+          "minimum": 0
+        },
+        "block_lag": {
+          "description": "The number of blocks behind the chain head that the indexer should lag. Useful for avoiding reorg issues by indexing slightly behind the tip.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint32",
+          "minimum": 0
+        },
         "contracts": {
           "description": "All the contracts that should be indexed on the given chain",
           "type": [

package/index.d.ts

@@ -18,6 +18,36 @@ export type { EffectCaller, Address } from "./src/Types.ts";
 /** Utility type to expand/flatten complex types for better IDE display. */
 export type Prettify<T> = { [K in keyof T]: T[K] } & {};
 
+/**
+ * Operator for filtering entity fields in getWhere queries.
+ * Only fields with `@index` in the schema can be queried at runtime.
+ */
+export type WhereOperator<T> = {
+  /** Matches entities where the field equals the given value. */
+  readonly _eq?: T;
+  /** Matches entities where the field is greater than the given value. */
+  readonly _gt?: T;
+  /** Matches entities where the field is less than the given value. */
+  readonly _lt?: T;
+  /** Matches entities where the field is greater than or equal to the given value. */
+  readonly _gte?: T;
+  /** Matches entities where the field is less than or equal to the given value. */
+  readonly _lte?: T;
+  /** Matches entities where the field equals any of the given values. */
+  readonly _in?: readonly T[];
+};
+
+/**
+ * Constructs a getWhere filter type from an entity type.
+ * Each field can be filtered using {@link WhereOperator} (`_eq`, `_gt`, `_lt`, `_gte`, `_lte`, `_in`).
+ *
+ * Note: only fields with `@index` in the schema can be queried at runtime.
+ * Attempting to filter on a non-indexed field will throw a descriptive error.
+ */
+export type GetWhereFilter<E> = {
+  [K in keyof E]?: WhereOperator<E[K]>;
+};
+
 import type {
   effect as Effect,
   effectArgs as EffectArgs,
@@ -234,6 +264,8 @@ type EvmChainConfig<Id extends number = number> = {
   readonly endBlock?: number;
   /** Number of blocks to keep for reorg handling (default: 200). */
   readonly maxReorgDepth?: number;
+  /** Number of blocks behind the chain head to lag (default: 0). */
+  readonly blockLag?: number;
 };
 
 /** EVM chain value (for runtime Indexer). */
@@ -287,6 +319,8 @@ type FuelChainConfig<Id extends number = number> = {
   readonly endBlock?: number;
   /** Number of blocks to keep for reorg handling (default: 200). */
   readonly maxReorgDepth?: number;
+  /** Number of blocks behind the chain head to lag (default: 0). */
+  readonly blockLag?: number;
 };
 
 /** Fuel chain value (for runtime Indexer). */
@@ -320,6 +354,8 @@ type SvmChainConfig<Id extends number = number> = {
   readonly endBlock?: number;
   /** Number of blocks to keep for reorg handling (default: 200). */
   readonly maxReorgDepth?: number;
+  /** Number of blocks behind the chain head to lag (default: 0). */
+  readonly blockLag?: number;
 };
 
 /** SVM chain value (for runtime Indexer). */
@@ -349,6 +385,7 @@ type IndexerConfigTypes = {
     contracts?: Record<string, {}>;
   };
   svm?: { chains: Record<string, { id: number }> };
+  entities?: Record<string, object>;
 };
 
 // Helper: Check if ecosystem is configured in a given config
@@ -497,12 +534,55 @@ export type TestIndexerChainConfig = {
   endBlock: number;
 };
 
-/** Progress returned after processing blocks with the test indexer. */
-export type TestIndexerProcessResult = {
-  /** Changes happened during the processing. */
-  changes: unknown[];
+/** Entity change value containing sets and/or deleted IDs. */
+type EntityChangeValue<Entity> = {
+  /** Entities that were created or updated. */
+  readonly sets?: readonly Entity[];
+  /** IDs of entities that were deleted. */
+  readonly deleted?: readonly string[];
 };
 
+/** A dynamic contract address registration. */
+type AddressRegistration = {
+  /** The contract address. */
+  readonly address: Address;
+  /** The contract name. */
+  readonly contract: string;
+};
+
+/** Extract entities from config. */
+type ConfigEntities<Config extends IndexerConfigTypes> =
+  Config["entities"] extends Record<string, object> ? Config["entities"] : {};
+
+/** Entity operations available on test indexer for direct entity manipulation. */
+type EntityOps<Entity> = {
+  /** Get an entity by ID. Returns undefined if not found. */
+  readonly get: (id: string) => Promise<Entity | undefined>;
+  /** Set (create or update) an entity. */
+  readonly set: (entity: Entity) => void;
+};
+
+/** A single change representing entity modifications at a specific block. */
+type EntityChange<Config extends IndexerConfigTypes> = {
+  /** The block where the changes occurred. */
+  readonly block: number;
+  /** The block hash (if available). */
+  readonly blockHash?: string;
+  /** The chain ID. */
+  readonly chainId: number;
+  /** Number of events processed in this block. */
+  readonly eventsProcessed: number;
+  /** Dynamic contract address registrations for this block. */
+  readonly addresses?: {
+    readonly sets?: readonly AddressRegistration[];
+  };
+} & {
+  readonly [K in keyof ConfigEntities<Config>]?: EntityChangeValue<
+    ConfigEntities<Config>[K]
+  >;
+};
+
+
 // Helper to extract chain IDs from config for test indexer
 type TestIndexerChainIds<Config extends IndexerConfigTypes> =
   HasEvm<Config> extends true
@@ -541,5 +621,15 @@ export type TestIndexerFromConfig<Config extends IndexerConfigTypes> = {
   /** Process blocks for the specified chains and return progress with checkpoints and changes. */
   process: (
     config: Prettify<TestIndexerProcessConfig<Config>>
-  ) => Promise<TestIndexerProcessResult>;
+  ) => Promise<{
+    /** Changes happened during the processing. */
+    readonly changes: readonly EntityChange<Config>[];
+  }>;
+} & (EcosystemCount<Config> extends 1
+  ? SingleEcosystemChains<Config>
+  : MultiEcosystemChains<Config>) & {
+  /** Entity operations for direct manipulation outside of handlers. */
+  readonly [K in keyof ConfigEntities<Config>]: EntityOps<
+    ConfigEntities<Config>[K]
+  >;
 };

package/package.json

@@ -1,71 +1,73 @@
 {
   "name": "envio",
-  "version": "v3.0.0-alpha.9",
+  "version": "3.0.0-main-alpha16-build-test",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
-  "bin": "./bin.js",
+  "bin": "./bin.mjs",
   "main": "./index.js",
   "types": "./index.d.ts",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/enviodev/hyperindex.git"
   },
+  "author": "envio contributors <about@envio.dev>",
+  "license": "GPL-3.0",
+  "bugs": {
+    "url": "https://github.com/enviodev/hyperindex/issues"
+  },
+  "homepage": "https://envio.dev",
   "keywords": [
     "blockchain",
     "indexer",
     "ethereum",
     "evm",
     "fuel",
+    "solana",
     "data",
     "dapp"
   ],
-  "author": "envio contributors <about@envio.dev>",
-  "license": "GPL-3.0",
-  "bugs": {
-    "url": "https://github.com/enviodev/hyperindex/issues"
-  },
-  "homepage": "https://envio.dev",
   "engines": {
     "node": ">=22.0.0"
   },
-  "optionalDependencies": {
-    "envio-linux-x64": "v3.0.0-alpha.9",
-    "envio-linux-arm64": "v3.0.0-alpha.9",
-    "envio-darwin-x64": "v3.0.0-alpha.9",
-    "envio-darwin-arm64": "v3.0.0-alpha.9"
-  },
+  "files": [
+    "bin.mjs",
+    "evm.schema.json",
+    "fuel.schema.json",
+    "svm.schema.json",
+    "rescript.json",
+    "index.d.ts",
+    "index.js",
+    "src"
+  ],
   "dependencies": {
-    "@clickhouse/client": "1.12.1",
+    "@clickhouse/client": "1.17.0",
     "@elastic/ecs-pino-format": "1.4.0",
     "@envio-dev/hyperfuel-client": "1.2.2",
-    "@envio-dev/hypersync-client": "0.7.0",
-    "bignumber.js": "9.1.2",
+    "@envio-dev/hypersync-client": "1.1.0",
+    "bignumber.js": "9.3.1",
     "eventsource": "4.1.0",
     "express": "4.19.2",
-    "pino": "10.1.0",
+    "pino": "10.3.1",
     "pino-pretty": "13.1.3",
-    "prom-client": "15.0.0",
+    "prom-client": "15.1.3",
     "yargs": "17.7.2",
     "rescript": "11.1.3",
     "rescript-schema": "9.3.4",
-    "viem": "2.21.0",
+    "viem": "2.46.2",
     "rescript-envsafe": "5.0.0",
     "dotenv": "16.4.5",
     "date-fns": "3.3.1",
-    "@rescript/react": "0.14.0",
-    "ink": "6.5.1",
+    "@rescript/react": "0.14.1",
+    "ink": "6.8.0",
     "ink-big-text": "2.0.0",
     "ink-spinner": "5.0.0",
     "postgres": "3.4.8",
     "tsx": "4.21.0"
   },
-  "files": [
-    "bin.js",
-    "evm.schema.json",
-    "fuel.schema.json",
-    "rescript.json",
-    "index.d.ts",
-    "index.js",
-    "src"
-  ]
-}
+  "optionalDependencies": {
+    "envio-linux-x64": "3.0.0-main-alpha16-build-test",
+    "envio-linux-arm64": "3.0.0-main-alpha16-build-test",
+    "envio-darwin-x64": "3.0.0-main-alpha16-build-test",
+    "envio-darwin-arm64": "3.0.0-main-alpha16-build-test"
+  }
+}

package/src/Batch.res

@@ -6,6 +6,7 @@ open Utils.UnsafeIntOperators
 type chainAfterBatch = {
   batchSize: int,
   progressBlockNumber: int,
+  sourceBlockNumber: int,
   totalEventsProcessed: int,
   fetchState: FetchState.t,
   isProgressAtHeadWhenBatchCreated: bool,
@@ -17,6 +18,7 @@ type chainBeforeBatch = {
   progressBlockNumber: int,
   sourceBlockNumber: int,
   totalEventsProcessed: int,
+  chainConfig: Config.chain,
 }
 
 type t = {
@@ -108,10 +110,11 @@ let getProgressedChainsById = {
           {
             batchSize,
             progressBlockNumber: progressBlockNumberAfterBatch,
+            sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
             totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
             fetchState: fetchStateAfterBatch,
             isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >=
-            chainBeforeBatch.sourceBlockNumber,
+            chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag,
           }: chainAfterBatch
         ),
       )

package/src/Batch.res.mjs

@@ -62,9 +62,10 @@ function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAft
     return {
             batchSize: batchSize,
             progressBlockNumber: progressBlockNumberAfterBatch,
+            sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
             totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
             fetchState: fetchStateAfterBatch,
-            isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber
+            isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag
           };
   }
   
@@ -80,7 +81,7 @@ function getProgressedChainsById(chainsBeforeBatch, batchSizePerChain, progressB
           var progressedChain;
           if (batchSize !== undefined) {
             var leftItems = fetchState.buffer.slice(batchSize);
-            progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, undefined, undefined, leftItems, undefined, undefined), batchSize);
+            progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, FetchState.updateInternal(fetchState, undefined, undefined, leftItems, undefined, undefined), batchSize);
           } else {
             progressedChain = getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAfterBatch, chainBeforeBatch.fetchState, 0);
           }

package/src/Benchmark.res

@@ -269,7 +269,7 @@ let addBlockRangeFetched = (
   ~queryName,
 ) => {
   let group = `BlockRangeFetched Summary for Chain ${chainId->Belt.Int.toString} ${queryName}`
-  let add = (label, value) => data->Data.addSummaryData(~group, ~label, ~value=Utils.magic(value))
+  let add = (label, value) => data->Data.addSummaryData(~group, ~label, ~value=value->(Utils.magic: int => float))
 
   add("Total Time Elapsed (ms)", totalTimeElapsed)
   add("Parsing Time Elapsed (ms)", parsingTimeElapsed)