envio 3.0.0-alpha.13 → 3.0.0-alpha.15

package/README.md

@@ -12,7 +12,7 @@ HyperIndex is a fast, developer-friendly multichain indexer, optimized for both
 ## Key Features
 
 - **[Indexer auto-generation](https://docs.envio.dev/docs/HyperIndex/contract-import)** – Generate Indexers directly from smart contract addresses
-- **High performance** – Historical backfills at over 10,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
+- **High performance** – Historical backfills at over 25,000+ events per second ([fastest in market](https://docs.envio.dev/blog/indexer-benchmarking-results))
 - **Local development** – Full-featured local environment with Docker
 - **[Multichain indexing](https://docs.envio.dev/docs/HyperIndex/multichain-indexing)** – Index any EVM-, SVM-, or Fuel-compatible blockchain
 - **Real-time indexing** – Instantly track blockchain events

package/evm.schema.json

@@ -73,17 +73,6 @@
         "$ref": "#/$defs/Chain"
       }
     },
-    "multichain": {
-      "description": "Multichain mode: 'ordered' processes events across chains in order, 'unordered' processes chain events in order, but non-deterministically relatively to other chains (default: unordered)",
-      "anyOf": [
-        {
-          "$ref": "#/$defs/Multichain"
-        },
-        {
-          "type": "null"
-        }
-      ]
-    },
     "rollback_on_reorg": {
       "description": "A flag to indicate if the indexer should rollback to the last known valid block on a reorg. This currently incurs a performance hit on historical sync and is recommended to turn this off while developing (default: true)",
       "type": [
@@ -335,7 +324,17 @@
             "integer",
             "null"
           ],
-          "format": "int32"
+          "format": "uint32",
+          "minimum": 0
+        },
+        "block_lag": {
+          "description": "The number of blocks behind the chain head that the indexer should lag. Useful for avoiding reorg issues by indexing slightly behind the tip.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint32",
+          "minimum": 0
         },
         "start_block": {
           "description": "The block at which the indexer should start ingesting data",
@@ -597,13 +596,6 @@
         }
       ]
     },
-    "Multichain": {
-      "type": "string",
-      "enum": [
-        "ordered",
-        "unordered"
-      ]
-    },
     "AddressFormat": {
       "type": "string",
       "enum": [

package/fuel.schema.json

@@ -196,6 +196,24 @@
             }
           ]
         },
+        "max_reorg_depth": {
+          "description": "The number of blocks from the head that the indexer should account for in case of reorgs.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint32",
+          "minimum": 0
+        },
+        "block_lag": {
+          "description": "The number of blocks behind the chain head that the indexer should lag. Useful for avoiding reorg issues by indexing slightly behind the tip.",
+          "type": [
+            "integer",
+            "null"
+          ],
+          "format": "uint32",
+          "minimum": 0
+        },
         "contracts": {
           "description": "All the contracts that should be indexed on the given chain",
           "type": [

package/index.d.ts

@@ -18,6 +18,36 @@ export type { EffectCaller, Address } from "./src/Types.ts";
 /** Utility type to expand/flatten complex types for better IDE display. */
 export type Prettify<T> = { [K in keyof T]: T[K] } & {};
 
+/**
+ * Operator for filtering entity fields in getWhere queries.
+ * Only fields with `@index` in the schema can be queried at runtime.
+ */
+export type WhereOperator<T> = {
+  /** Matches entities where the field equals the given value. */
+  readonly _eq?: T;
+  /** Matches entities where the field is greater than the given value. */
+  readonly _gt?: T;
+  /** Matches entities where the field is less than the given value. */
+  readonly _lt?: T;
+  /** Matches entities where the field is greater than or equal to the given value. */
+  readonly _gte?: T;
+  /** Matches entities where the field is less than or equal to the given value. */
+  readonly _lte?: T;
+  /** Matches entities where the field equals any of the given values. */
+  readonly _in?: readonly T[];
+};
+
+/**
+ * Constructs a getWhere filter type from an entity type.
+ * Each field can be filtered using {@link WhereOperator} (`_eq`, `_gt`, `_lt`, `_gte`, `_lte`, `_in`).
+ *
+ * Note: only fields with `@index` in the schema can be queried at runtime.
+ * Attempting to filter on a non-indexed field will throw a descriptive error.
+ */
+export type GetWhereFilter<E> = {
+  [K in keyof E]?: WhereOperator<E[K]>;
+};
+
 import type {
   effect as Effect,
   effectArgs as EffectArgs,
@@ -234,6 +264,8 @@ type EvmChainConfig<Id extends number = number> = {
   readonly endBlock?: number;
   /** Number of blocks to keep for reorg handling (default: 200). */
   readonly maxReorgDepth?: number;
+  /** Number of blocks behind the chain head to lag (default: 0). */
+  readonly blockLag?: number;
 };
 
 /** EVM chain value (for runtime Indexer). */
@@ -287,6 +319,8 @@ type FuelChainConfig<Id extends number = number> = {
   readonly endBlock?: number;
   /** Number of blocks to keep for reorg handling (default: 200). */
   readonly maxReorgDepth?: number;
+  /** Number of blocks behind the chain head to lag (default: 0). */
+  readonly blockLag?: number;
 };
 
 /** Fuel chain value (for runtime Indexer). */
@@ -320,6 +354,8 @@ type SvmChainConfig<Id extends number = number> = {
   readonly endBlock?: number;
   /** Number of blocks to keep for reorg handling (default: 200). */
   readonly maxReorgDepth?: number;
+  /** Number of blocks behind the chain head to lag (default: 0). */
+  readonly blockLag?: number;
 };
 
 /** SVM chain value (for runtime Indexer). */
@@ -589,7 +625,9 @@ export type TestIndexerFromConfig<Config extends IndexerConfigTypes> = {
     /** Changes happened during the processing. */
     readonly changes: readonly EntityChange<Config>[];
   }>;
-} & {
+} & (EcosystemCount<Config> extends 1
+  ? SingleEcosystemChains<Config>
+  : MultiEcosystemChains<Config>) & {
   /** Entity operations for direct manipulation outside of handlers. */
   readonly [K in keyof ConfigEntities<Config>]: EntityOps<
     ConfigEntities<Config>[K]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v3.0.0-alpha.13",
+  "version": "3.0.0-alpha.15",
   "type": "module",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
@@ -10,30 +10,35 @@
     "type": "git",
     "url": "git+https://github.com/enviodev/hyperindex.git"
   },
+  "author": "envio contributors <about@envio.dev>",
+  "license": "GPL-3.0",
+  "bugs": {
+    "url": "https://github.com/enviodev/hyperindex/issues"
+  },
+  "homepage": "https://envio.dev",
   "keywords": [
     "blockchain",
     "indexer",
     "ethereum",
     "evm",
     "fuel",
+    "solana",
     "data",
     "dapp"
   ],
-  "author": "envio contributors <about@envio.dev>",
-  "license": "GPL-3.0",
-  "bugs": {
-    "url": "https://github.com/enviodev/hyperindex/issues"
-  },
-  "homepage": "https://envio.dev",
   "engines": {
     "node": ">=22.0.0"
   },
-  "optionalDependencies": {
-    "envio-linux-x64": "v3.0.0-alpha.13",
-    "envio-linux-arm64": "v3.0.0-alpha.13",
-    "envio-darwin-x64": "v3.0.0-alpha.13",
-    "envio-darwin-arm64": "v3.0.0-alpha.13"
-  },
+  "files": [
+    "bin.js",
+    "evm.schema.json",
+    "fuel.schema.json",
+    "svm.schema.json",
+    "rescript.json",
+    "index.d.ts",
+    "index.js",
+    "src"
+  ],
   "dependencies": {
     "@clickhouse/client": "1.12.1",
     "@elastic/ecs-pino-format": "1.4.0",
@@ -59,13 +64,10 @@
     "postgres": "3.4.8",
     "tsx": "4.21.0"
   },
-  "files": [
-    "bin.js",
-    "evm.schema.json",
-    "fuel.schema.json",
-    "rescript.json",
-    "index.d.ts",
-    "index.js",
-    "src"
-  ]
-}
+  "optionalDependencies": {
+    "envio-linux-x64": "3.0.0-alpha.15",
+    "envio-linux-arm64": "3.0.0-alpha.15",
+    "envio-darwin-x64": "3.0.0-alpha.15",
+    "envio-darwin-arm64": "3.0.0-alpha.15"
+  }
+}

package/src/Batch.res

@@ -18,6 +18,7 @@ type chainBeforeBatch = {
   progressBlockNumber: int,
   sourceBlockNumber: int,
   totalEventsProcessed: int,
+  chainConfig: Config.chain,
 }
 
 type t = {
@@ -113,7 +114,7 @@ let getProgressedChainsById = {
             totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
             fetchState: fetchStateAfterBatch,
             isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >=
-            chainBeforeBatch.sourceBlockNumber,
+            chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag,
           }: chainAfterBatch
         ),
       )

package/src/Batch.res.mjs

@@ -65,7 +65,7 @@ function getChainAfterBatchIfProgressed(chainBeforeBatch, progressBlockNumberAft
             sourceBlockNumber: chainBeforeBatch.sourceBlockNumber,
             totalEventsProcessed: chainBeforeBatch.totalEventsProcessed + batchSize,
             fetchState: fetchStateAfterBatch,
-            isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber
+            isProgressAtHeadWhenBatchCreated: progressBlockNumberAfterBatch >= chainBeforeBatch.sourceBlockNumber - chainBeforeBatch.chainConfig.blockLag
           };
   }
   

package/src/ChainFetcher.res

@@ -15,7 +15,6 @@ type t = {
   isProgressAtHead: bool,
   timestampCaughtUpToHeadOrEndblock: option<Js.Date.t>,
   committedProgressBlockNumber: int,
-  firstEventBlockNumber: option<int>,
   numEventsProcessed: int,
   numBatchesFetched: int,
   reorgDetection: ReorgDetection.t,
@@ -28,10 +27,10 @@ let make = (
   ~dynamicContracts: array<Internal.indexingContract>,
   ~startBlock,
   ~endBlock,
-  ~firstEventBlockNumber,
+  ~firstEventBlock=None,
   ~progressBlockNumber,
   ~config: Config.t,
-  ~registrations: EventRegister.registrations,
+  ~registrations: HandlerRegister.registrations,
   ~targetBufferSize,
   ~logger,
   ~timestampCaughtUpToHeadOrEndblock,
@@ -149,7 +148,7 @@ let make = (
     registrations.onBlockByChainId->Utils.Dict.dangerouslyGetNonOption(chainConfig.id->Int.toString)
   switch onBlockConfigs {
   | Some(onBlockConfigs) =>
-    // TODO: Move it to the EventRegister module
+    // TODO: Move it to the HandlerRegister module
     // so the error is thrown with better stack trace
     onBlockConfigs->Array.forEach(onBlockConfig => {
       if onBlockConfig.startBlock->Option.getWithDefault(startBlock) < startBlock {
@@ -183,9 +182,10 @@ let make = (
     // FIXME: Shouldn't set with full history
     ~blockLag=Pervasives.max(
       !config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth,
-      Env.indexingBlockLag->Option.getWithDefault(0),
+      chainConfig.blockLag,
     ),
     ~onBlockConfigs?,
+    ~firstEventBlock,
   )
 
   let chainReorgCheckpoints = reorgCheckpoints->Array.keepMapU(reorgCheckpoint => {
@@ -258,7 +258,6 @@ let make = (
     ),
     isProgressAtHead: false,
     fetchState,
-    firstEventBlockNumber,
     committedProgressBlockNumber: progressBlockNumber,
     timestampCaughtUpToHeadOrEndblock,
     numEventsProcessed,
@@ -283,7 +282,6 @@ let makeFromConfig = (
     ~endBlock=chainConfig.endBlock,
     ~reorgCheckpoints=[],
     ~maxReorgDepth=chainConfig.maxReorgDepth,
-    ~firstEventBlockNumber=None,
     ~progressBlockNumber=-1,
     ~timestampCaughtUpToHeadOrEndblock=None,
     ~numEventsProcessed=0,
@@ -328,7 +326,7 @@ let makeFromDbState = async (
     ~registrations,
     ~reorgCheckpoints,
     ~maxReorgDepth=resumedChainState.maxReorgDepth,
-    ~firstEventBlockNumber=resumedChainState.firstEventBlockNumber,
+    ~firstEventBlock=resumedChainState.firstEventBlockNumber,
     ~progressBlockNumber,
     ~timestampCaughtUpToHeadOrEndblock=Env.updateSyncTimeOnRestart
       ? None

package/src/ChainFetcher.res.mjs

@@ -23,7 +23,8 @@ import * as HyperFuelSource from "./sources/HyperFuelSource.res.mjs";
 import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 import * as SafeCheckpointTracking from "./SafeCheckpointTracking.res.mjs";
 
-function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlockNumber, progressBlockNumber, config, registrations, targetBufferSize, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, numBatchesFetched, isInReorgThreshold, reorgCheckpoints, maxReorgDepth, knownHeightOpt) {
+function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlockOpt, progressBlockNumber, config, registrations, targetBufferSize, logger, timestampCaughtUpToHeadOrEndblock, numEventsProcessed, numBatchesFetched, isInReorgThreshold, reorgCheckpoints, maxReorgDepth, knownHeightOpt) {
+  var firstEventBlock = firstEventBlockOpt !== undefined ? Caml_option.valFromOption(firstEventBlockOpt) : undefined;
   var knownHeight = knownHeightOpt !== undefined ? knownHeightOpt : 0;
   var eventRouter = EventRouter.empty();
   var contracts = [];
@@ -99,7 +100,7 @@ function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlo
             
           }));
   }
-  var fetchState = FetchState.make(startBlock, endBlock, eventConfigs, contracts, config.maxAddrInPartition, chainConfig.id, targetBufferSize, knownHeight, progressBlockNumber, onBlockConfigs, Caml.int_max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, Belt_Option.getWithDefault(Env.indexingBlockLag, 0)));
+  var fetchState = FetchState.make(startBlock, endBlock, eventConfigs, contracts, config.maxAddrInPartition, chainConfig.id, targetBufferSize, knownHeight, progressBlockNumber, onBlockConfigs, Caml.int_max(!config.shouldRollbackOnReorg || isInReorgThreshold ? 0 : chainConfig.maxReorgDepth, chainConfig.blockLag), Caml_option.some(firstEventBlock));
   var chainReorgCheckpoints = Belt_Array.keepMapU(reorgCheckpoints, (function (reorgCheckpoint) {
           if (reorgCheckpoint.chain_id === chainConfig.id) {
             return reorgCheckpoint;
@@ -156,7 +157,6 @@ function make(chainConfig, dynamicContracts, startBlock, endBlock, firstEventBlo
           isProgressAtHead: false,
           timestampCaughtUpToHeadOrEndblock: timestampCaughtUpToHeadOrEndblock,
           committedProgressBlockNumber: progressBlockNumber,
-          firstEventBlockNumber: firstEventBlockNumber,
           numEventsProcessed: numEventsProcessed,
           numBatchesFetched: numBatchesFetched,
           reorgDetection: ReorgDetection.make(chainReorgCheckpoints, maxReorgDepth, config.shouldRollbackOnReorg),
@@ -178,7 +178,7 @@ async function makeFromDbState(chainConfig, resumedChainState, reorgCheckpoints,
       });
   Prometheus.ProgressEventsCount.set(resumedChainState.numEventsProcessed, chainId);
   var progressBlockNumber = resumedChainState.progressBlockNumber >= 0 ? resumedChainState.progressBlockNumber : resumedChainState.startBlock - 1 | 0;
-  return make(chainConfig, resumedChainState.dynamicContracts, resumedChainState.startBlock, resumedChainState.endBlock, resumedChainState.firstEventBlockNumber, progressBlockNumber, config, registrations, targetBufferSize, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, 0, isInReorgThreshold, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber);
+  return make(chainConfig, resumedChainState.dynamicContracts, resumedChainState.startBlock, resumedChainState.endBlock, Caml_option.some(resumedChainState.firstEventBlockNumber), progressBlockNumber, config, registrations, targetBufferSize, logger, Env.updateSyncTimeOnRestart ? undefined : resumedChainState.timestampCaughtUpToHeadOrEndblock, resumedChainState.numEventsProcessed, 0, isInReorgThreshold, reorgCheckpoints, resumedChainState.maxReorgDepth, resumedChainState.sourceBlockNumber);
 }
 
 function getContractStartBlock(config, chain, contractName) {
@@ -263,7 +263,6 @@ function handleQueryResult(chainFetcher, query, newItems, newItemsWithDcs, lates
           isProgressAtHead: chainFetcher.isProgressAtHead,
           timestampCaughtUpToHeadOrEndblock: chainFetcher.timestampCaughtUpToHeadOrEndblock,
           committedProgressBlockNumber: chainFetcher.committedProgressBlockNumber,
-          firstEventBlockNumber: chainFetcher.firstEventBlockNumber,
           numEventsProcessed: chainFetcher.numEventsProcessed,
           numBatchesFetched: chainFetcher.numBatchesFetched,
           reorgDetection: chainFetcher.reorgDetection,

package/src/ChainManager.res

@@ -125,6 +125,7 @@ let createBatch = (chainManager: t, ~batchSizeTarget: int, ~isRollback: bool): B
       totalEventsProcessed: cf.numEventsProcessed,
       sourceBlockNumber: cf.fetchState.knownHeight,
       reorgDetection: cf.reorgDetection,
+      chainConfig: cf.chainConfig,
     }),
     ~multichain=chainManager.multichain,
     ~batchSizeTarget,

package/src/ChainManager.res.mjs

@@ -93,7 +93,8 @@ function createBatch(chainManager, batchSizeTarget, isRollback) {
                             reorgDetection: cf.reorgDetection,
                             progressBlockNumber: cf.committedProgressBlockNumber,
                             sourceBlockNumber: cf.fetchState.knownHeight,
-                            totalEventsProcessed: cf.numEventsProcessed
+                            totalEventsProcessed: cf.numEventsProcessed,
+                            chainConfig: cf.chainConfig
                           };
                   })), chainManager.multichain, batchSizeTarget);
 }

package/src/Config.gen.ts

@@ -0,0 +1,19 @@
+/* TypeScript file generated from Config.res by genType. */
+
+/* eslint-disable */
+/* tslint:disable */
+
+import type {t as Address_t} from './Address.gen.js';
+
+export type DynamicContractRegistry_t = {
+  readonly id: string; 
+  readonly chain_id: number; 
+  readonly registering_event_block_number: number; 
+  readonly registering_event_log_index: number; 
+  readonly registering_event_block_timestamp: number; 
+  readonly registering_event_contract_name: string; 
+  readonly registering_event_name: string; 
+  readonly registering_event_src_address: Address_t; 
+  readonly contract_address: Address_t; 
+  readonly contract_name: string
+};