envio 2.27.6 → 2.28.0-alpha.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.27.6",
+  "version": "v2.28.0-alpha.2",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.27.6",
-    "envio-linux-arm64": "v2.27.6",
-    "envio-darwin-x64": "v2.27.6",
-    "envio-darwin-arm64": "v2.27.6"
+    "envio-linux-x64": "v2.28.0-alpha.2",
+    "envio-linux-arm64": "v2.28.0-alpha.2",
+    "envio-darwin-x64": "v2.28.0-alpha.2",
+    "envio-darwin-arm64": "v2.28.0-alpha.2"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/rescript.json

@@ -13,6 +13,9 @@
     "in-source": true
   },
   "gentypeconfig": {
+    "shims": {
+      "Js": "Js"
+    },
     "generatedFileExtension": ".gen.ts"
   },
   "bs-dependencies": ["rescript-schema"],

package/src/FetchState.res

@@ -818,20 +818,27 @@ let getNextQuery = (
       if (
         p->checkIsFetchingPartition->not && p.latestFetchedBlock.blockNumber < maxQueryBlockNumber
       ) {
-        switch p->makePartitionQuery(
-          ~indexingContracts,
-          ~endBlock=switch blockLag {
-          | 0 => endBlock
-          | _ =>
-            switch endBlock {
-            | Some(endBlock) => Some(Pervasives.min(headBlock, endBlock))
-            // Force head block as an endBlock when blockLag is set
-            // because otherwise HyperSync might return bigger range
-            | None => Some(headBlock)
-            }
-          },
-          ~mergeTarget,
-        ) {
+        let endBlock = switch blockLag {
+        | 0 => endBlock
+        | _ =>
+          switch endBlock {
+          | Some(endBlock) => Some(Pervasives.min(headBlock, endBlock))
+          // Force head block as an endBlock when blockLag is set
+          // because otherwise HyperSync might return bigger range
+          | None => Some(headBlock)
+          }
+        }
+        // Enforce the respose range up until target block
+        // Otherwise for indexers with 100+ partitions
+        // we might blow up the buffer size to more than 600k events
+        // simply because of HyperSync returning extra blocks
+        let endBlock = switch (endBlock, maxQueryBlockNumber < currentBlockHeight) {
+        | (Some(endBlock), true) => Some(Pervasives.min(maxQueryBlockNumber, endBlock))
+        | (None, true) => Some(maxQueryBlockNumber)
+        | (_, false) => endBlock
+        }
+
+        switch p->makePartitionQuery(~indexingContracts, ~endBlock, ~mergeTarget) {
         | Some(q) => queries->Array.push(q)
         | None => ()
         }

package/src/FetchState.res.js

@@ -571,11 +571,20 @@ function getNextQuery(param, concurrencyLimit, targetBufferSize, currentBlockHei
     if (!(!checkIsFetchingPartition(p) && p.latestFetchedBlock.blockNumber < maxQueryBlockNumber)) {
       return ;
     }
-    var q = makePartitionQuery(p, indexingContracts, blockLag !== 0 ? (
-            endBlock !== undefined ? (
-                headBlock < endBlock ? headBlock : endBlock
-              ) : headBlock
-          ) : endBlock, mergeTarget);
+    var endBlock$1 = blockLag !== 0 ? (
+        endBlock !== undefined ? (
+            headBlock < endBlock ? headBlock : endBlock
+          ) : headBlock
+      ) : endBlock;
+    var match = maxQueryBlockNumber < currentBlockHeight;
+    var endBlock$2 = endBlock$1 !== undefined ? (
+        match ? (
+            maxQueryBlockNumber < endBlock$1 ? maxQueryBlockNumber : endBlock$1
+          ) : endBlock$1
+      ) : (
+        match ? maxQueryBlockNumber : endBlock$1
+      );
+    var q = makePartitionQuery(p, indexingContracts, endBlock$2, mergeTarget);
     if (q !== undefined) {
       queries.push(q);
       return ;

package/src/Hasura.res

@@ -222,24 +222,43 @@ let trackDatabase = async (
   ~endpoint,
   ~auth,
   ~pgSchema,
-  ~allStaticTables,
-  ~allEntityTables,
+  ~userEntities: array<Internal.entityConfig>,
   ~aggregateEntities,
   ~responseLimit,
   ~schema,
 ) => {
+  let trackOnlyInternalTableNames = [
+    InternalTable.Chains.table.tableName,
+    InternalTable.EventSyncState.table.tableName,
+    InternalTable.PersistedState.table.tableName,
+    InternalTable.EndOfBlockRangeScannedData.table.tableName,
+    InternalTable.DynamicContractRegistry.table.tableName,
+  ]
+  let exposedInternalTableNames = [
+    InternalTable.RawEvents.table.tableName,
+    InternalTable.Views.metaViewName,
+    InternalTable.Views.chainMetadataViewName,
+  ]
+  let userTableNames = userEntities->Js.Array2.map(entity => entity.table.tableName)
+
   Logging.info("Tracking tables in Hasura")
 
   let _ = await clearHasuraMetadata(~endpoint, ~auth)
-  let tableNames =
-    [allStaticTables, allEntityTables]
-    ->Belt.Array.concatMany
-    ->Js.Array2.map(({tableName}: Table.table) => tableName)
 
-  await trackTables(~endpoint, ~auth, ~pgSchema, ~tableNames)
+  await trackTables(
+    ~endpoint,
+    ~auth,
+    ~pgSchema,
+    ~tableNames=[
+      exposedInternalTableNames,
+      trackOnlyInternalTableNames,
+      userTableNames,
+    ]->Belt.Array.concatMany,
+  )
 
   let _ =
-    await tableNames
+    await [exposedInternalTableNames, userTableNames]
+    ->Belt.Array.concatMany
     ->Js.Array2.map(tableName =>
       createSelectPermissions(
         ~endpoint,
@@ -251,11 +270,11 @@ let trackDatabase = async (
       )
     )
     ->Js.Array2.concatMany(
-      allEntityTables->Js.Array2.map(table => {
-        let {tableName} = table
+      userEntities->Js.Array2.map(entityConfig => {
+        let {tableName} = entityConfig.table
         [
           //Set array relationships
-          table
+          entityConfig.table
           ->Table.getDerivedFromFields
           ->Js.Array2.map(derivedFromField => {
             //determines the actual name of the underlying relational field (if it's an entity mapping then suffixes _id for eg.)
@@ -275,7 +294,7 @@ let trackDatabase = async (
             )
           }),
           //Set object relationships
-          table
+          entityConfig.table
           ->Table.getLinkedEntityFields
           ->Js.Array2.map(((field, linkedEntityName)) => {
             createEntityRelationship(

package/src/Hasura.res.js

@@ -7,6 +7,7 @@ var Utils = require("./Utils.res.js");
 var Schema = require("./db/Schema.res.js");
 var Logging = require("./Logging.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var InternalTable = require("./db/InternalTable.res.js");
 var Caml_splice_call = require("rescript/lib/js/caml_splice_call.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
@@ -203,26 +204,43 @@ async function createEntityRelationship(pgSchema, endpoint, auth, tableName, rel
   }
 }
 
-async function trackDatabase(endpoint, auth, pgSchema, allStaticTables, allEntityTables, aggregateEntities, responseLimit, schema) {
+async function trackDatabase(endpoint, auth, pgSchema, userEntities, aggregateEntities, responseLimit, schema) {
+  var trackOnlyInternalTableNames = [
+    InternalTable.Chains.table.tableName,
+    InternalTable.EventSyncState.table.tableName,
+    InternalTable.PersistedState.table.tableName,
+    InternalTable.EndOfBlockRangeScannedData.table.tableName,
+    InternalTable.DynamicContractRegistry.table.tableName
+  ];
+  var exposedInternalTableNames = [
+    InternalTable.RawEvents.table.tableName,
+    InternalTable.Views.metaViewName,
+    InternalTable.Views.chainMetadataViewName
+  ];
+  var userTableNames = userEntities.map(function (entity) {
+        return entity.table.tableName;
+      });
   Logging.info("Tracking tables in Hasura");
   await clearHasuraMetadata(endpoint, auth);
-  var tableNames = Belt_Array.concatMany([
-          allStaticTables,
-          allEntityTables
-        ]).map(function (param) {
-        return param.tableName;
-      });
-  await trackTables(endpoint, auth, pgSchema, tableNames);
-  await Promise.all(Caml_splice_call.spliceObjApply(tableNames.map(function (tableName) {
+  await trackTables(endpoint, auth, pgSchema, Belt_Array.concatMany([
+            exposedInternalTableNames,
+            trackOnlyInternalTableNames,
+            userTableNames
+          ]));
+  await Promise.all(Caml_splice_call.spliceObjApply(Belt_Array.concatMany([
+                  exposedInternalTableNames,
+                  userTableNames
+                ]).map(function (tableName) {
                 return createSelectPermissions(auth, endpoint, tableName, pgSchema, responseLimit, aggregateEntities);
-              }), "concat", [allEntityTables.map(function (table) {
-                  var tableName = table.tableName;
+              }), "concat", [userEntities.map(function (entityConfig) {
+                  var match = entityConfig.table;
+                  var tableName = match.tableName;
                   return [
-                            Table.getDerivedFromFields(table).map(function (derivedFromField) {
+                            Table.getDerivedFromFields(entityConfig.table).map(function (derivedFromField) {
                                   var relationalFieldName = Utils.unwrapResultExn(Schema.getDerivedFromFieldName(schema, derivedFromField));
                                   return createEntityRelationship(pgSchema, endpoint, auth, tableName, "array", relationalFieldName, derivedFromField.fieldName, derivedFromField.derivedFromEntity, true);
                                 }),
-                            Table.getLinkedEntityFields(table).map(function (param) {
+                            Table.getLinkedEntityFields(entityConfig.table).map(function (param) {
                                   var field = param[0];
                                   return createEntityRelationship(pgSchema, endpoint, auth, tableName, "object", field.fieldName, field.fieldName, param[1], false);
                                 })

package/src/Internal.res

@@ -176,13 +176,16 @@ let fuelTransferParamsSchema = S.schema(s => {
 })
 
 type entity = private {id: string}
-type entityConfig = {
+type genericEntityConfig<'entity> = {
   name: string,
-  schema: S.t<entity>,
-  rowsSchema: S.t<array<entity>>,
+  schema: S.t<'entity>,
+  rowsSchema: S.t<array<'entity>>,
   table: Table.table,
-  entityHistory: EntityHistory.t<entity>,
+  entityHistory: EntityHistory.t<'entity>,
 }
+type entityConfig = genericEntityConfig<entity>
+external fromGenericEntityConfig: genericEntityConfig<'entity> => entityConfig = "%identity"
+
 type enum
 type enumConfig<'enum> = {
   name: string,

package/src/InternalConfig.res

@@ -0,0 +1,20 @@
+// TODO: rename the file to Config.res after finishing the migration from codegen
+// And turn it into PublicConfig instead
+// For internal use we should create Indexer.res with a stateful type
+
+type contract = {
+  name: string,
+  abi: EvmTypes.Abi.t,
+  addresses: array<Address.t>,
+  events: array<Internal.eventConfig>,
+  startBlock: option<int>,
+}
+
+type chain = {
+  id: int,
+  startBlock: int,
+  endBlock?: int,
+  confirmedBlockThreshold: int,
+  contracts: array<contract>,
+  sources: array<Source.t>,
+}

package/src/InternalConfig.res.js

@@ -0,0 +1,2 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+/* This output is empty. Its source's type definitions, externals and/or unused code got optimized away. */

package/src/Js.shim.ts

@@ -0,0 +1,11 @@
+export type Json_t =
+  | string
+  | boolean
+  | number
+  | null
+  | { [key: string]: Json_t }
+  | Json_t[];
+
+export type t = unknown;
+
+export type Exn_t = Error;

package/src/LoadManager.res

@@ -66,9 +66,10 @@ let schedule = async loadManager => {
       }
     })
 
-    if inputsToLoad->Utils.Array.isEmpty->not {
+    let isSuccess = if inputsToLoad->Utils.Array.isEmpty->not {
       try {
         await group.load(inputsToLoad)
+        true
       } catch {
       | exn => {
           let exn = exn->Utils.prettifyExn
@@ -76,16 +77,21 @@ let schedule = async loadManager => {
             let call = calls->Js.Dict.unsafeGet(inputKey)
             call.reject(exn)
           })
+          false
         }
       }
+    } else {
+      true
     }
 
     if currentInputKeys->Utils.Array.isEmpty->not {
-      currentInputKeys->Js.Array2.forEach(inputKey => {
-        let call = calls->Js.Dict.unsafeGet(inputKey)
-        calls->Utils.Dict.deleteInPlace(inputKey)
-        call.resolve(group.getUnsafeInMemory(inputKey))
-      })
+      if isSuccess {
+        currentInputKeys->Js.Array2.forEach(inputKey => {
+          let call = calls->Js.Dict.unsafeGet(inputKey)
+          calls->Utils.Dict.deleteInPlace(inputKey)
+          call.resolve(group.getUnsafeInMemory(inputKey))
+        })
+      }
 
       // Clean up executed batch to reset
       // provided load function which

package/src/LoadManager.res.js

@@ -40,9 +40,13 @@ async function schedule(loadManager) {
               }
               
             });
-        if (!Utils.$$Array.isEmpty(inputsToLoad)) {
+        var isSuccess;
+        if (Utils.$$Array.isEmpty(inputsToLoad)) {
+          isSuccess = true;
+        } else {
           try {
             await group.load(inputsToLoad);
+            isSuccess = true;
           }
           catch (raw_exn){
             var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
@@ -51,16 +55,19 @@ async function schedule(loadManager) {
                     var call = calls[inputKey];
                     call.reject(exn$1);
                   }));
+            isSuccess = false;
           }
         }
         if (Utils.$$Array.isEmpty(currentInputKeys)) {
           return ;
         }
-        currentInputKeys.forEach(function (inputKey) {
-              var call = calls[inputKey];
-              Utils.Dict.deleteInPlace(calls, inputKey);
-              call.resolve(group.getUnsafeInMemory(inputKey));
-            });
+        if (isSuccess) {
+          currentInputKeys.forEach(function (inputKey) {
+                var call = calls[inputKey];
+                Utils.Dict.deleteInPlace(calls, inputKey);
+                call.resolve(group.getUnsafeInMemory(inputKey));
+              });
+        }
         var latestGroup = groups[key];
         if (Utils.$$Array.isEmpty(Object.keys(latestGroup.calls))) {
           return Utils.Dict.deleteInPlace(groups, key);

package/src/Persistence.res

@@ -13,6 +13,12 @@ type effectCacheRecord = {
   mutable count: int,
 }
 
+type initialState = {
+  cleanRun: bool,
+  cache: dict<effectCacheRecord>,
+  chains: array<InternalTable.Chains.t>,
+}
+
 type operator = [#">" | #"="]
 
 type storage = {
@@ -22,10 +28,11 @@ type storage = {
   // Should initialize the storage so we can start interacting with it
   // Eg create connection, schema, tables, etc.
   initialize: (
+    ~chainConfigs: array<InternalConfig.chain>=?,
     ~entities: array<Internal.entityConfig>=?,
-    ~generalTables: array<Table.table>=?,
     ~enums: array<Internal.enumConfig<Internal.enum>>=?,
-  ) => promise<unit>,
+  ) => promise<initialState>,
+  loadInitialState: unit => promise<initialState>,
   @raises("StorageError")
   loadByIdsOrThrow: 'item. (
     ~ids: array<string>,
@@ -55,10 +62,6 @@ type storage = {
   ) => promise<unit>,
   // This is to download cache from the database to .envio/cache
   dumpEffectCache: unit => promise<unit>,
-  // This is not good, but the function does two things:
-  // - Gets info about existing cache tables
-  // - if withUpload is true, it also populates the cache from .envio/cache to the database
-  restoreEffectCache: (~withUpload: bool) => promise<array<effectCacheRecord>>,
 }
 
 exception StorageError({message: string, reason: exn})
@@ -66,11 +69,10 @@ exception StorageError({message: string, reason: exn})
 type storageStatus =
   | Unknown
   | Initializing(promise<unit>)
-  | Ready({cleanRun: bool, cache: dict<effectCacheRecord>})
+  | Ready(initialState)
 
 type t = {
   userEntities: array<Internal.entityConfig>,
-  staticTables: array<Table.table>,
   allEntities: array<Internal.entityConfig>,
   allEnums: array<Internal.enumConfig<Internal.enum>>,
   mutable storageStatus: storageStatus,
@@ -86,18 +88,15 @@ let entityHistoryActionEnumConfig: Internal.enumConfig<EntityHistory.RowAction.t
 
 let make = (
   ~userEntities,
-  ~dcRegistryEntityConfig,
   // TODO: Should only pass userEnums and create internal config in runtime
   ~allEnums,
-  ~staticTables,
   ~storage,
 ) => {
-  let allEntities = userEntities->Js.Array2.concat([dcRegistryEntityConfig])
+  let allEntities = userEntities->Js.Array2.concat([InternalTable.DynamicContractRegistry.config])
   let allEnums =
     allEnums->Js.Array2.concat([entityHistoryActionEnumConfig->Internal.fromGenericEnumConfig])
   {
     userEntities,
-    staticTables,
     allEntities,
     allEnums,
     storageStatus: Unknown,
@@ -106,17 +105,7 @@ let make = (
 }
 
 let init = {
-  let loadInitialCache = async (persistence, ~withUpload) => {
-    let effectCacheRecords = await persistence.storage.restoreEffectCache(~withUpload)
-    let cache = Js.Dict.empty()
-    effectCacheRecords->Js.Array2.forEach(record => {
-      Prometheus.EffectCacheCount.set(~count=record.count, ~effectName=record.effectName)
-      cache->Js.Dict.set(record.effectName, record)
-    })
-    cache
-  }
-
-  async (persistence, ~reset=false) => {
+  async (persistence, ~chainConfigs, ~reset=false) => {
     try {
       let shouldRun = switch persistence.storageStatus {
       | Unknown => true
@@ -135,17 +124,14 @@ let init = {
         if reset || !(await persistence.storage.isInitialized()) {
           Logging.info(`Initializing the indexer storage...`)
 
-          await persistence.storage.initialize(
+          let initialState = await persistence.storage.initialize(
             ~entities=persistence.allEntities,
-            ~generalTables=persistence.staticTables,
             ~enums=persistence.allEnums,
+            ~chainConfigs,
           )
 
           Logging.info(`The indexer storage is ready. Uploading cache...`)
-          persistence.storageStatus = Ready({
-            cleanRun: true,
-            cache: await loadInitialCache(persistence, ~withUpload=true),
-          })
+          persistence.storageStatus = Ready(initialState)
         } else if (
           // In case of a race condition,
           // we want to set the initial status to Ready only once.
@@ -155,10 +141,7 @@ let init = {
           }
         ) {
           Logging.info(`The indexer storage is ready.`)
-          persistence.storageStatus = Ready({
-            cleanRun: false,
-            cache: await loadInitialCache(persistence, ~withUpload=false),
-          })
+          persistence.storageStatus = Ready(await persistence.storage.loadInitialState())
         }
         resolveRef.contents()
       }
@@ -178,6 +161,15 @@ let getInitializedStorageOrThrow = persistence => {
   }
 }
 
+let getInitializedState = persistence => {
+  switch persistence.storageStatus {
+  | Unknown
+  | Initializing(_) =>
+    Js.Exn.raiseError(`Failed to access the initial state. The Persistence layer is not initialized.`)
+  | Ready(initialState) => initialState
+  }
+}
+
 let setEffectCacheOrThrow = async (persistence, ~effect: Internal.effect, ~items) => {
   switch persistence.storageStatus {
   | Unknown

package/src/Persistence.res.js

@@ -6,6 +6,7 @@ var Logging = require("./Logging.res.js");
 var Prometheus = require("./Prometheus.res.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
+var InternalTable = require("./db/InternalTable.res.js");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
@@ -24,12 +25,11 @@ var entityHistoryActionEnumConfig = {
   default: "SET"
 };
 
-function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, storage) {
-  var allEntities = userEntities.concat([dcRegistryEntityConfig]);
+function make(userEntities, allEnums, storage) {
+  var allEntities = userEntities.concat([InternalTable.DynamicContractRegistry.config]);
   var allEnums$1 = allEnums.concat([entityHistoryActionEnumConfig]);
   return {
           userEntities: userEntities,
-          staticTables: staticTables,
           allEntities: allEntities,
           allEnums: allEnums$1,
           storageStatus: "Unknown",
@@ -37,17 +37,7 @@ function make(userEntities, dcRegistryEntityConfig, allEnums, staticTables, stor
         };
 }
 
-async function loadInitialCache(persistence, withUpload) {
-  var effectCacheRecords = await persistence.storage.restoreEffectCache(withUpload);
-  var cache = {};
-  effectCacheRecords.forEach(function (record) {
-        Prometheus.EffectCacheCount.set(record.count, record.effectName);
-        cache[record.effectName] = record;
-      });
-  return cache;
-}
-
-async function init(persistence, resetOpt) {
+async function init(persistence, chainConfigs, resetOpt) {
   var reset = resetOpt !== undefined ? resetOpt : false;
   try {
     var promise = persistence.storageStatus;
@@ -75,12 +65,11 @@ async function init(persistence, resetOpt) {
     };
     if (reset || !await persistence.storage.isInitialized()) {
       Logging.info("Initializing the indexer storage...");
-      await persistence.storage.initialize(persistence.allEntities, persistence.staticTables, persistence.allEnums);
+      var initialState = await persistence.storage.initialize(chainConfigs, persistence.allEntities, persistence.allEnums);
       Logging.info("The indexer storage is ready. Uploading cache...");
       persistence.storageStatus = {
         TAG: "Ready",
-        cleanRun: true,
-        cache: await loadInitialCache(persistence, true)
+        _0: initialState
       };
     } else {
       var match = persistence.storageStatus;
@@ -90,8 +79,7 @@ async function init(persistence, resetOpt) {
         Logging.info("The indexer storage is ready.");
         persistence.storageStatus = {
           TAG: "Ready",
-          cleanRun: false,
-          cache: await loadInitialCache(persistence, false)
+          _0: await persistence.storage.loadInitialState()
         };
       }
       
@@ -113,6 +101,15 @@ function getInitializedStorageOrThrow(persistence) {
   }
 }
 
+function getInitializedState(persistence) {
+  var initialState = persistence.storageStatus;
+  if (typeof initialState !== "object" || initialState.TAG === "Initializing") {
+    return Js_exn.raiseError("Failed to access the initial state. The Persistence layer is not initialized.");
+  } else {
+    return initialState._0;
+  }
+}
+
 async function setEffectCacheOrThrow(persistence, effect, items) {
   var match = persistence.storageStatus;
   if (typeof match !== "object") {
@@ -121,7 +118,7 @@ async function setEffectCacheOrThrow(persistence, effect, items) {
   if (match.TAG === "Initializing") {
     return Js_exn.raiseError("Failed to access the indexer storage. The Persistence layer is not initialized.");
   }
-  var cache = match.cache;
+  var cache = match._0.cache;
   var storage = persistence.storage;
   var effectName = effect.name;
   var c = cache[effectName];
@@ -147,5 +144,6 @@ exports.entityHistoryActionEnumConfig = entityHistoryActionEnumConfig;
 exports.make = make;
 exports.init = init;
 exports.getInitializedStorageOrThrow = getInitializedStorageOrThrow;
+exports.getInitializedState = getInitializedState;
 exports.setEffectCacheOrThrow = setEffectCacheOrThrow;
 /* Logging Not a pure module */