envio 3.0.0-alpha.2 → 3.0.0-alpha.21

package/src/Main.res.mjs

@@ -0,0 +1,341 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Env from "./Env.res.mjs";
+import * as Tui from "./tui/Tui.res.mjs";
+import * as Caml from "rescript/lib/es6/caml.js";
+import * as Utils from "./Utils.res.mjs";
+import * as Js_dict from "rescript/lib/es6/js_dict.js";
+import * as Logging from "./Logging.res.mjs";
+import Express from "express";
+import * as Process from "process";
+import * as ChainMap from "./ChainMap.res.mjs";
+import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
+import * as FetchState from "./FetchState.res.mjs";
+import * as Prometheus from "./Prometheus.res.mjs";
+import * as Belt_Option from "rescript/lib/es6/belt_Option.js";
+import * as Caml_option from "rescript/lib/es6/caml_option.js";
+import * as GlobalState from "./GlobalState.res.mjs";
+import * as Persistence from "./Persistence.res.mjs";
+import * as PromClient from "prom-client";
+import Yargs from "yargs/yargs";
+import * as ChainFetcher from "./ChainFetcher.res.mjs";
+import * as ChainManager from "./ChainManager.res.mjs";
+import * as HandlerLoader from "./HandlerLoader.res.mjs";
+import * as SourceManager from "./sources/SourceManager.res.mjs";
+import * as Helpers from "yargs/helpers";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as GlobalStateManager from "./GlobalStateManager.res.mjs";
+
+var chainDataSchema = S$RescriptSchema.schema(function (s) {
+      return {
+              chainId: s.m(S$RescriptSchema.$$float),
+              poweredByHyperSync: s.m(S$RescriptSchema.bool),
+              firstEventBlockNumber: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
+              latestProcessedBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
+              timestampCaughtUpToHeadOrEndblock: s.m(S$RescriptSchema.option(S$RescriptSchema.datetime(S$RescriptSchema.string, undefined))),
+              numEventsProcessed: s.m(S$RescriptSchema.$$float),
+              latestFetchedBlockNumber: s.m(S$RescriptSchema.$$int),
+              currentBlockHeight: s.m(S$RescriptSchema.$$int),
+              numBatchesFetched: s.m(S$RescriptSchema.$$int),
+              startBlock: s.m(S$RescriptSchema.$$int),
+              endBlock: s.m(S$RescriptSchema.option(S$RescriptSchema.$$int)),
+              numAddresses: s.m(S$RescriptSchema.$$int)
+            };
+    });
+
+var stateSchema = S$RescriptSchema.union([
+      S$RescriptSchema.literal({
+            status: "disabled"
+          }),
+      S$RescriptSchema.literal({
+            status: "initializing"
+          }),
+      S$RescriptSchema.schema(function (s) {
+            return {
+                    status: "active",
+                    envioVersion: s.m(S$RescriptSchema.string),
+                    chains: s.m(S$RescriptSchema.array(chainDataSchema)),
+                    indexerStartTime: s.m(S$RescriptSchema.datetime(S$RescriptSchema.string, undefined)),
+                    isPreRegisteringDynamicContracts: false,
+                    isUnorderedMultichainMode: s.m(S$RescriptSchema.bool),
+                    rollbackOnReorg: s.m(S$RescriptSchema.bool)
+                  };
+          })
+    ]);
+
+var globalGsManagerRef = {
+  contents: undefined
+};
+
+function getGlobalIndexer(config) {
+  var indexer = Object.create(null);
+  Object.defineProperty(Object.defineProperty(indexer, "name", {
+            enumerable: true,
+            value: config.name
+          }), "description", {
+        enumerable: true,
+        value: config.description
+      });
+  var chainIds = [];
+  var chains = Object.create(null);
+  Belt_Array.forEach(ChainMap.values(config.chainMap), (function (chainConfig) {
+          var chainIdStr = String(chainConfig.id);
+          chainIds.push(chainConfig.id);
+          var chainObj = Object.create(null);
+          Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(Object.defineProperty(chainObj, "id", {
+                                enumerable: true,
+                                value: chainConfig.id
+                              }), "startBlock", {
+                            enumerable: true,
+                            value: chainConfig.startBlock
+                          }), "endBlock", {
+                        enumerable: true,
+                        value: chainConfig.endBlock
+                      }), "name", {
+                    enumerable: true,
+                    value: chainConfig.name
+                  }), "isLive", {
+                enumerable: true,
+                get: (function () {
+                    var gsManager = globalGsManagerRef.contents;
+                    if (gsManager === undefined) {
+                      return false;
+                    }
+                    var state = GlobalStateManager.getState(Caml_option.valFromOption(gsManager));
+                    var chain = ChainMap.Chain.makeUnsafe(chainConfig.id);
+                    var chainFetcher = ChainMap.get(state.chainManager.chainFetchers, chain);
+                    return ChainFetcher.isReady(chainFetcher);
+                  })
+              });
+          Belt_Array.forEach(chainConfig.contracts, (function (contract) {
+                  var contractObj = Object.create(null);
+                  Object.defineProperty(Object.defineProperty(Object.defineProperty(contractObj, "name", {
+                                enumerable: true,
+                                value: contract.name
+                              }), "abi", {
+                            enumerable: true,
+                            value: contract.abi
+                          }), "addresses", {
+                        enumerable: true,
+                        get: (function () {
+                            var gsManager = globalGsManagerRef.contents;
+                            if (gsManager === undefined) {
+                              return contract.addresses;
+                            }
+                            var state = GlobalStateManager.getState(Caml_option.valFromOption(gsManager));
+                            var chain = ChainMap.Chain.makeUnsafe(chainConfig.id);
+                            var chainFetcher = ChainMap.get(state.chainManager.chainFetchers, chain);
+                            var indexingContracts = chainFetcher.fetchState.indexingContracts;
+                            var addresses = [];
+                            var values = Js_dict.values(indexingContracts);
+                            for(var idx = 0 ,idx_finish = values.length; idx < idx_finish; ++idx){
+                              var indexingContract = values[idx];
+                              if (indexingContract.contractName === contract.name) {
+                                addresses.push(indexingContract.address);
+                              }
+                              
+                            }
+                            return addresses;
+                          })
+                      });
+                  Object.defineProperty(chainObj, contract.name, {
+                        enumerable: true,
+                        value: contractObj
+                      });
+                }));
+          Object.defineProperty(chains, chainIdStr, {
+                enumerable: true,
+                value: chainObj
+              });
+          if (chainConfig.name !== chainIdStr) {
+            Object.defineProperty(chains, chainConfig.name, {
+                  enumerable: false,
+                  value: chainObj
+                });
+            return ;
+          }
+          
+        }));
+  Object.defineProperty(indexer, "chainIds", {
+        enumerable: true,
+        value: chainIds
+      });
+  Object.defineProperty(indexer, "chains", {
+        enumerable: true,
+        value: chains
+      });
+  return indexer;
+}
+
+function startServer(getState, ctx, isDevelopmentMode) {
+  var app = Express();
+  var consoleCorsMiddleware = function (req, res, next) {
+    var origin = Js_dict.get(req.headers, "origin");
+    if (origin !== undefined && (origin === Env.prodEnvioAppUrl || origin === Env.envioAppUrl)) {
+      res.setHeader("Access-Control-Allow-Origin", origin);
+    }
+    res.setHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS");
+    res.setHeader("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
+    if (req.method === "OPTIONS") {
+      res.sendStatus(200);
+      return ;
+    } else {
+      return next();
+    }
+  };
+  app.use("/console", consoleCorsMiddleware);
+  app.use("/metrics", consoleCorsMiddleware);
+  app.use("/metrics/runtime", consoleCorsMiddleware);
+  app.get("/healthz", (function (_req, res) {
+          res.sendStatus(200);
+        }));
+  app.get("/console/state", (function (_req, res) {
+          var state = isDevelopmentMode ? getState() : ({
+                status: "disabled"
+              });
+          res.json(S$RescriptSchema.reverseConvertToJsonOrThrow(state, stateSchema));
+        }));
+  app.post("/console/syncCache", (function (_req, res) {
+          if (isDevelopmentMode) {
+            Persistence.getInitializedStorageOrThrow(ctx.persistence).dumpEffectCache().then(function () {
+                  res.json(true);
+                });
+          } else {
+            res.json(false);
+          }
+        }));
+  var runtimeRegistry = new PromClient.Registry();
+  PromClient.collectDefaultMetrics({
+        register: runtimeRegistry
+      });
+  app.get("/metrics", (function (_req, res) {
+          res.set("Content-Type", PromClient.register.contentType);
+          PromClient.register.metrics().then(function (metrics) {
+                return res.end(metrics);
+              });
+        }));
+  app.get("/metrics/runtime", (function (_req, res) {
+          res.set("Content-Type", runtimeRegistry.contentType);
+          runtimeRegistry.metrics().then(function (metrics) {
+                return res.end(metrics);
+              });
+        }));
+  var server = app.listen(Env.serverPort);
+  server.on("error", (function (err) {
+          var code = err.code;
+          if (code === "EADDRINUSE") {
+            Logging.error("Port " + String(Env.serverPort) + " is already in use. To fix this either:" + ("\n  1. Kill the process using the port: lsof -ti :" + String(Env.serverPort) + " | xargs kill -9") + "\n  2. Use a different port by setting the ENVIO_INDEXER_PORT environment variable: ENVIO_INDEXER_PORT=9899 envio start");
+          } else {
+            Logging.errorWithExn(err, "Failed to start indexer server");
+          }
+          Process.exit(1);
+        }));
+}
+
+async function start(makeGeneratedConfig, persistence, isTestOpt, exitAfterFirstEventBlockOpt, patchConfig) {
+  var isTest = isTestOpt !== undefined ? isTestOpt : false;
+  var exitAfterFirstEventBlock = exitAfterFirstEventBlockOpt !== undefined ? exitAfterFirstEventBlockOpt : false;
+  var mainArgs = Yargs(Helpers.hideBin(process.argv)).argv;
+  var shouldUseTui = !isTest && !Belt_Option.getWithDefault(mainArgs["tui-off"], Env.tuiOffEnvVar);
+  var isDevelopmentMode = !isTest && Env.Db.password === "testing";
+  var configWithoutRegistrations = makeGeneratedConfig();
+  var registrations = await HandlerLoader.registerAllHandlers(configWithoutRegistrations);
+  var config = makeGeneratedConfig();
+  var config$1 = isTest ? ({
+        name: config.name,
+        description: config.description,
+        handlers: config.handlers,
+        contractHandlers: config.contractHandlers,
+        shouldRollbackOnReorg: false,
+        shouldSaveFullHistory: config.shouldSaveFullHistory,
+        multichain: config.multichain,
+        chainMap: config.chainMap,
+        defaultChain: config.defaultChain,
+        ecosystem: config.ecosystem,
+        enableRawEvents: config.enableRawEvents,
+        maxAddrInPartition: config.maxAddrInPartition,
+        batchSize: config.batchSize,
+        lowercaseAddresses: config.lowercaseAddresses,
+        addContractNameToContractNameMapping: config.addContractNameToContractNameMapping,
+        userEntitiesByName: config.userEntitiesByName,
+        userEntities: config.userEntities,
+        allEntities: config.allEntities,
+        allEnums: config.allEnums
+      }) : config;
+  var config$2 = patchConfig !== undefined ? patchConfig(config$1, registrations) : config$1;
+  var ctx = {
+    registrations: registrations,
+    config: config$2,
+    persistence: persistence
+  };
+  var envioVersion = Utils.EnvioPackage.value.version;
+  Prometheus.Info.set(envioVersion);
+  Prometheus.ProcessStartTimeSeconds.set();
+  Prometheus.RollbackEnabled.set(config$2.shouldRollbackOnReorg);
+  if (!isTest) {
+    startServer((function () {
+            var gsManager = globalGsManagerRef.contents;
+            if (gsManager === undefined) {
+              return {
+                      status: "initializing"
+                    };
+            }
+            var state = GlobalStateManager.getState(Caml_option.valFromOption(gsManager));
+            var chains = Belt_Array.map(ChainMap.values(state.chainManager.chainFetchers), (function (cf) {
+                    var latestFetchedBlockNumber = Caml.int_max(FetchState.bufferBlockNumber(cf.fetchState), 0);
+                    var knownHeight = ChainFetcher.hasProcessedToEndblock(cf) ? Belt_Option.getWithDefault(cf.fetchState.endBlock, cf.fetchState.knownHeight) : cf.fetchState.knownHeight;
+                    return {
+                            chainId: cf.chainConfig.id,
+                            poweredByHyperSync: SourceManager.getActiveSource(cf.sourceManager).poweredByHyperSync,
+                            firstEventBlockNumber: cf.fetchState.firstEventBlock,
+                            latestProcessedBlock: cf.committedProgressBlockNumber === -1 ? undefined : cf.committedProgressBlockNumber,
+                            timestampCaughtUpToHeadOrEndblock: cf.timestampCaughtUpToHeadOrEndblock,
+                            numEventsProcessed: cf.numEventsProcessed,
+                            latestFetchedBlockNumber: latestFetchedBlockNumber,
+                            currentBlockHeight: knownHeight,
+                            numBatchesFetched: 0,
+                            startBlock: cf.fetchState.startBlock,
+                            endBlock: cf.fetchState.endBlock,
+                            numAddresses: FetchState.numAddresses(cf.fetchState)
+                          };
+                  }));
+            var match = config$2.multichain;
+            var tmp;
+            tmp = match === "ordered" ? false : true;
+            return {
+                    status: "active",
+                    envioVersion: envioVersion,
+                    chains: chains,
+                    indexerStartTime: state.indexerStartTime,
+                    isPreRegisteringDynamicContracts: false,
+                    isUnorderedMultichainMode: tmp,
+                    rollbackOnReorg: config$2.shouldRollbackOnReorg
+                  };
+          }), ctx, isDevelopmentMode);
+  }
+  await Persistence.init(persistence, ChainMap.values(config$2.chainMap), undefined);
+  var chainManager = await ChainManager.makeFromDbState(Persistence.getInitializedState(persistence), config$2, registrations);
+  var globalState = GlobalState.make(ctx, chainManager, isDevelopmentMode, shouldUseTui, exitAfterFirstEventBlock);
+  var gsManager = GlobalStateManager.make(globalState, undefined);
+  if (shouldUseTui) {
+    Tui.start(function () {
+          return GlobalStateManager.getState(gsManager);
+        });
+  }
+  globalGsManagerRef.contents = Caml_option.some(gsManager);
+  GlobalStateManager.dispatchTask(gsManager, {
+        TAG: "NextQuery",
+        _0: "CheckAllChains"
+      });
+  return GlobalStateManager.dispatchTask(gsManager, "ProcessEventBatch");
+}
+
+export {
+  chainDataSchema ,
+  stateSchema ,
+  globalGsManagerRef ,
+  getGlobalIndexer ,
+  startServer ,
+  start ,
+}
+/* chainDataSchema Not a pure module */

package/src/Persistence.res

@@ -19,10 +19,11 @@ type initialChainState = {
   endBlock: option<int>,
   maxReorgDepth: int,
   progressBlockNumber: int,
-  numEventsProcessed: int,
+  numEventsProcessed: float,
   firstEventBlockNumber: option<int>,
   timestampCaughtUpToHeadOrEndblock: option<Js.Date.t>,
   dynamicContracts: array<Internal.indexingContract>,
+  sourceBlockNumber: int,
 }
 
 type initialState = {
@@ -74,24 +75,9 @@ type storage = {
     ~table: Table.table,
     ~rowsSchema: S.t<array<'item>>,
   ) => promise<array<'item>>,
-  @raises("StorageError")
-  setOrThrow: 'item. (
-    ~items: array<'item>,
-    ~table: Table.table,
-    ~itemSchema: S.t<'item>,
-  ) => promise<unit>,
-  @raises("StorageError")
-  setEffectCacheOrThrow: (
-    ~effect: Internal.effect,
-    ~items: array<Internal.effectCacheItem>,
-    ~initialize: bool,
-  ) => promise<unit>,
   // This is to download cache from the database to .envio/cache
   dumpEffectCache: unit => promise<unit>,
-  // Execute raw SQL query
-  executeUnsafe: string => promise<unknown>,
-  // Check if entity history has rows
-  hasEntityHistoryRows: unit => promise<bool>,
+  reset: unit => promise<unit>,
   // Update chain metadata
   setChainMeta: dict<InternalTable.Chains.metaFields> => promise<unknown>,
   // Prune old checkpoints
@@ -106,7 +92,7 @@ type storage = {
   getRollbackTargetCheckpoint: (
     ~reorgChainId: int,
     ~lastKnownValidBlockNumber: int,
-  ) => promise<array<{"id": Internal.checkpointId}>>,
+  ) => promise<option<Internal.checkpointId>>,
   // Get rollback progress diff
   getRollbackProgressDiff: (
     ~rollbackTargetCheckpointId: Internal.checkpointId,
@@ -156,7 +142,8 @@ let make = (
   ~allEnums,
   ~storage,
 ) => {
-  let allEntities = userEntities->Js.Array2.concat([InternalTable.DynamicContractRegistry.config])
+  let allEntities =
+    userEntities->Js.Array2.concat([InternalTable.DynamicContractRegistry.entityConfig])
   let allEnums =
     allEnums->Js.Array2.concat([EntityHistory.RowAction.config->Table.fromGenericEnumConfig])
   {
@@ -217,8 +204,7 @@ let init = {
         resolveRef.contents()
       }
     } catch {
-    | exn =>
-      exn->ErrorHandling.mkLogAndRaise(~msg=`EE800: Failed to initialize the indexer storage.`)
+    | exn => exn->ErrorHandling.mkLogAndRaise(~msg=`Failed to initialize the indexer storage.`)
     }
   }
 }

package/src/Persistence.res.mjs

@@ -1,6 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Utils from "./Utils.res.mjs";
+import * as Config from "./Config.res.mjs";
 import * as Js_exn from "rescript/lib/es6/js_exn.js";
 import * as Logging from "./Logging.res.mjs";
 import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
@@ -9,7 +10,6 @@ import * as EntityHistory from "./db/EntityHistory.res.mjs";
 import * as ErrorHandling from "./ErrorHandling.res.mjs";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
 import * as InMemoryTable from "./InMemoryTable.res.mjs";
-import * as InternalTable from "./db/InternalTable.res.mjs";
 import * as Caml_exceptions from "rescript/lib/es6/caml_exceptions.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
 import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
@@ -17,7 +17,7 @@ import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 var StorageError = /* @__PURE__ */Caml_exceptions.create("Persistence.StorageError");
 
 function make(userEntities, allEnums, storage) {
-  var allEntities = userEntities.concat([InternalTable.DynamicContractRegistry.config]);
+  var allEntities = userEntities.concat([Config.DynamicContractRegistry.entityConfig]);
   var allEnums$1 = allEnums.concat([EntityHistory.RowAction.config]);
   return {
           userEntities: userEntities,
@@ -88,7 +88,7 @@ async function init(persistence, chainConfigs, resetOpt) {
   }
   catch (raw_exn){
     var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
-    return ErrorHandling.mkLogAndRaise(undefined, "EE800: Failed to initialize the indexer storage.", exn);
+    return ErrorHandling.mkLogAndRaise(undefined, "Failed to initialize the indexer storage.", exn);
   }
 }
 

package/src/PgStorage.gen.ts

@@ -0,0 +1,10 @@
+/* TypeScript file generated from PgStorage.res by genType. */
+
+/* eslint-disable */
+/* tslint:disable */
+
+import * as PgStorageJS from './PgStorage.res.mjs';
+
+import type {sql as Postgres_sql} from '../src/bindings/Postgres.gen.js';
+
+export const makeClient: () => Postgres_sql = PgStorageJS.makeClient as any;