envio 3.0.0-alpha.0 → 3.0.0-alpha.10

package/src/PgStorage.res

@@ -1,5 +1,30 @@
 let getCacheRowCountFnName = "get_cache_row_count"
 
+// Only needed for some old tests
+// Remove @genType in the future
+@genType
+let makeClient = () => {
+  Postgres.makeSql(
+    ~config={
+      host: Env.Db.host,
+      port: Env.Db.port,
+      username: Env.Db.user,
+      password: Env.Db.password,
+      database: Env.Db.database,
+      ssl: Env.Db.ssl,
+      // TODO: think how we want to pipe these logs to pino.
+      onnotice: ?(
+        Env.userLogLevel == Some(#warn) || Env.userLogLevel == Some(#error)
+          ? None
+          : Some(_str => ())
+      ),
+      transform: {undefined: Null},
+      max: 2,
+      // debug: (~connection, ~query, ~params as _, ~types as _) => Js.log2(connection, query),
+    },
+  )
+}
+
 let makeCreateIndexQuery = (~tableName, ~indexFields, ~pgSchema) => {
   let indexName = tableName ++ "_" ++ indexFields->Js.Array2.joinWith("_")
   let index = indexFields->Belt.Array.map(idx => `"${idx}"`)->Js.Array2.joinWith(", ")
@@ -1056,8 +1081,7 @@ let make = (
   }
 
   let cacheDirPath = NodeJs.Path.resolve([
-    // Right outside of the generated directory
-    "..",
+    // Right at the project root
     ".envio",
     "cache",
   ])
@@ -1223,6 +1247,7 @@ let make = (
         firstEventBlockNumber: None,
         timestampCaughtUpToHeadOrEndblock: None,
         dynamicContracts: [],
+        sourceBlockNumber: 0,
       }),
       checkpointId: InternalTable.Checkpoints.initialCheckpointId,
     }
@@ -1425,6 +1450,7 @@ let make = (
           numEventsProcessed: rawInitialState.numEventsProcessed,
           progressBlockNumber: rawInitialState.progressBlockNumber,
           dynamicContracts: rawInitialState.dynamicContracts,
+          sourceBlockNumber: rawInitialState.sourceBlockNumber,
         })
       }),
       sql
@@ -1454,37 +1480,6 @@ let make = (
 
   let executeUnsafe = query => sql->Postgres.unsafe(query)
 
-  let hasEntityHistoryRows = async () => {
-    // Query for all entity history tables (they have the prefix "envio_history_")
-    let historyTables = await sql->Postgres.unsafe(
-      `SELECT table_name FROM information_schema.tables 
-       WHERE table_schema = '${pgSchema}' 
-       AND table_name LIKE 'envio_history_%';`,
-    )
-
-    if historyTables->Utils.Array.isEmpty {
-      false
-    } else {
-      // Check if any of these tables have rows
-      let checks =
-        await historyTables
-        ->Belt.Array.map(async (table: {"table_name": string}) => {
-          try {
-            let query = `SELECT EXISTS(SELECT 1 FROM "${pgSchema}"."${table["table_name"]}" LIMIT 1);`
-            let result: array<{"exists": bool}> = (await sql->Postgres.unsafe(query))->Utils.magic
-            switch result {
-            | [row] => row["exists"]
-            | _ => false
-            }
-          } catch {
-          | _ => false
-          }
-        })
-        ->Promise.all
-      checks->Belt.Array.some(v => v)
-    }
-  }
-
   let setChainMeta = chainsData =>
     InternalTable.Chains.setMeta(sql, ~pgSchema, ~chainsData)->Promise.thenResolve(_ =>
       %raw(`undefined`)
@@ -1591,7 +1586,6 @@ let make = (
     setEffectCacheOrThrow,
     dumpEffectCache,
     executeUnsafe,
-    hasEntityHistoryRows,
     setChainMeta,
     pruneStaleCheckpoints,
     pruneStaleEntityHistory,

package/src/PgStorage.res.d.mts

@@ -0,0 +1,5 @@
+// This is to prevent tsc --noEmit from failing
+// when importing code from .res.mjs files in genType .ts files
+// After we upgrade GenType and it starts to include ts-ignore,
+// the line can be removed.
+export const makeClient: any;

package/src/PgStorage.res.mjs

@@ -1,6 +1,7 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 
 import * as Fs from "fs";
+import * as Env from "./Env.res.mjs";
 import * as Path from "path";
 import * as $$Array from "rescript/lib/es6/array.js";
 import * as Table from "./db/Table.res.mjs";
@@ -12,7 +13,9 @@ import * as Schema from "./db/Schema.res.mjs";
 import * as Js_dict from "rescript/lib/es6/js_dict.js";
 import * as Logging from "./Logging.res.mjs";
 import * as $$Promise from "./bindings/Promise.res.mjs";
+import * as Caml_obj from "rescript/lib/es6/caml_obj.js";
 import * as Internal from "./Internal.res.mjs";
+import Postgres from "postgres";
 import * as Belt_Array from "rescript/lib/es6/belt_Array.js";
 import * as Prometheus from "./Prometheus.res.mjs";
 import * as Caml_option from "rescript/lib/es6/caml_option.js";
@@ -26,6 +29,24 @@ import * as Caml_js_exceptions from "rescript/lib/es6/caml_js_exceptions.js";
 
 var getCacheRowCountFnName = "get_cache_row_count";
 
+function makeClient() {
+  return Postgres({
+              host: Env.Db.host,
+              port: Env.Db.port,
+              database: Env.Db.database,
+              username: Env.Db.user,
+              password: Env.Db.password,
+              ssl: Env.Db.ssl,
+              max: 2,
+              onnotice: Caml_obj.equal(Env.userLogLevel, "warn") || Caml_obj.equal(Env.userLogLevel, "error") ? undefined : (function (_str) {
+                    
+                  }),
+              transform: {
+                undefined: null
+              }
+            });
+}
+
 function makeCreateIndexQuery(tableName, indexFields, pgSchema) {
   var indexName = tableName + "_" + indexFields.join("_");
   var index = Belt_Array.map(indexFields, (function (idx) {
@@ -697,7 +718,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
   var psqlExecOptions = {
     env: psqlExecOptions_env
   };
-  var cacheDirPath = Path.resolve("..", ".envio", "cache");
+  var cacheDirPath = Path.resolve(".envio", "cache");
   var isInitialized = async function () {
     var envioTables = await sql.unsafe("SELECT table_schema FROM information_schema.tables WHERE table_schema = '" + pgSchema + "' AND (table_name = 'event_sync_state' OR table_name = '" + InternalTable.Chains.table.tableName + "');");
     return Utils.$$Array.notEmpty(envioTables);
@@ -816,7 +837,8 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
                           numEventsProcessed: 0,
                           firstEventBlockNumber: undefined,
                           timestampCaughtUpToHeadOrEndblock: undefined,
-                          dynamicContracts: []
+                          dynamicContracts: [],
+                          sourceBlockNumber: 0
                         };
                 }),
             checkpointId: InternalTable.Checkpoints.initialCheckpointId,
@@ -968,7 +990,8 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
                                       numEventsProcessed: rawInitialState.numEventsProcessed,
                                       firstEventBlockNumber: Caml_option.null_to_opt(rawInitialState.firstEventBlockNumber),
                                       timestampCaughtUpToHeadOrEndblock: Caml_option.null_to_opt(rawInitialState.timestampCaughtUpToHeadOrEndblock),
-                                      dynamicContracts: rawInitialState.dynamicContracts
+                                      dynamicContracts: rawInitialState.dynamicContracts,
+                                      sourceBlockNumber: rawInitialState.sourceBlockNumber
                                     };
                             }));
               }),
@@ -990,29 +1013,6 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
   var executeUnsafe = function (query) {
     return sql.unsafe(query);
   };
-  var hasEntityHistoryRows = async function () {
-    var historyTables = await sql.unsafe("SELECT table_name FROM information_schema.tables \n       WHERE table_schema = '" + pgSchema + "' \n       AND table_name LIKE 'envio_history_%';");
-    if (Utils.$$Array.isEmpty(historyTables)) {
-      return false;
-    }
-    var checks = await Promise.all(Belt_Array.map(historyTables, (async function (table) {
-                try {
-                  var query = "SELECT EXISTS(SELECT 1 FROM \"" + pgSchema + "\".\"" + table.table_name + "\" LIMIT 1);";
-                  var result = await sql.unsafe(query);
-                  if (result.length !== 1) {
-                    return false;
-                  } else {
-                    return result[0].exists;
-                  }
-                }
-                catch (exn){
-                  return false;
-                }
-              })));
-    return Belt_Array.some(checks, (function (v) {
-                  return v;
-                }));
-  };
   var setChainMeta = function (chainsData) {
     return InternalTable.Chains.setMeta(sql, pgSchema, chainsData).then(function (param) {
                 return undefined;
@@ -1060,7 +1060,6 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
           setEffectCacheOrThrow: setEffectCacheOrThrow,
           dumpEffectCache: dumpEffectCache,
           executeUnsafe: executeUnsafe,
-          hasEntityHistoryRows: hasEntityHistoryRows,
           setChainMeta: setChainMeta,
           pruneStaleCheckpoints: pruneStaleCheckpoints,
           pruneStaleEntityHistory: pruneStaleEntityHistory,
@@ -1075,6 +1074,7 @@ var maxItemsPerQuery = 500;
 
 export {
   getCacheRowCountFnName ,
+  makeClient ,
   makeCreateIndexQuery ,
   makeCreateTableIndicesQuery ,
   makeCreateTableQuery ,

package/src/Prometheus.res

@@ -34,12 +34,6 @@ let allChainsSyncedToHead = PromClient.Gauge.makeGauge({
   "labelNames": [],
 })
 
-let sourceChainHeight = PromClient.Gauge.makeGauge({
-  "name": "chain_block_height",
-  "help": "Chain Height of Source Chain",
-  "labelNames": ["chainId"],
-})
-
 module Labels = {
   let rec schemaIsString = (schema: S.t<'a>) =>
     switch schema->S.classify {
@@ -233,8 +227,14 @@ let incrementStorageWriteCounter = () => {
   storageWriteCounter->PromClient.Counter.inc
 }
 
-let setSourceChainHeight = (~blockNumber, ~chainId) => {
-  sourceChainHeight
+let knownHeightGauge = PromClient.Gauge.makeGauge({
+  "name": "envio_indexing_known_height",
+  "help": "The latest known block number reported by the active indexing source. This value may lag behind the actual chain height, as it is updated only when needed.",
+  "labelNames": ["chainId"],
+})
+
+let setKnownHeight = (~blockNumber, ~chainId) => {
+  knownHeightGauge
   ->PromClient.Gauge.labels({"chainId": chainId})
   ->PromClient.Gauge.set(blockNumber)
 }

package/src/Prometheus.res.mjs

@@ -44,12 +44,6 @@ var allChainsSyncedToHead = new PromClient.Gauge({
       labelNames: []
     });
 
-var sourceChainHeight = new PromClient.Gauge({
-      name: "chain_block_height",
-      help: "Chain Height of Source Chain",
-      labelNames: ["chainId"]
-    });
-
 function schemaIsString(_schema) {
   while(true) {
     var schema = _schema;
@@ -333,8 +327,14 @@ function incrementStorageWriteCounter() {
   storageWriteCounter.inc();
 }
 
-function setSourceChainHeight(blockNumber, chainId) {
-  sourceChainHeight.labels({
+var knownHeightGauge = new PromClient.Gauge({
+      name: "envio_indexing_known_height",
+      help: "The latest known block number reported by the active indexing source. This value may lag behind the actual chain height, as it is updated only when needed.",
+      labelNames: ["chainId"]
+    });
+
+function setKnownHeight(blockNumber, chainId) {
+  knownHeightGauge.labels({
           chainId: chainId
         }).set(blockNumber);
 }
@@ -854,7 +854,6 @@ export {
   storageWriteTimeCounter ,
   storageWriteCounter ,
   allChainsSyncedToHead ,
-  sourceChainHeight ,
   Labels ,
   metricNames ,
   MakeSafePromMetric ,
@@ -867,7 +866,8 @@ export {
   incrementExecuteBatchDurationCounter ,
   incrementStorageWriteTimeCounter ,
   incrementStorageWriteCounter ,
-  setSourceChainHeight ,
+  knownHeightGauge ,
+  setKnownHeight ,
   setAllChainsSyncedToHead ,
   BenchmarkCounters ,
   chainIdLabelsSchema ,

package/src/ReorgDetection.res

@@ -77,13 +77,10 @@ let make = (
   }
 }
 
-let getDataByBlockNumberCopyInThreshold = (
-  {dataByBlockNumber, maxReorgDepth}: t,
-  ~currentBlockHeight,
-) => {
+let getDataByBlockNumberCopyInThreshold = ({dataByBlockNumber, maxReorgDepth}: t, ~knownHeight) => {
   // Js engine automatically orders numeric object keys
   let ascBlockNumberKeys = dataByBlockNumber->Js.Dict.keys
-  let thresholdBlockNumber = currentBlockHeight - maxReorgDepth
+  let thresholdBlockNumber = knownHeight - maxReorgDepth
 
   let copy = Js.Dict.empty()
 
@@ -105,10 +102,9 @@ let getDataByBlockNumberCopyInThreshold = (
 let registerReorgGuard = (
   {maxReorgDepth, shouldRollbackOnReorg} as self: t,
   ~reorgGuard: reorgGuard,
-  ~currentBlockHeight,
+  ~knownHeight,
 ) => {
-  let dataByBlockNumberCopyInThreshold =
-    self->getDataByBlockNumberCopyInThreshold(~currentBlockHeight)
+  let dataByBlockNumberCopyInThreshold = self->getDataByBlockNumberCopyInThreshold(~knownHeight)
 
   let {rangeLastBlock, prevRangeLastBlock} = reorgGuard
 
@@ -248,12 +244,12 @@ let rollbackToValidBlockNumber = (
   }
 }
 
-let getThresholdBlockNumbersBelowBlock = (self: t, ~blockNumber: int, ~currentBlockHeight) => {
+let getThresholdBlockNumbersBelowBlock = (self: t, ~blockNumber: int, ~knownHeight) => {
   let arr = []
 
   // Js engine automatically orders numeric object keys
   let ascBlockNumberKeys = self.dataByBlockNumber->Js.Dict.keys
-  let thresholdBlockNumber = currentBlockHeight - self.maxReorgDepth
+  let thresholdBlockNumber = knownHeight - self.maxReorgDepth
 
   for idx in 0 to ascBlockNumberKeys->Array.length - 1 {
     let blockNumberKey = ascBlockNumberKeys->Js.Array2.unsafe_get(idx)

package/src/ReorgDetection.res.mjs

@@ -30,10 +30,10 @@ function make(chainReorgCheckpoints, maxReorgDepth, shouldRollbackOnReorg) {
         };
 }
 
-function getDataByBlockNumberCopyInThreshold(param, currentBlockHeight) {
+function getDataByBlockNumberCopyInThreshold(param, knownHeight) {
   var dataByBlockNumber = param.dataByBlockNumber;
   var ascBlockNumberKeys = Object.keys(dataByBlockNumber);
-  var thresholdBlockNumber = currentBlockHeight - param.maxReorgDepth | 0;
+  var thresholdBlockNumber = knownHeight - param.maxReorgDepth | 0;
   var copy = {};
   for(var idx = 0 ,idx_finish = ascBlockNumberKeys.length; idx < idx_finish; ++idx){
     var blockNumberKey = ascBlockNumberKeys[idx];
@@ -47,10 +47,10 @@ function getDataByBlockNumberCopyInThreshold(param, currentBlockHeight) {
   return copy;
 }
 
-function registerReorgGuard(self, reorgGuard, currentBlockHeight) {
+function registerReorgGuard(self, reorgGuard, knownHeight) {
   var maxReorgDepth = self.maxReorgDepth;
   var shouldRollbackOnReorg = self.shouldRollbackOnReorg;
-  var dataByBlockNumberCopyInThreshold = getDataByBlockNumberCopyInThreshold(self, currentBlockHeight);
+  var dataByBlockNumberCopyInThreshold = getDataByBlockNumberCopyInThreshold(self, knownHeight);
   var prevRangeLastBlock = reorgGuard.prevRangeLastBlock;
   var rangeLastBlock = reorgGuard.rangeLastBlock;
   var scannedBlock = dataByBlockNumberCopyInThreshold[String(rangeLastBlock.blockNumber)];
@@ -159,10 +159,10 @@ function rollbackToValidBlockNumber(param, blockNumber) {
         };
 }
 
-function getThresholdBlockNumbersBelowBlock(self, blockNumber, currentBlockHeight) {
+function getThresholdBlockNumbersBelowBlock(self, blockNumber, knownHeight) {
   var arr = [];
   var ascBlockNumberKeys = Object.keys(self.dataByBlockNumber);
-  var thresholdBlockNumber = currentBlockHeight - self.maxReorgDepth | 0;
+  var thresholdBlockNumber = knownHeight - self.maxReorgDepth | 0;
   for(var idx = 0 ,idx_finish = ascBlockNumberKeys.length; idx < idx_finish; ++idx){
     var blockNumberKey = ascBlockNumberKeys[idx];
     var scannedBlock = self.dataByBlockNumber[blockNumberKey];