envio 2.27.5 → 2.28.0-alpha.1

package/src/PgStorage.res.js

@@ -16,6 +16,7 @@ var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Persistence = require("./Persistence.res.js");
+var InternalTable = require("./db/InternalTable.res.js");
 var Child_process = require("child_process");
 var Caml_exceptions = require("rescript/lib/js/caml_exceptions.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
@@ -68,11 +69,18 @@ function makeCreateTableQuery(table, pgSchema) {
         ) + ");";
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt) {
-  var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
+function makeInitializeTransaction(pgSchema, pgUser, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt) {
+  var chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
   var entities = entitiesOpt !== undefined ? entitiesOpt : [];
   var enums = enumsOpt !== undefined ? enumsOpt : [];
   var isEmptyPgSchema = isEmptyPgSchemaOpt !== undefined ? isEmptyPgSchemaOpt : false;
+  var generalTables = [
+    InternalTable.EventSyncState.table,
+    InternalTable.Chains.table,
+    InternalTable.PersistedState.table,
+    InternalTable.EndOfBlockRangeScannedData.table,
+    InternalTable.RawEvents.table
+  ];
   var allTables = $$Array.copy(generalTables);
   var allEntityTables = [];
   entities.forEach(function (entity) {
@@ -107,12 +115,16 @@ function makeInitializeTransaction(pgSchema, pgUser, generalTablesOpt, entitiesO
     contents: ""
   };
   entities.forEach(function (entity) {
-        functionsQuery.contents = functionsQuery.contents + "\n" + entity.entityHistory.createInsertFnQuery;
+        functionsQuery.contents = functionsQuery.contents + "\n" + entity.entityHistory.makeInsertFnQuery(pgSchema);
         Table.getDerivedFromFields(entity.table).forEach(function (derivedFromField) {
               var indexField = Utils.unwrapResultExn(Schema.getDerivedFromFieldName(derivedSchema, derivedFromField));
               query.contents = query.contents + "\n" + makeCreateIndexQuery(derivedFromField.derivedFromEntity, [indexField], pgSchema);
             });
       });
+  var initialChainsValuesQuery = InternalTable.Chains.makeInitialValuesQuery(pgSchema, chainConfigs);
+  if (initialChainsValuesQuery !== undefined) {
+    query.contents = query.contents + "\n" + initialChainsValuesQuery;
+  }
   functionsQuery.contents = functionsQuery.contents + "\n" + ("CREATE OR REPLACE FUNCTION " + getCacheRowCountFnName + "(table_name text) \nRETURNS integer AS $$\nDECLARE\n  result integer;\nBEGIN\n  EXECUTE format('SELECT COUNT(*) FROM \"" + pgSchema + "\".%I', table_name) INTO result;\n  RETURN result;\nEND;\n$$ LANGUAGE plpgsql;");
   return [query.contents].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
 }
@@ -129,6 +141,10 @@ function makeLoadByIdsQuery(pgSchema, tableName) {
   return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = ANY($1::text[]);";
 }
 
+function makeLoadAllQuery(pgSchema, tableName) {
+  return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\";";
+}
+
 function makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents) {
   var match = Table.toSqlParams(table, itemSchema, pgSchema);
   var quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
@@ -177,13 +193,9 @@ function makeInsertValuesSetQuery(pgSchema, table, itemSchema, itemsCount) {
         ) + ";";
 }
 
-var rawEventsTableName = "raw_events";
-
-var eventSyncStateTableName = "event_sync_state";
-
 function makeTableBatchSetQuery(pgSchema, table, itemSchema) {
   var match = Table.toSqlParams(table, itemSchema, pgSchema);
-  var isRawEvents = table.tableName === rawEventsTableName;
+  var isRawEvents = table.tableName === InternalTable.RawEvents.table.tableName;
   if (isRawEvents || !match.hasArrayField) {
     return {
             query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents),
@@ -277,7 +289,7 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
     throw {
           RE_EXN_ID: Persistence.StorageError,
           message: "Failed to insert items into table \"" + table.tableName + "\"",
-          reason: Internal.prettifyExn(exn),
+          reason: Utils.prettifyExn(exn),
           Error: new Error()
         };
   }
@@ -285,13 +297,24 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
 
 function setEntityHistoryOrThrow(sql, entityHistory, rows, shouldCopyCurrentEntity, shouldRemoveInvalidUtf8Opt) {
   var shouldRemoveInvalidUtf8 = shouldRemoveInvalidUtf8Opt !== undefined ? shouldRemoveInvalidUtf8Opt : false;
-  return Promise.all(Belt_Array.map(rows, (function (historyRow) {
-                    var row = S$RescriptSchema.reverseConvertToJsonOrThrow(historyRow, entityHistory.schema);
-                    if (shouldRemoveInvalidUtf8) {
-                      removeInvalidUtf8InPlace([row]);
-                    }
-                    return entityHistory.insertFn(sql, row, shouldCopyCurrentEntity !== undefined ? shouldCopyCurrentEntity : !Belt_Option.getWithDefault(historyRow.containsRollbackDiffChange, false));
-                  })));
+  return Belt_Array.map(rows, (function (historyRow) {
+                var row = S$RescriptSchema.reverseConvertToJsonOrThrow(historyRow, entityHistory.schema);
+                if (shouldRemoveInvalidUtf8) {
+                  removeInvalidUtf8InPlace([row]);
+                }
+                return $$Promise.$$catch(entityHistory.insertFn(sql, row, shouldCopyCurrentEntity !== undefined ? shouldCopyCurrentEntity : !Belt_Option.getWithDefault(historyRow.containsRollbackDiffChange, false)), (function (exn) {
+                              var reason = Utils.prettifyExn(exn);
+                              var detail = (reason?.detail || "");
+                              throw {
+                                    RE_EXN_ID: Persistence.StorageError,
+                                    message: "Failed to insert history item into table \"" + entityHistory.table.tableName + "\"." + (
+                                      detail !== "" ? " Details: " + detail : ""
+                                    ),
+                                    reason: reason,
+                                    Error: new Error()
+                                  };
+                            }));
+              }));
 }
 
 function makeSchemaTableNamesQuery(pgSchema) {
@@ -370,29 +393,112 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, onI
   };
   var cacheDirPath = Path.resolve("..", ".envio", "cache");
   var isInitialized = async function () {
-    var envioTables = await sql.unsafe("SELECT table_schema FROM information_schema.tables WHERE table_schema = '" + pgSchema + "' AND table_name = '" + eventSyncStateTableName + "';");
+    var envioTables = await sql.unsafe("SELECT table_schema FROM information_schema.tables WHERE table_schema = '" + pgSchema + "' AND table_name = '" + InternalTable.EventSyncState.table.tableName + "' OR table_name = '" + InternalTable.Chains.table.tableName + "';");
     return Utils.$$Array.notEmpty(envioTables);
   };
-  var initialize = async function (entitiesOpt, generalTablesOpt, enumsOpt) {
+  var restoreEffectCache = async function (withUpload) {
+    if (withUpload) {
+      var nothingToUploadErrorMessage = "Nothing to upload.";
+      var match = await Promise.all([
+            $$Promise.$$catch(Fs.promises.readdir(cacheDirPath).then(function (e) {
+                      return {
+                              TAG: "Ok",
+                              _0: e
+                            };
+                    }), (function (param) {
+                    return Promise.resolve({
+                                TAG: "Error",
+                                _0: nothingToUploadErrorMessage
+                              });
+                  })),
+            getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort)
+          ]);
+      var exit = 0;
+      var message;
+      var entries = match[0];
+      if (entries.TAG === "Ok") {
+        var psqlExec = match[1];
+        if (psqlExec.TAG === "Ok") {
+          var psqlExec$1 = psqlExec._0;
+          var cacheFiles = entries._0.filter(function (entry) {
+                return entry.endsWith(".tsv");
+              });
+          await Promise.all(cacheFiles.map(function (entry) {
+                    var effectName = entry.slice(0, -4);
+                    var table = Internal.makeCacheTable(effectName);
+                    return sql.unsafe(makeCreateTableQuery(table, pgSchema)).then(function () {
+                                var inputFile = Path.join(cacheDirPath, entry);
+                                var command = psqlExec$1 + " -c 'COPY \"" + pgSchema + "\".\"" + table.tableName + "\" FROM STDIN WITH (FORMAT text, HEADER);' < " + inputFile;
+                                return new Promise((function (resolve, reject) {
+                                              Child_process.exec(command, psqlExecOptions, (function (error, stdout, param) {
+                                                      if (error === null) {
+                                                        return resolve(stdout);
+                                                      } else {
+                                                        return reject(error);
+                                                      }
+                                                    }));
+                                            }));
+                              });
+                  }));
+          Logging.info("Successfully uploaded cache.");
+        } else {
+          message = match[1]._0;
+          exit = 1;
+        }
+      } else {
+        message = entries._0;
+        exit = 1;
+      }
+      if (exit === 1) {
+        if (message === nothingToUploadErrorMessage) {
+          Logging.info("No cache found to upload.");
+        } else {
+          Logging.error("Failed to upload cache, continuing without it. " + message);
+        }
+      }
+      
+    }
+    var cacheTableInfo = await sql.unsafe(makeSchemaCacheTableInfoQuery(pgSchema));
+    if (withUpload && Utils.$$Array.notEmpty(cacheTableInfo) && onNewTables !== undefined) {
+      await onNewTables(cacheTableInfo.map(function (info) {
+                return info.table_name;
+              }));
+    }
+    var cache = {};
+    cacheTableInfo.forEach(function (param) {
+          var effectName = param.table_name.slice(cacheTablePrefixLength);
+          cache[effectName] = {
+            effectName: effectName,
+            count: param.count
+          };
+        });
+    return cache;
+  };
+  var initialize = async function (chainConfigsOpt, entitiesOpt, enumsOpt) {
+    var chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
     var entities = entitiesOpt !== undefined ? entitiesOpt : [];
-    var generalTables = generalTablesOpt !== undefined ? generalTablesOpt : [];
     var enums = enumsOpt !== undefined ? enumsOpt : [];
     var schemaTableNames = await sql.unsafe(makeSchemaTableNamesQuery(pgSchema));
     if (Utils.$$Array.notEmpty(schemaTableNames) && !schemaTableNames.some(function (table) {
-            return table.table_name === eventSyncStateTableName;
+            return table.table_name === InternalTable.EventSyncState.table.tableName;
           })) {
       Js_exn.raiseError("Cannot run Envio migrations on PostgreSQL schema \"" + pgSchema + "\" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: \"pnpm envio local db-migrate down\"\n2. Or specify a different schema name by setting the \"ENVIO_PG_PUBLIC_SCHEMA\" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.");
     }
-    var queries = makeInitializeTransaction(pgSchema, pgUser, generalTables, entities, enums, Utils.$$Array.isEmpty(schemaTableNames));
+    var queries = makeInitializeTransaction(pgSchema, pgUser, chainConfigs, entities, enums, Utils.$$Array.isEmpty(schemaTableNames));
     await sql.begin(function (sql) {
           return queries.map(function (query) {
                       return sql.unsafe(query);
                     });
         });
+    var cache = await restoreEffectCache(true);
     if (onInitialize !== undefined) {
-      return await onInitialize();
+      await onInitialize();
     }
-    
+    return {
+            cleanRun: true,
+            cache: cache,
+            chains: chainConfigs.map(InternalTable.Chains.initialFromConfig)
+          };
   };
   var loadByIdsOrThrow = async function (ids, table, rowsSchema) {
     var rows;
@@ -523,93 +629,29 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, onI
     }
     catch (raw_exn){
       var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn);
-      return Logging.errorWithExn(Internal.prettifyExn(exn$1), "Failed to dump cache.");
+      return Logging.errorWithExn(Utils.prettifyExn(exn$1), "Failed to dump cache.");
     }
   };
-  var restoreEffectCache = async function (withUpload) {
-    if (withUpload) {
-      var nothingToUploadErrorMessage = "Nothing to upload.";
-      var match = await Promise.all([
-            $$Promise.$$catch(Fs.promises.readdir(cacheDirPath).then(function (e) {
-                      return {
-                              TAG: "Ok",
-                              _0: e
-                            };
-                    }), (function (param) {
-                    return Promise.resolve({
-                                TAG: "Error",
-                                _0: nothingToUploadErrorMessage
-                              });
-                  })),
-            getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort)
-          ]);
-      var exit = 0;
-      var message;
-      var entries = match[0];
-      if (entries.TAG === "Ok") {
-        var psqlExec = match[1];
-        if (psqlExec.TAG === "Ok") {
-          var psqlExec$1 = psqlExec._0;
-          var cacheFiles = entries._0.filter(function (entry) {
-                return entry.endsWith(".tsv");
-              });
-          await Promise.all(cacheFiles.map(function (entry) {
-                    var effectName = entry.slice(0, -4);
-                    var table = Internal.makeCacheTable(effectName);
-                    return sql.unsafe(makeCreateTableQuery(table, pgSchema)).then(function () {
-                                var inputFile = Path.join(cacheDirPath, entry);
-                                var command = psqlExec$1 + " -c 'COPY \"" + pgSchema + "\".\"" + table.tableName + "\" FROM STDIN WITH (FORMAT text, HEADER);' < " + inputFile;
-                                return new Promise((function (resolve, reject) {
-                                              Child_process.exec(command, psqlExecOptions, (function (error, stdout, param) {
-                                                      if (error === null) {
-                                                        return resolve(stdout);
-                                                      } else {
-                                                        return reject(error);
-                                                      }
-                                                    }));
-                                            }));
-                              });
-                  }));
-          Logging.info("Successfully uploaded cache.");
-        } else {
-          message = match[1]._0;
-          exit = 1;
-        }
-      } else {
-        message = entries._0;
-        exit = 1;
-      }
-      if (exit === 1) {
-        if (message === nothingToUploadErrorMessage) {
-          Logging.info("No cache found to upload.");
-        } else {
-          Logging.error("Failed to upload cache, continuing without it. " + message);
-        }
-      }
-      
-    }
-    var cacheTableInfo = await sql.unsafe(makeSchemaCacheTableInfoQuery(pgSchema));
-    if (withUpload && Utils.$$Array.notEmpty(cacheTableInfo) && onNewTables !== undefined) {
-      await onNewTables(cacheTableInfo.map(function (info) {
-                return info.table_name;
-              }));
-    }
-    return cacheTableInfo.map(function (info) {
-                return {
-                        effectName: info.table_name.slice(cacheTablePrefixLength),
-                        count: info.count
-                      };
-              });
+  var loadInitialState = async function () {
+    var match = await Promise.all([
+          restoreEffectCache(false),
+          sql.unsafe(makeLoadAllQuery(pgSchema, InternalTable.Chains.table.tableName))
+        ]);
+    return {
+            cleanRun: false,
+            cache: match[0],
+            chains: match[1]
+          };
   };
   return {
           isInitialized: isInitialized,
           initialize: initialize,
+          loadInitialState: loadInitialState,
           loadByIdsOrThrow: loadByIdsOrThrow,
           loadByFieldOrThrow: loadByFieldOrThrow,
           setOrThrow: setOrThrow$1,
           setEffectCacheOrThrow: setEffectCacheOrThrow,
-          dumpEffectCache: dumpEffectCache,
-          restoreEffectCache: restoreEffectCache
+          dumpEffectCache: dumpEffectCache
         };
 }
 
@@ -623,10 +665,9 @@ exports.makeInitializeTransaction = makeInitializeTransaction;
 exports.makeLoadByIdQuery = makeLoadByIdQuery;
 exports.makeLoadByFieldQuery = makeLoadByFieldQuery;
 exports.makeLoadByIdsQuery = makeLoadByIdsQuery;
+exports.makeLoadAllQuery = makeLoadAllQuery;
 exports.makeInsertUnnestSetQuery = makeInsertUnnestSetQuery;
 exports.makeInsertValuesSetQuery = makeInsertValuesSetQuery;
-exports.rawEventsTableName = rawEventsTableName;
-exports.eventSyncStateTableName = eventSyncStateTableName;
 exports.maxItemsPerQuery = maxItemsPerQuery;
 exports.makeTableBatchSetQuery = makeTableBatchSetQuery;
 exports.chunkArray = chunkArray;

package/src/Prometheus.res

@@ -213,9 +213,9 @@ let incrementExecuteBatchDurationCounter = (~duration) => {
   executeBatchDurationCounter->PromClient.Counter.incMany(duration)
 }
 
-let setSourceChainHeight = (~blockNumber, ~chain) => {
+let setSourceChainHeight = (~blockNumber, ~chainId) => {
   sourceChainHeight
-  ->PromClient.Gauge.labels({"chainId": chain->ChainMap.Chain.toString})
+  ->PromClient.Gauge.labels({"chainId": chainId})
   ->PromClient.Gauge.set(blockNumber)
 }
 

package/src/Prometheus.res.js

@@ -4,7 +4,6 @@
 var Utils = require("./Utils.res.js");
 var Hrtime = require("./bindings/Hrtime.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
-var ChainMap = require("./ChainMap.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var Belt_Result = require("rescript/lib/js/belt_Result.js");
 var PromClient = require("prom-client");
@@ -315,9 +314,9 @@ function incrementExecuteBatchDurationCounter(duration) {
   executeBatchDurationCounter.inc(duration);
 }
 
-function setSourceChainHeight(blockNumber, chain) {
+function setSourceChainHeight(blockNumber, chainId) {
   sourceChainHeight.labels({
-          chainId: ChainMap.Chain.toString(chain)
+          chainId: chainId
         }).set(blockNumber);
 }
 

package/src/Time.res

@@ -17,7 +17,7 @@ let rec retryAsyncWithExponentialBackOff = async (
       let log = retryCount === 0 ? Logging.childTrace : Logging.childWarn
       logger->log({
         "msg": `Retrying query ${nextRetryCount->Belt.Int.toString}/${maxRetries->Belt.Int.toString} in ${backOffMillis->Belt.Int.toString}ms - waiting for correct result.`,
-        "err": exn->Internal.prettifyExn,
+        "err": exn->Utils.prettifyExn,
       })
       await resolvePromiseAfterDelay(~delayMilliseconds=backOffMillis)
 

package/src/Time.res.js

@@ -3,7 +3,6 @@
 
 var Utils = require("./Utils.res.js");
 var Logging = require("./Logging.res.js");
-var Internal = require("./Internal.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
@@ -26,7 +25,7 @@ async function retryAsyncWithExponentialBackOff(backOffMillisOpt, multiplicative
       var log = retryCount === 0 ? Logging.childTrace : Logging.childWarn;
       log(logger, {
             msg: "Retrying query " + String(nextRetryCount) + "/" + String(maxRetries) + " in " + String(backOffMillis) + "ms - waiting for correct result.",
-            err: Internal.prettifyExn(exn)
+            err: Utils.prettifyExn(exn)
           });
       await Utils.delay(backOffMillis);
       return await retryAsyncWithExponentialBackOff(Math.imul(backOffMillis, multiplicative), multiplicative, nextRetryCount, maxRetries, logger, f);

package/src/Utils.res

@@ -584,3 +584,10 @@ module Hash = {
     }
   }
 }
+
+let prettifyExn = exn => {
+  switch exn->Js.Exn.anyToExnInternal {
+  | Js.Exn.Error(e) => e->(magic: Js.Exn.t => exn)
+  | exn => exn
+  }
+}

package/src/Utils.res.js

@@ -10,6 +10,7 @@ var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Caml_splice_call = require("rescript/lib/js/caml_splice_call.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
+var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
 function delay(milliseconds) {
   return new Promise((function (resolve, param) {
@@ -530,6 +531,15 @@ var Hash = {
   makeOrThrow: makeOrThrow
 };
 
+function prettifyExn(exn) {
+  var e = Caml_js_exceptions.internalToOCamlException(exn);
+  if (e.RE_EXN_ID === Js_exn.$$Error) {
+    return e._1;
+  } else {
+    return e;
+  }
+}
+
 exports.delay = delay;
 exports.$$Object = $$Object;
 exports.$$Error = $$Error;
@@ -548,4 +558,5 @@ exports.$$WeakMap = $$WeakMap;
 exports.$$Map = $$Map;
 exports.$$Proxy = $$Proxy;
 exports.Hash = Hash;
+exports.prettifyExn = prettifyExn;
 /* variantTag Not a pure module */

package/src/bindings/Pino.res

@@ -46,7 +46,7 @@ let createPinoMessageWithError = (message, err): pinoMessageBlobWithError => {
   // serializers type below: `type serializers = {err: Js.Json.t => Js.Json.t}`
   Utils.magic({
     "msg": message,
-    "err": err,
+    "err": err->Utils.prettifyExn,
   })
 }
 

package/src/bindings/Pino.res.js

@@ -2,6 +2,7 @@
 'use strict';
 
 var Pino = require("pino");
+var Utils = require("../Utils.res.js");
 var Caml_obj = require("rescript/lib/js/caml_obj.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var Belt_Option = require("rescript/lib/js/belt_Option.js");
@@ -14,7 +15,7 @@ function createPinoMessage(message) {
 function createPinoMessageWithError(message, err) {
   return {
           msg: message,
-          err: err
+          err: Utils.prettifyExn(err)
         };
 }