envio 2.31.0-alpha.0 → 2.31.0-alpha.2

package/src/PgStorage.res

@@ -65,7 +65,7 @@ let makeInitializeTransaction = (
   let generalTables = [
     InternalTable.Chains.table,
     InternalTable.PersistedState.table,
-    InternalTable.EndOfBlockRangeScannedData.table,
+    InternalTable.Checkpoints.table,
     InternalTable.RawEvents.table,
   ]
 
@@ -116,13 +116,8 @@ GRANT ALL ON SCHEMA "${pgSchema}" TO public;`,
     }
   })
 
-  let functionsQuery = ref("")
-
   // Add derived indices
   entities->Js.Array2.forEach((entity: Internal.entityConfig) => {
-    functionsQuery :=
-      functionsQuery.contents ++ "\n" ++ entity.entityHistory.makeInsertFnQuery(~pgSchema)
-
     entity.table
     ->Table.getDerivedFromFields
     ->Js.Array2.forEach(derivedFromField => {
@@ -149,10 +144,8 @@ GRANT ALL ON SCHEMA "${pgSchema}" TO public;`,
   | None => ()
   }
 
-  // Add cache row count function
-  functionsQuery :=
-    functionsQuery.contents ++
-    "\n" ++
+  [
+    query.contents,
     `CREATE OR REPLACE FUNCTION ${getCacheRowCountFnName}(table_name text) 
 RETURNS integer AS $$
 DECLARE
@@ -161,11 +154,8 @@ BEGIN
   EXECUTE format('SELECT COUNT(*) FROM "${pgSchema}".%I', table_name) INTO result;
   RETURN result;
 END;
-$$ LANGUAGE plpgsql;`
-
-  [query.contents]->Js.Array2.concat(
-    functionsQuery.contents !== "" ? [functionsQuery.contents] : [],
-  )
+$$ LANGUAGE plpgsql;`,
+  ]
 }
 
 let makeLoadByIdQuery = (~pgSchema, ~tableName) => {
@@ -270,6 +260,11 @@ let makeTableBatchSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema: S.t<'
   // FIXME what about Fuel params?
   let isRawEvents = table.tableName === InternalTable.RawEvents.table.tableName
 
+  // Currently history update table uses S.object with transformation for schema,
+  // which is being lossed during conversion to dbSchema.
+  // So use simple insert values for now.
+  let isHistoryUpdate = table.tableName->Js.String2.startsWith("envio_history_")
+
   // Should experiment how much it'll affect performance
   // Although, it should be fine not to perform the validation check,
   // since the values are validated by type system.
@@ -277,7 +272,7 @@ let makeTableBatchSetQuery = (~pgSchema, ~table: Table.table, ~itemSchema: S.t<'
   // db write fails to show a better user error.
   let typeValidation = false
 
-  if isRawEvents || !hasArrayField {
+  if (isRawEvents || !hasArrayField) && !isHistoryUpdate {
     {
       "query": makeInsertUnnestSetQuery(~pgSchema, ~table, ~itemSchema, ~isRawEvents),
       "convertOrThrow": S.compile(
@@ -325,7 +320,7 @@ let chunkArray = (arr: array<'a>, ~chunkSize) => {
 let removeInvalidUtf8InPlace = entities =>
   entities->Js.Array2.forEach(item => {
     let dict = item->(Utils.magic: 'a => dict<unknown>)
-    dict->Utils.Dict.forEachWithKey((key, value) => {
+    dict->Utils.Dict.forEachWithKey((value, key) => {
       if value->Js.typeof === "string" {
         let value = value->(Utils.magic: unknown => string)
         // We mutate here, since we don't care
@@ -414,44 +409,6 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
   }
 }
 
-let setEntityHistoryOrThrow = (
-  sql,
-  ~entityHistory: EntityHistory.t<'entity>,
-  ~rows: array<EntityHistory.historyRow<'entity>>,
-  ~shouldCopyCurrentEntity=?,
-  ~shouldRemoveInvalidUtf8=false,
-) => {
-  rows->Belt.Array.map(historyRow => {
-    let row = historyRow->S.reverseConvertToJsonOrThrow(entityHistory.schema)
-    if shouldRemoveInvalidUtf8 {
-      [row]->removeInvalidUtf8InPlace
-    }
-    entityHistory.insertFn(
-      sql,
-      row,
-      ~shouldCopyCurrentEntity=switch shouldCopyCurrentEntity {
-      | Some(v) => v
-      | None => {
-          let containsRollbackDiffChange =
-            historyRow.containsRollbackDiffChange->Belt.Option.getWithDefault(false)
-          !containsRollbackDiffChange
-        }
-      },
-    )->Promise.catch(exn => {
-      let reason = exn->Utils.prettifyExn
-      let detail = %raw(`reason?.detail || ""`)
-      raise(
-        Persistence.StorageError({
-          message: `Failed to insert history item into table "${entityHistory.table.tableName}".${detail !== ""
-              ? ` Details: ${detail}`
-              : ""}`,
-          reason,
-        }),
-      )
-    })
-  })
-}
-
 type schemaTableName = {
   @as("table_name")
   tableName: string,
@@ -707,7 +664,19 @@ let make = (
     {
       cleanRun: true,
       cache,
-      chains: chainConfigs->Js.Array2.map(InternalTable.Chains.initialFromConfig),
+      reorgCheckpoints: [],
+      chains: chainConfigs->Js.Array2.map((chainConfig): Persistence.initialChainState => {
+        id: chainConfig.id,
+        startBlock: chainConfig.startBlock,
+        endBlock: chainConfig.endBlock,
+        maxReorgDepth: chainConfig.maxReorgDepth,
+        progressBlockNumber: -1,
+        numEventsProcessed: 0,
+        firstEventBlockNumber: None,
+        timestampCaughtUpToHeadOrEndblock: None,
+        dynamicContracts: [],
+      }),
+      checkpointId: InternalTable.Checkpoints.initialCheckpointId,
     }
   }
 
@@ -895,19 +864,38 @@ let make = (
   }
 
   let resumeInitialState = async (): Persistence.initialState => {
-    let (cache, chains) = await Promise.all2((
+    let (cache, chains, checkpointIdResult, reorgCheckpoints) = await Promise.all4((
       restoreEffectCache(~withUpload=false),
+      InternalTable.Chains.getInitialState(
+        sql,
+        ~pgSchema,
+      )->Promise.thenResolve(rawInitialStates => {
+        rawInitialStates->Belt.Array.map((rawInitialState): Persistence.initialChainState => {
+          id: rawInitialState.id,
+          startBlock: rawInitialState.startBlock,
+          endBlock: rawInitialState.endBlock->Js.Null.toOption,
+          maxReorgDepth: rawInitialState.maxReorgDepth,
+          firstEventBlockNumber: rawInitialState.firstEventBlockNumber->Js.Null.toOption,
+          timestampCaughtUpToHeadOrEndblock: rawInitialState.timestampCaughtUpToHeadOrEndblock->Js.Null.toOption,
+          numEventsProcessed: rawInitialState.numEventsProcessed,
+          progressBlockNumber: rawInitialState.progressBlockNumber,
+          dynamicContracts: rawInitialState.dynamicContracts,
+        })
+      }),
       sql
-      ->Postgres.unsafe(
-        makeLoadAllQuery(~pgSchema, ~tableName=InternalTable.Chains.table.tableName),
-      )
-      ->(Utils.magic: promise<array<unknown>> => promise<array<InternalTable.Chains.t>>),
+      ->Postgres.unsafe(InternalTable.Checkpoints.makeCommitedCheckpointIdQuery(~pgSchema))
+      ->(Utils.magic: promise<array<unknown>> => promise<array<{"id": int}>>),
+      sql
+      ->Postgres.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(~pgSchema))
+      ->(Utils.magic: promise<array<unknown>> => promise<array<Internal.reorgCheckpoint>>),
     ))
 
     {
       cleanRun: false,
+      reorgCheckpoints,
       cache,
       chains,
+      checkpointId: (checkpointIdResult->Belt.Array.getUnsafe(0))["id"],
     }
   }
 

package/src/PgStorage.res.js

@@ -13,7 +13,6 @@ var Logging = require("./Logging.res.js");
 var $$Promise = require("./bindings/Promise.res.js");
 var Internal = require("./Internal.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
-var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Persistence = require("./Persistence.res.js");
 var InternalTable = require("./db/InternalTable.res.js");
@@ -51,7 +50,7 @@ function makeCreateTableQuery(table, pgSchema) {
             var fieldType = field.fieldType;
             var fieldName = Table.getDbFieldName(field);
             var tmp;
-            tmp = fieldType === "TIMESTAMP" || fieldType === "TIMESTAMP WITH TIME ZONE" || fieldType === "JSONB" || fieldType === "SERIAL" || fieldType === "TEXT" || fieldType === "DOUBLE PRECISION" || fieldType === "NUMERIC" || fieldType === "BOOLEAN" || fieldType === "INTEGER" || fieldType === "TIMESTAMP WITH TIME ZONE NULL" || fieldType.startsWith("NUMERIC(") ? fieldType : "\"" + pgSchema + "\"." + fieldType;
+            tmp = fieldType === "TIMESTAMP" || fieldType === "TIMESTAMP WITH TIME ZONE" || fieldType === "JSONB" || fieldType === "SERIAL" || fieldType === "TEXT" || fieldType === "DOUBLE PRECISION" || fieldType === "NUMERIC" || fieldType === "BOOLEAN" || fieldType === "BIGINT" || fieldType === "INTEGER" || fieldType === "TIMESTAMP WITH TIME ZONE NULL" || fieldType.startsWith("NUMERIC(") ? fieldType : "\"" + pgSchema + "\"." + fieldType;
             return "\"" + fieldName + "\" " + tmp + (
                     field.isArray ? "[]" : ""
                   ) + (
@@ -77,7 +76,7 @@ function makeInitializeTransaction(pgSchema, pgUser, chainConfigsOpt, entitiesOp
   var generalTables = [
     InternalTable.Chains.table,
     InternalTable.PersistedState.table,
-    InternalTable.EndOfBlockRangeScannedData.table,
+    InternalTable.Checkpoints.table,
     InternalTable.RawEvents.table
   ];
   var allTables = $$Array.copy(generalTables);
@@ -110,11 +109,7 @@ function makeInitializeTransaction(pgSchema, pgUser, chainConfigsOpt, entitiesOp
         }
         
       });
-  var functionsQuery = {
-    contents: ""
-  };
   entities.forEach(function (entity) {
-        functionsQuery.contents = functionsQuery.contents + "\n" + entity.entityHistory.makeInsertFnQuery(pgSchema);
         Table.getDerivedFromFields(entity.table).forEach(function (derivedFromField) {
               var indexField = Utils.unwrapResultExn(Schema.getDerivedFromFieldName(derivedSchema, derivedFromField));
               query.contents = query.contents + "\n" + makeCreateIndexQuery(derivedFromField.derivedFromEntity, [indexField], pgSchema);
@@ -126,8 +121,10 @@ function makeInitializeTransaction(pgSchema, pgUser, chainConfigsOpt, entitiesOp
   if (initialChainsValuesQuery !== undefined) {
     query.contents = query.contents + "\n" + initialChainsValuesQuery;
   }
-  functionsQuery.contents = functionsQuery.contents + "\n" + ("CREATE OR REPLACE FUNCTION " + getCacheRowCountFnName + "(table_name text) \nRETURNS integer AS $$\nDECLARE\n  result integer;\nBEGIN\n  EXECUTE format('SELECT COUNT(*) FROM \"" + pgSchema + "\".%I', table_name) INTO result;\n  RETURN result;\nEND;\n$$ LANGUAGE plpgsql;");
-  return [query.contents].concat(functionsQuery.contents !== "" ? [functionsQuery.contents] : []);
+  return [
+          query.contents,
+          "CREATE OR REPLACE FUNCTION " + getCacheRowCountFnName + "(table_name text) \nRETURNS integer AS $$\nDECLARE\n  result integer;\nBEGIN\n  EXECUTE format('SELECT COUNT(*) FROM \"" + pgSchema + "\".%I', table_name) INTO result;\n  RETURN result;\nEND;\n$$ LANGUAGE plpgsql;"
+        ];
 }
 
 function makeLoadByIdQuery(pgSchema, tableName) {
@@ -197,7 +194,8 @@ function makeInsertValuesSetQuery(pgSchema, table, itemSchema, itemsCount) {
 function makeTableBatchSetQuery(pgSchema, table, itemSchema) {
   var match = Table.toSqlParams(table, itemSchema, pgSchema);
   var isRawEvents = table.tableName === InternalTable.RawEvents.table.tableName;
-  if (isRawEvents || !match.hasArrayField) {
+  var isHistoryUpdate = table.tableName.startsWith("envio_history_");
+  if ((isRawEvents || !match.hasArrayField) && !isHistoryUpdate) {
     return {
             query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents),
             convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.unnest(match.dbSchema), "Output", "Input", "Sync", false),
@@ -231,7 +229,7 @@ function chunkArray(arr, chunkSize) {
 
 function removeInvalidUtf8InPlace(entities) {
   entities.forEach(function (item) {
-        Utils.Dict.forEachWithKey(item, (function (key, value) {
+        Utils.Dict.forEachWithKey(item, (function (value, key) {
                 if (typeof value === "string") {
                   item[key] = value.replaceAll("\x00", "");
                   return ;
@@ -296,28 +294,6 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
   }
 }
 
-function setEntityHistoryOrThrow(sql, entityHistory, rows, shouldCopyCurrentEntity, shouldRemoveInvalidUtf8Opt) {
-  var shouldRemoveInvalidUtf8 = shouldRemoveInvalidUtf8Opt !== undefined ? shouldRemoveInvalidUtf8Opt : false;
-  return Belt_Array.map(rows, (function (historyRow) {
-                var row = S$RescriptSchema.reverseConvertToJsonOrThrow(historyRow, entityHistory.schema);
-                if (shouldRemoveInvalidUtf8) {
-                  removeInvalidUtf8InPlace([row]);
-                }
-                return $$Promise.$$catch(entityHistory.insertFn(sql, row, shouldCopyCurrentEntity !== undefined ? shouldCopyCurrentEntity : !Belt_Option.getWithDefault(historyRow.containsRollbackDiffChange, false)), (function (exn) {
-                              var reason = Utils.prettifyExn(exn);
-                              var detail = (reason?.detail || "");
-                              throw {
-                                    RE_EXN_ID: Persistence.StorageError,
-                                    message: "Failed to insert history item into table \"" + entityHistory.table.tableName + "\"." + (
-                                      detail !== "" ? " Details: " + detail : ""
-                                    ),
-                                    reason: reason,
-                                    Error: new Error()
-                                  };
-                            }));
-              }));
-}
-
 function makeSchemaTableNamesQuery(pgSchema) {
   return "SELECT table_name FROM information_schema.tables WHERE table_schema = '" + pgSchema + "';";
 }
@@ -498,7 +474,21 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, onI
     return {
             cleanRun: true,
             cache: cache,
-            chains: chainConfigs.map(InternalTable.Chains.initialFromConfig)
+            chains: chainConfigs.map(function (chainConfig) {
+                  return {
+                          id: chainConfig.id,
+                          startBlock: chainConfig.startBlock,
+                          endBlock: chainConfig.endBlock,
+                          maxReorgDepth: chainConfig.maxReorgDepth,
+                          progressBlockNumber: -1,
+                          numEventsProcessed: 0,
+                          firstEventBlockNumber: undefined,
+                          timestampCaughtUpToHeadOrEndblock: undefined,
+                          dynamicContracts: []
+                        };
+                }),
+            checkpointId: InternalTable.Checkpoints.initialCheckpointId,
+            reorgCheckpoints: []
           };
   };
   var loadByIdsOrThrow = async function (ids, table, rowsSchema) {
@@ -636,12 +626,30 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, onI
   var resumeInitialState = async function () {
     var match = await Promise.all([
           restoreEffectCache(false),
-          sql.unsafe(makeLoadAllQuery(pgSchema, InternalTable.Chains.table.tableName))
+          InternalTable.Chains.getInitialState(sql, pgSchema).then(function (rawInitialStates) {
+                return Belt_Array.map(rawInitialStates, (function (rawInitialState) {
+                              return {
+                                      id: rawInitialState.id,
+                                      startBlock: rawInitialState.startBlock,
+                                      endBlock: Caml_option.null_to_opt(rawInitialState.endBlock),
+                                      maxReorgDepth: rawInitialState.maxReorgDepth,
+                                      progressBlockNumber: rawInitialState.progressBlockNumber,
+                                      numEventsProcessed: rawInitialState.numEventsProcessed,
+                                      firstEventBlockNumber: Caml_option.null_to_opt(rawInitialState.firstEventBlockNumber),
+                                      timestampCaughtUpToHeadOrEndblock: Caml_option.null_to_opt(rawInitialState.timestampCaughtUpToHeadOrEndblock),
+                                      dynamicContracts: rawInitialState.dynamicContracts
+                                    };
+                            }));
+              }),
+          sql.unsafe(InternalTable.Checkpoints.makeCommitedCheckpointIdQuery(pgSchema)),
+          sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema))
         ]);
     return {
             cleanRun: false,
             cache: match[0],
-            chains: match[1]
+            chains: match[1],
+            checkpointId: match[2][0].id,
+            reorgCheckpoints: match[3]
           };
   };
   return {
@@ -677,7 +685,6 @@ exports.pgErrorMessageSchema = pgErrorMessageSchema;
 exports.PgEncodingError = PgEncodingError;
 exports.setQueryCache = setQueryCache;
 exports.setOrThrow = setOrThrow;
-exports.setEntityHistoryOrThrow = setEntityHistoryOrThrow;
 exports.makeSchemaTableNamesQuery = makeSchemaTableNamesQuery;
 exports.cacheTablePrefixLength = cacheTablePrefixLength;
 exports.makeSchemaCacheTableInfoQuery = makeSchemaCacheTableInfoQuery;

package/src/Prometheus.res

@@ -477,9 +477,15 @@ module RollbackSuccess = {
     "help": "Number of successful rollbacks on reorg",
   })
 
-  let increment = (~timeMillis: Hrtime.milliseconds) => {
+  let eventsCounter = PromClient.Counter.makeCounter({
+    "name": "envio_rollback_events_count",
+    "help": "Number of events rollbacked on reorg",
+  })
+
+  let increment = (~timeMillis: Hrtime.milliseconds, ~rollbackedProcessedEvents) => {
     timeCounter->PromClient.Counter.incMany(timeMillis->Hrtime.intFromMillis)
     counter->PromClient.Counter.inc
+    eventsCounter->PromClient.Counter.incMany(rollbackedProcessedEvents)
   }
 }
 

package/src/Prometheus.res.js

@@ -579,14 +579,21 @@ var counter$3 = new PromClient.Counter({
       help: "Number of successful rollbacks on reorg"
     });
 
-function increment$3(timeMillis) {
+var eventsCounter = new PromClient.Counter({
+      name: "envio_rollback_events_count",
+      help: "Number of events rollbacked on reorg"
+    });
+
+function increment$3(timeMillis, rollbackedProcessedEvents) {
   timeCounter.inc(Hrtime.intFromMillis(timeMillis));
   counter$3.inc();
+  eventsCounter.inc(rollbackedProcessedEvents);
 }
 
 var RollbackSuccess = {
   timeCounter: timeCounter,
   counter: counter$3,
+  eventsCounter: eventsCounter,
   increment: increment$3
 };