envio 2.32.1 → 3.0.0-alpha-main-clickhouse-sink

package/src/PgStorage.res.js

@@ -6,6 +6,8 @@ var Path = require("path");
 var $$Array = require("rescript/lib/js/array.js");
 var Table = require("./db/Table.res.js");
 var Utils = require("./Utils.res.js");
+var Config = require("./Config.res.js");
+var Hrtime = require("./bindings/Hrtime.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Schema = require("./db/Schema.res.js");
 var Js_dict = require("rescript/lib/js/js_dict.js");
@@ -13,6 +15,7 @@ var Logging = require("./Logging.res.js");
 var $$Promise = require("./bindings/Promise.res.js");
 var Internal = require("./Internal.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
+var Prometheus = require("./Prometheus.res.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Persistence = require("./Persistence.res.js");
 var EntityHistory = require("./db/EntityHistory.res.js");
@@ -48,20 +51,11 @@ function makeCreateTableIndicesQuery(table, pgSchema) {
 function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText) {
   var fieldsMapped = Belt_Array.map(Table.getFields(table), (function (field) {
             var defaultValue = field.defaultValue;
-            var isArray = field.isArray;
-            var fieldType = field.fieldType;
+            var isNullable = field.isNullable;
             var fieldName = Table.getDbFieldName(field);
-            var tmp;
-            tmp = fieldType === "TIMESTAMP" || fieldType === "TIMESTAMP WITH TIME ZONE" || fieldType === "JSONB" || fieldType === "SERIAL" || fieldType === "TEXT" || fieldType === "DOUBLE PRECISION" || fieldType === "NUMERIC" || fieldType === "BOOLEAN" || fieldType === "BIGINT" || fieldType === "INTEGER" || fieldType === "TIMESTAMP WITH TIME ZONE NULL" ? (
-                fieldType === "NUMERIC" && isArray && isNumericArrayAsText ? "TEXT" : fieldType
-              ) : (
-                fieldType.startsWith("NUMERIC(") ? fieldType : "\"" + pgSchema + "\"." + fieldType
-              );
-            return "\"" + fieldName + "\" " + tmp + (
-                    isArray ? "[]" : ""
-                  ) + (
+            return "\"" + fieldName + "\" " + Table.getPgFieldType(field.fieldType, pgSchema, field.isArray, isNumericArrayAsText, isNullable) + (
                     defaultValue !== undefined ? " DEFAULT " + defaultValue : (
-                        field.isNullable ? "" : " NOT NULL"
+                        isNullable ? "" : " NOT NULL"
                       )
                   );
           })).join(", ");
@@ -74,6 +68,68 @@ function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText) {
         ) + ");";
 }
 
+function getEntityHistory(entityConfig) {
+  var cache = entityConfig.pgEntityHistoryCache;
+  if (cache !== undefined) {
+    return cache;
+  }
+  var dataFields = Belt_Array.keepMap(entityConfig.table.fields, (function (field) {
+          if (field.TAG !== "Field") {
+            return ;
+          }
+          var field$1 = field._0;
+          var match = field$1.fieldName;
+          if (match === "id") {
+            return {
+                    TAG: "Field",
+                    _0: {
+                      fieldName: "id",
+                      fieldType: field$1.fieldType,
+                      fieldSchema: field$1.fieldSchema,
+                      isArray: field$1.isArray,
+                      isNullable: field$1.isNullable,
+                      isPrimaryKey: true,
+                      isIndex: field$1.isIndex,
+                      linkedEntity: field$1.linkedEntity,
+                      defaultValue: field$1.defaultValue
+                    }
+                  };
+          } else {
+            return {
+                    TAG: "Field",
+                    _0: {
+                      fieldName: field$1.fieldName,
+                      fieldType: field$1.fieldType,
+                      fieldSchema: field$1.fieldSchema,
+                      isArray: field$1.isArray,
+                      isNullable: true,
+                      isPrimaryKey: field$1.isPrimaryKey,
+                      isIndex: false,
+                      linkedEntity: field$1.linkedEntity,
+                      defaultValue: field$1.defaultValue
+                    }
+                  };
+          }
+        }));
+  var actionField = Table.mkField(EntityHistory.changeFieldName, EntityHistory.changeFieldType, S$RescriptSchema.never, undefined, undefined, undefined, undefined, undefined, undefined);
+  var checkpointIdField = Table.mkField(EntityHistory.checkpointIdFieldName, EntityHistory.checkpointIdFieldType, EntityHistory.unsafeCheckpointIdSchema, undefined, undefined, undefined, true, undefined, undefined);
+  var entityTableName = entityConfig.table.tableName;
+  var historyTableName = EntityHistory.historyTableName(entityTableName, entityConfig.index);
+  var table = Table.mkTable(historyTableName, undefined, Belt_Array.concat(dataFields, [
+            checkpointIdField,
+            actionField
+          ]));
+  var setChangeSchema = EntityHistory.makeSetUpdateSchema(entityConfig.schema);
+  var cache_setChangeSchemaRows = S$RescriptSchema.array(setChangeSchema);
+  var cache$1 = {
+    table: table,
+    setChangeSchema: setChangeSchema,
+    setChangeSchemaRows: cache_setChangeSchemaRows
+  };
+  entityConfig.pgEntityHistoryCache = cache$1;
+  return cache$1;
+}
+
 function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt) {
   var chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
   var entities = entitiesOpt !== undefined ? entitiesOpt : [];
@@ -87,10 +143,10 @@ function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfi
   ];
   var allTables = $$Array.copy(generalTables);
   var allEntityTables = [];
-  entities.forEach(function (entity) {
-        allEntityTables.push(entity.table);
-        allTables.push(entity.table);
-        allTables.push(entity.entityHistory.table);
+  entities.forEach(function (entityConfig) {
+        allEntityTables.push(entityConfig.table);
+        allTables.push(entityConfig.table);
+        allTables.push(getEntityHistory(entityConfig).table);
       });
   var derivedSchema = Schema.make(allEntityTables);
   var query = {
@@ -145,6 +201,14 @@ function makeLoadByIdsQuery(pgSchema, tableName) {
   return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = ANY($1::text[]);";
 }
 
+function makeDeleteByIdQuery(pgSchema, tableName) {
+  return "DELETE FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = $1;";
+}
+
+function makeDeleteByIdsQuery(pgSchema, tableName) {
+  return "DELETE FROM \"" + pgSchema + "\".\"" + tableName + "\" WHERE id = ANY($1::text[]);";
+}
+
 function makeLoadAllQuery(pgSchema, tableName) {
   return "SELECT * FROM \"" + pgSchema + "\".\"" + tableName + "\";";
 }
@@ -360,7 +424,267 @@ async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort) {
   return result;
 }
 
-function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, onInitialize, onNewTables) {
+async function deleteByIdsOrThrow(sql, pgSchema, ids, table) {
+  try {
+    await (
+      ids.length !== 1 ? sql.unsafe(makeDeleteByIdsQuery(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeDeleteByIdQuery(pgSchema, table.tableName), ids, {prepare: true})
+    );
+    return ;
+  }
+  catch (raw_exn){
+    var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+    throw {
+          RE_EXN_ID: Persistence.StorageError,
+          message: "Failed deleting \"" + table.tableName + "\" from storage by ids",
+          reason: exn,
+          Error: new Error()
+        };
+  }
+}
+
+function makeInsertDeleteUpdatesQuery(entityConfig, pgSchema) {
+  var historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
+  var allHistoryFieldNames = Belt_Array.keepMap(entityConfig.table.fields, (function (fieldOrDerived) {
+          if (fieldOrDerived.TAG === "Field") {
+            return Table.getDbFieldName(fieldOrDerived._0);
+          }
+          
+        }));
+  allHistoryFieldNames.push(EntityHistory.checkpointIdFieldName);
+  allHistoryFieldNames.push(EntityHistory.changeFieldName);
+  var allHistoryFieldNamesStr = Belt_Array.map(allHistoryFieldNames, (function (name) {
+            return "\"" + name + "\"";
+          })).join(", ");
+  var selectParts = Belt_Array.map(allHistoryFieldNames, (function (fieldName) {
+          if (fieldName === Table.idFieldName) {
+            return "u." + Table.idFieldName;
+          } else if (fieldName === EntityHistory.checkpointIdFieldName) {
+            return "u." + EntityHistory.checkpointIdFieldName;
+          } else if (fieldName === EntityHistory.changeFieldName) {
+            return "'" + "DELETE" + "'";
+          } else {
+            return "NULL";
+          }
+        }));
+  var selectPartsStr = selectParts.join(", ");
+  var checkpointIdPgType = Table.getPgFieldType(EntityHistory.checkpointIdFieldType, pgSchema, false, false, false);
+  return "INSERT INTO \"" + pgSchema + "\".\"" + historyTableName + "\" (" + allHistoryFieldNamesStr + ")\nSELECT " + selectPartsStr + "\nFROM UNNEST($1::text[], $2::" + checkpointIdPgType + "[]) AS u(" + Table.idFieldName + ", " + EntityHistory.checkpointIdFieldName + ")";
+}
+
+function executeSet(sql, items, dbFunction) {
+  if (items.length !== 0) {
+    return dbFunction(sql, items);
+  } else {
+    return Promise.resolve();
+  }
+}
+
+async function writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, escapeTables) {
+  try {
+    var shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold);
+    var specificError = {
+      contents: undefined
+    };
+    var setRawEvents = function (__x) {
+      return executeSet(__x, rawEvents, (function (sql, items) {
+                    return setOrThrow(sql, items, InternalTable.RawEvents.table, InternalTable.RawEvents.schema, pgSchema);
+                  }));
+    };
+    var setEntities = Belt_Array.map(updatedEntities, (function (param) {
+            var updates = param.updates;
+            var entityConfig = param.entityConfig;
+            var entitiesToSet = [];
+            var idsToDelete = [];
+            updates.forEach(function (row) {
+                  var match = row.latestChange;
+                  if (match.type === "SET") {
+                    entitiesToSet.push(match.entity);
+                    return ;
+                  }
+                  idsToDelete.push(match.entityId);
+                });
+            var shouldRemoveInvalidUtf8 = escapeTables !== undefined && Caml_option.valFromOption(escapeTables).has(entityConfig.table) ? true : false;
+            return async function (sql) {
+              try {
+                var promises = [];
+                if (shouldSaveHistory) {
+                  var backfillHistoryIds = new Set();
+                  var batchSetUpdates = [];
+                  var batchDeleteCheckpointIds = [];
+                  var batchDeleteEntityIds = [];
+                  updates.forEach(function (update) {
+                        var containsRollbackDiffChange = update.containsRollbackDiffChange;
+                        update.history.forEach(function (change) {
+                              if (!containsRollbackDiffChange) {
+                                backfillHistoryIds.add(change.entityId);
+                              }
+                              if (change.type === "SET") {
+                                batchSetUpdates.push(change);
+                                return ;
+                              }
+                              batchDeleteEntityIds.push(change.entityId);
+                              batchDeleteCheckpointIds.push(change.checkpointId);
+                            });
+                      });
+                  if (backfillHistoryIds.size !== 0) {
+                    await EntityHistory.backfillHistory(sql, pgSchema, entityConfig.name, entityConfig.index, Array.from(backfillHistoryIds));
+                  }
+                  if (Utils.$$Array.notEmpty(batchDeleteCheckpointIds)) {
+                    promises.push(sql.unsafe(makeInsertDeleteUpdatesQuery(entityConfig, pgSchema), [
+                              batchDeleteEntityIds,
+                              batchDeleteCheckpointIds
+                            ], {prepare: true}));
+                  }
+                  if (Utils.$$Array.notEmpty(batchSetUpdates)) {
+                    if (shouldRemoveInvalidUtf8) {
+                      var entities = batchSetUpdates.map(function (batchSetUpdate) {
+                            if (batchSetUpdate.type === "SET") {
+                              return batchSetUpdate.entity;
+                            } else {
+                              return Js_exn.raiseError("Expected Set action");
+                            }
+                          });
+                      removeInvalidUtf8InPlace(entities);
+                    }
+                    var entityHistory = getEntityHistory(entityConfig);
+                    promises.push(setOrThrow(sql, batchSetUpdates, entityHistory.table, entityHistory.setChangeSchema, pgSchema));
+                  }
+                  
+                }
+                if (Utils.$$Array.notEmpty(entitiesToSet)) {
+                  if (shouldRemoveInvalidUtf8) {
+                    removeInvalidUtf8InPlace(entitiesToSet);
+                  }
+                  promises.push(setOrThrow(sql, entitiesToSet, entityConfig.table, entityConfig.schema, pgSchema));
+                }
+                if (Utils.$$Array.notEmpty(idsToDelete)) {
+                  promises.push(deleteByIdsOrThrow(sql, pgSchema, idsToDelete, entityConfig.table));
+                }
+                await Promise.all(promises);
+                return ;
+              }
+              catch (raw_exn){
+                var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+                var normalizedExn = Caml_js_exceptions.internalToOCamlException(exn.RE_EXN_ID === "JsError" || exn.RE_EXN_ID !== Persistence.StorageError ? exn : exn.reason);
+                if (normalizedExn.RE_EXN_ID === "JsError") {
+                  var val;
+                  try {
+                    val = S$RescriptSchema.parseOrThrow(normalizedExn._1, pgErrorMessageSchema);
+                  }
+                  catch (exn$1){
+                    return ;
+                  }
+                  switch (val) {
+                    case "current transaction is aborted, commands ignored until end of transaction block" :
+                        return ;
+                    case "invalid byte sequence for encoding \"UTF8\": 0x00" :
+                        specificError.contents = {
+                          RE_EXN_ID: PgEncodingError,
+                          table: entityConfig.table
+                        };
+                        return ;
+                    default:
+                      specificError.contents = Utils.prettifyExn(exn);
+                      return ;
+                  }
+                } else {
+                  if (normalizedExn.RE_EXN_ID !== S$RescriptSchema.Raised) {
+                    return ;
+                  }
+                  throw normalizedExn;
+                }
+              }
+            };
+          }));
+    var rollbackTables = rollbackTargetCheckpointId !== undefined ? (function (sql) {
+          var promises = allEntities.map(function (entityConfig) {
+                return EntityHistory.rollback(sql, pgSchema, entityConfig.name, entityConfig.index, rollbackTargetCheckpointId);
+              });
+          promises.push(InternalTable.Checkpoints.rollback(sql, pgSchema, rollbackTargetCheckpointId));
+          return Promise.all(promises);
+        }) : undefined;
+    try {
+      await Promise.all([
+            sql.begin(async function (sql) {
+                  if (rollbackTables !== undefined) {
+                    await rollbackTables(sql);
+                  }
+                  var setOperations = Belt_Array.concat([
+                        (function (sql) {
+                            return InternalTable.Chains.setProgressedChains(sql, pgSchema, Utils.Dict.mapValuesToArray(batch.progressedChainsById, (function (chainAfterBatch) {
+                                              return {
+                                                      chainId: chainAfterBatch.fetchState.chainId,
+                                                      progressBlockNumber: chainAfterBatch.progressBlockNumber,
+                                                      totalEventsProcessed: chainAfterBatch.totalEventsProcessed
+                                                    };
+                                            })));
+                          }),
+                        setRawEvents
+                      ], setEntities);
+                  if (shouldSaveHistory) {
+                    setOperations.push(function (sql) {
+                          return InternalTable.Checkpoints.insert(sql, pgSchema, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointBlockHashes, batch.checkpointEventsProcessed);
+                        });
+                  }
+                  await Promise.all(Belt_Array.map(setOperations, (function (dbFunc) {
+                              return dbFunc(sql);
+                            })));
+                  if (sinkPromise === undefined) {
+                    return ;
+                  }
+                  var exn = await Caml_option.valFromOption(sinkPromise);
+                  if (exn === undefined) {
+                    return ;
+                  }
+                  throw exn;
+                }),
+            Promise.all(Belt_Array.map(updatedEffectsCache, (function (param) {
+                        return setEffectCacheOrThrow(param.effect, param.items, param.shouldInitialize);
+                      })))
+          ]);
+      var specificError$1 = specificError.contents;
+      if (specificError$1 === undefined) {
+        return ;
+      }
+      throw specificError$1;
+    }
+    catch (raw_exn){
+      var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
+      var specificError$2 = specificError.contents;
+      throw specificError$2 !== undefined ? specificError$2 : exn;
+    }
+  }
+  catch (raw_exn$1){
+    var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn$1);
+    if (exn$1.RE_EXN_ID === PgEncodingError) {
+      var escapeTables$1 = escapeTables !== undefined ? Caml_option.valFromOption(escapeTables) : new Set();
+      escapeTables$1.add(exn$1.table);
+      return await writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, Caml_option.some(escapeTables$1));
+    }
+    throw exn$1;
+  }
+}
+
+function makeGetRollbackRestoredEntitiesQuery(entityConfig, pgSchema) {
+  var dataFieldNames = Belt_Array.keepMap(entityConfig.table.fields, (function (fieldOrDerived) {
+          if (fieldOrDerived.TAG === "Field") {
+            return Table.getDbFieldName(fieldOrDerived._0);
+          }
+          
+        }));
+  var dataFieldsCommaSeparated = Belt_Array.map(dataFieldNames, (function (name) {
+            return "\"" + name + "\"";
+          })).join(", ");
+  var historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
+  return "SELECT DISTINCT ON (" + Table.idFieldName + ") " + dataFieldsCommaSeparated + "\nFROM \"" + pgSchema + "\".\"" + historyTableName + "\"\nWHERE \"" + EntityHistory.checkpointIdFieldName + "\" <= $1\n  AND EXISTS (\n    SELECT 1\n    FROM \"" + pgSchema + "\".\"" + historyTableName + "\" h\n    WHERE h." + Table.idFieldName + " = \"" + historyTableName + "\"." + Table.idFieldName + "\n      AND h.\"" + EntityHistory.checkpointIdFieldName + "\" > $1\n  )\nORDER BY " + Table.idFieldName + ", \"" + EntityHistory.checkpointIdFieldName + "\" DESC";
+}
+
+function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema) {
+  var historyTableName = EntityHistory.historyTableName(entityConfig.name, entityConfig.index);
+  return "SELECT DISTINCT " + Table.idFieldName + "\nFROM \"" + pgSchema + "\".\"" + historyTableName + "\"\nWHERE \"" + EntityHistory.checkpointIdFieldName + "\" > $1\nAND NOT EXISTS (\n  SELECT 1\n  FROM \"" + pgSchema + "\".\"" + historyTableName + "\" h\n  WHERE h." + Table.idFieldName + " = \"" + historyTableName + "\"." + Table.idFieldName + "\n    AND h.\"" + EntityHistory.checkpointIdFieldName + "\" <= $1\n)";
+}
+
+function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, sink, onInitialize, onNewTables) {
   var psqlExecOptions_env = Js_dict.fromArray([
         [
           "PGPASSWORD",
@@ -467,6 +791,9 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
           })) {
       Js_exn.raiseError("Cannot run Envio migrations on PostgreSQL schema \"" + pgSchema + "\" because it contains non-Envio tables. Running migrations would delete all data in this schema.\n\nTo resolve this:\n1. If you want to use this schema, first backup any important data, then drop it with: \"pnpm envio local db-migrate down\"\n2. Or specify a different schema name by setting the \"ENVIO_PG_PUBLIC_SCHEMA\" environment variable\n3. Or manually drop the schema in your database if you're certain the data is not needed.");
     }
+    if (sink !== undefined) {
+      await sink.initialize(chainConfigs, entities, enums);
+    }
     var queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, entities, enums, Utils.$$Array.isEmpty(schemaTableNames));
     await sql.begin(function (sql) {
           return Promise.all(queries.map(function (query) {
@@ -649,14 +976,81 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
           sql.unsafe(InternalTable.Checkpoints.makeCommitedCheckpointIdQuery(pgSchema)),
           sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema))
         ]);
+    var checkpointId = match[2][0].id;
+    if (sink !== undefined) {
+      await sink.resume(checkpointId);
+    }
     return {
             cleanRun: false,
             cache: match[0],
             chains: match[1],
-            checkpointId: match[2][0].id,
+            checkpointId: checkpointId,
             reorgCheckpoints: match[3]
           };
   };
+  var executeUnsafe = function (query) {
+    return sql.unsafe(query);
+  };
+  var hasEntityHistoryRows = async function () {
+    var historyTables = await sql.unsafe("SELECT table_name FROM information_schema.tables \n       WHERE table_schema = '" + pgSchema + "' \n       AND table_name LIKE 'envio_history_%';");
+    if (Utils.$$Array.isEmpty(historyTables)) {
+      return false;
+    }
+    var checks = await Promise.all(Belt_Array.map(historyTables, (async function (table) {
+                try {
+                  var query = "SELECT EXISTS(SELECT 1 FROM \"" + pgSchema + "\".\"" + table.table_name + "\" LIMIT 1);";
+                  var result = await sql.unsafe(query);
+                  if (result.length !== 1) {
+                    return false;
+                  } else {
+                    return result[0].exists;
+                  }
+                }
+                catch (exn){
+                  return false;
+                }
+              })));
+    return Belt_Array.some(checks, (function (v) {
+                  return v;
+                }));
+  };
+  var setChainMeta = function (chainsData) {
+    return InternalTable.Chains.setMeta(sql, pgSchema, chainsData).then(function (param) {
+                return undefined;
+              });
+  };
+  var pruneStaleCheckpoints = function (safeCheckpointId) {
+    return InternalTable.Checkpoints.pruneStaleCheckpoints(sql, pgSchema, safeCheckpointId);
+  };
+  var pruneStaleEntityHistory = function (entityName, entityIndex, safeCheckpointId) {
+    return EntityHistory.pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, safeCheckpointId);
+  };
+  var getRollbackTargetCheckpoint = function (reorgChainId, lastKnownValidBlockNumber) {
+    return InternalTable.Checkpoints.getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValidBlockNumber);
+  };
+  var getRollbackProgressDiff = function (rollbackTargetCheckpointId) {
+    return InternalTable.Checkpoints.getRollbackProgressDiff(sql, pgSchema, rollbackTargetCheckpointId);
+  };
+  var getRollbackData = async function (entityConfig, rollbackTargetCheckpointId) {
+    return await Promise.all([
+                sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId], {prepare: true}),
+                sql.unsafe(makeGetRollbackRestoredEntitiesQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId], {prepare: true})
+              ]);
+  };
+  var writeBatchMethod = async function (batch, rawEvents, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, updatedEffectsCache, updatedEntities) {
+    var sinkPromise;
+    if (sink !== undefined) {
+      var timerRef = Hrtime.makeTimer();
+      sinkPromise = Caml_option.some(sink.writeBatch(batch, updatedEntities).then(function () {
+                  Prometheus.SinkWrite.increment(sink.name, Hrtime.intFromMillis(Hrtime.toMillis(Hrtime.timeSince(timerRef))));
+                }).catch(function (exn) {
+                return exn;
+              }));
+    } else {
+      sinkPromise = undefined;
+    }
+    return await writeBatch(sql, batch, rawEvents, pgSchema, rollbackTargetCheckpointId, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, undefined);
+  };
   return {
           isInitialized: isInitialized,
           initialize: initialize,
@@ -665,7 +1059,16 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
           loadByFieldOrThrow: loadByFieldOrThrow,
           setOrThrow: setOrThrow$1,
           setEffectCacheOrThrow: setEffectCacheOrThrow,
-          dumpEffectCache: dumpEffectCache
+          dumpEffectCache: dumpEffectCache,
+          executeUnsafe: executeUnsafe,
+          hasEntityHistoryRows: hasEntityHistoryRows,
+          setChainMeta: setChainMeta,
+          pruneStaleCheckpoints: pruneStaleCheckpoints,
+          pruneStaleEntityHistory: pruneStaleEntityHistory,
+          getRollbackTargetCheckpoint: getRollbackTargetCheckpoint,
+          getRollbackProgressDiff: getRollbackProgressDiff,
+          getRollbackData: getRollbackData,
+          writeBatch: writeBatchMethod
         };
 }
 
@@ -675,10 +1078,13 @@ exports.getCacheRowCountFnName = getCacheRowCountFnName;
 exports.makeCreateIndexQuery = makeCreateIndexQuery;
 exports.makeCreateTableIndicesQuery = makeCreateTableIndicesQuery;
 exports.makeCreateTableQuery = makeCreateTableQuery;
+exports.getEntityHistory = getEntityHistory;
 exports.makeInitializeTransaction = makeInitializeTransaction;
 exports.makeLoadByIdQuery = makeLoadByIdQuery;
 exports.makeLoadByFieldQuery = makeLoadByFieldQuery;
 exports.makeLoadByIdsQuery = makeLoadByIdsQuery;
+exports.makeDeleteByIdQuery = makeDeleteByIdQuery;
+exports.makeDeleteByIdsQuery = makeDeleteByIdsQuery;
 exports.makeLoadAllQuery = makeLoadAllQuery;
 exports.makeInsertUnnestSetQuery = makeInsertUnnestSetQuery;
 exports.makeInsertValuesSetQuery = makeInsertValuesSetQuery;
@@ -694,5 +1100,11 @@ exports.makeSchemaTableNamesQuery = makeSchemaTableNamesQuery;
 exports.cacheTablePrefixLength = cacheTablePrefixLength;
 exports.makeSchemaCacheTableInfoQuery = makeSchemaCacheTableInfoQuery;
 exports.getConnectedPsqlExec = getConnectedPsqlExec;
+exports.deleteByIdsOrThrow = deleteByIdsOrThrow;
+exports.makeInsertDeleteUpdatesQuery = makeInsertDeleteUpdatesQuery;
+exports.executeSet = executeSet;
+exports.writeBatch = writeBatch;
+exports.makeGetRollbackRestoredEntitiesQuery = makeGetRollbackRestoredEntitiesQuery;
+exports.makeGetRollbackRemovedIdsQuery = makeGetRollbackRemovedIdsQuery;
 exports.make = make;
 /* pgErrorMessageSchema Not a pure module */

package/src/Platform.res

@@ -0,0 +1,141 @@
+type name = | @as("evm") Evm | @as("fuel") Fuel
+
+type t = {
+  name: name,
+  blockFields: array<string>,
+  transactionFields: array<string>,
+  blockNumberName: string,
+  blockTimestampName: string,
+  blockHashName: string,
+  getNumber: Internal.eventBlock => int,
+  getTimestamp: Internal.eventBlock => int,
+  getId: Internal.eventBlock => string,
+  cleanUpRawEventFieldsInPlace: Js.Json.t => unit,
+}
+
+module Evm = {
+  @get external getNumber: Internal.eventBlock => int = "number"
+  @get external getTimestamp: Internal.eventBlock => int = "timestamp"
+  @get external getId: Internal.eventBlock => string = "hash"
+  
+  let cleanUpRawEventFieldsInPlace: Js.Json.t => unit = %raw(`fields => {
+    delete fields.hash
+    delete fields.number
+    delete fields.timestamp
+  }`)
+}
+
+let evm: t = {
+  name: Evm,
+  blockFields: [
+    "number",
+    "timestamp",
+    "hash",
+    "parentHash",
+    "nonce",
+    "sha3Uncles",
+    "logsBloom",
+    "transactionsRoot",
+    "stateRoot",
+    "receiptsRoot",
+    "miner",
+    "difficulty",
+    "totalDifficulty",
+    "extraData",
+    "size",
+    "gasLimit",
+    "gasUsed",
+    "uncles",
+    "baseFeePerGas",
+    "blobGasUsed",
+    "excessBlobGas",
+    "parentBeaconBlockRoot",
+    "withdrawalsRoot",
+    "l1BlockNumber",
+    "sendCount",
+    "sendRoot",
+    "mixHash",
+  ],
+  transactionFields: [
+    "transactionIndex",
+    "hash",
+    "from",
+    "to",
+    "gas",
+    "gasPrice",
+    "maxPriorityFeePerGas",
+    "maxFeePerGas",
+    "cumulativeGasUsed",
+    "effectiveGasPrice",
+    "gasUsed",
+    "input",
+    "nonce",
+    "value",
+    "v",
+    "r",
+    "s",
+    "contractAddress",
+    "logsBloom",
+    "root",
+    "status",
+    "yParity",
+    "chainId",
+    "maxFeePerBlobGas",
+    "blobVersionedHashes",
+    "kind",
+    "l1Fee",
+    "l1GasPrice",
+    "l1GasUsed",
+    "l1FeeScalar",
+    "gasUsedForL1",
+    "accessList",
+    "authorizationList",
+  ],
+  blockNumberName: "number",
+  blockTimestampName: "timestamp",
+  blockHashName: "hash",
+  getNumber: Evm.getNumber,
+  getTimestamp: Evm.getTimestamp,
+  getId: Evm.getId,
+  cleanUpRawEventFieldsInPlace: Evm.cleanUpRawEventFieldsInPlace,
+}
+
+module Fuel = {
+  @get external getNumber: Internal.eventBlock => int = "height"
+  @get external getTimestamp: Internal.eventBlock => int = "time"
+  @get external getId: Internal.eventBlock => string = "id"
+  
+  let cleanUpRawEventFieldsInPlace: Js.Json.t => unit = %raw(`fields => {
+    delete fields.id
+    delete fields.height
+    delete fields.time
+  }`)
+}
+
+let fuel: t = {
+  name: Fuel,
+  blockFields: ["id", "height", "time"],
+  transactionFields: ["id"],
+  blockNumberName: "height",
+  blockTimestampName: "time",
+  blockHashName: "id",
+  getNumber: Fuel.getNumber,
+  getTimestamp: Fuel.getTimestamp,
+  getId: Fuel.getId,
+  cleanUpRawEventFieldsInPlace: Fuel.cleanUpRawEventFieldsInPlace,
+}
+
+let fromName = (name: name): t => {
+  switch name {
+  | Evm => evm
+  | Fuel => fuel
+  }
+}
+
+// Create a block event object for block handlers based on platform
+let makeBlockEvent = (~blockNumber: int, ~chainId: int, platform: t): Internal.blockEvent => {
+  let blockEvent = Js.Dict.empty()
+  blockEvent->Js.Dict.set("chainId", chainId->Utils.magic)
+  blockEvent->Js.Dict.set(platform.blockNumberName, blockNumber->Utils.magic)
+  blockEvent->Utils.magic
+}