envio 2.27.5-rc.0 → 2.27.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "envio",
-  "version": "v2.27.5-rc.0",
+  "version": "v2.27.6",
   "description": "A latency and sync speed optimized, developer friendly blockchain data indexer.",
   "bin": "./bin.js",
   "main": "./index.js",
@@ -25,10 +25,10 @@
   },
   "homepage": "https://envio.dev",
   "optionalDependencies": {
-    "envio-linux-x64": "v2.27.5-rc.0",
-    "envio-linux-arm64": "v2.27.5-rc.0",
-    "envio-darwin-x64": "v2.27.5-rc.0",
-    "envio-darwin-arm64": "v2.27.5-rc.0"
+    "envio-linux-x64": "v2.27.6",
+    "envio-linux-arm64": "v2.27.6",
+    "envio-darwin-x64": "v2.27.6",
+    "envio-darwin-arm64": "v2.27.6"
   },
   "dependencies": {
     "@envio-dev/hypersync-client": "0.6.5",

package/src/ErrorHandling.res

@@ -6,18 +6,17 @@ let make = (exn, ~logger=Logging.getLogger(), ~msg=?) => {
 
 let log = (self: t) => {
   switch self {
-  | {exn, msg: Some(msg), logger} =>
-    logger->Logging.childErrorWithExn(exn->Internal.prettifyExn, msg)
-  | {exn, msg: None, logger} => logger->Logging.childError(exn->Internal.prettifyExn)
+  | {exn, msg: Some(msg), logger} => logger->Logging.childErrorWithExn(exn->Utils.prettifyExn, msg)
+  | {exn, msg: None, logger} => logger->Logging.childError(exn->Utils.prettifyExn)
   }
 }
 
 let raiseExn = (self: t) => {
-  self.exn->Internal.prettifyExn->raise
+  self.exn->Utils.prettifyExn->raise
 }
 
 let mkLogAndRaise = (~logger=?, ~msg=?, exn) => {
-  let exn = exn->Internal.prettifyExn
+  let exn = exn->Utils.prettifyExn
   exn->make(~logger?, ~msg?)->log
   exn->raise
 }

package/src/ErrorHandling.res.js

@@ -1,8 +1,8 @@
 // Generated by ReScript, PLEASE EDIT WITH CARE
 'use strict';
 
+var Utils = require("./Utils.res.js");
 var Logging = require("./Logging.res.js");
-var Internal = require("./Internal.res.js");
 
 function make(exn, loggerOpt, msg) {
   var logger = loggerOpt !== undefined ? loggerOpt : Logging.getLogger();
@@ -18,18 +18,18 @@ function log(self) {
   var exn = self.exn;
   var logger = self.logger;
   if (msg !== undefined) {
-    return Logging.childErrorWithExn(logger, Internal.prettifyExn(exn), msg);
+    return Logging.childErrorWithExn(logger, Utils.prettifyExn(exn), msg);
   } else {
-    return Logging.childError(logger, Internal.prettifyExn(exn));
+    return Logging.childError(logger, Utils.prettifyExn(exn));
   }
 }
 
 function raiseExn(self) {
-  throw Internal.prettifyExn(self.exn);
+  throw Utils.prettifyExn(self.exn);
 }
 
 function mkLogAndRaise(logger, msg, exn) {
-  var exn$1 = Internal.prettifyExn(exn);
+  var exn$1 = Utils.prettifyExn(exn);
   log(make(exn$1, logger, msg));
   throw exn$1;
 }
@@ -44,7 +44,7 @@ function unwrapLogAndRaise(logger, msg, result) {
 
 function logAndRaise(self) {
   log(self);
-  throw Internal.prettifyExn(self.exn);
+  throw Utils.prettifyExn(self.exn);
 }
 
 exports.make = make;
@@ -53,4 +53,4 @@ exports.raiseExn = raiseExn;
 exports.mkLogAndRaise = mkLogAndRaise;
 exports.unwrapLogAndRaise = unwrapLogAndRaise;
 exports.logAndRaise = logAndRaise;
-/* Logging Not a pure module */
+/* Utils Not a pure module */

package/src/Hasura.res

@@ -83,7 +83,7 @@ let clearHasuraMetadata = async (~endpoint, ~auth) => {
   | exn =>
     Logging.error({
       "msg": `EE806: There was an issue clearing metadata in hasura - indexing may still work - but you may have issues querying the data in hasura.`,
-      "err": exn->Internal.prettifyExn,
+      "err": exn->Utils.prettifyExn,
     })
   }
 }
@@ -125,7 +125,7 @@ let trackTables = async (~endpoint, ~auth, ~pgSchema, ~tableNames: array<string>
     Logging.error({
       "msg": `EE807: There was an issue tracking tables in hasura - indexing may still work - but you may have issues querying the data in hasura.`,
       "tableNames": tableNames,
-      "err": exn->Internal.prettifyExn,
+      "err": exn->Utils.prettifyExn,
     })
   }
 }
@@ -172,7 +172,7 @@ let createSelectPermissions = async (
     Logging.error({
       "msg": `EE808: There was an issue setting up view permissions for the ${tableName} table in hasura - indexing may still work - but you may have issues querying the data in hasura.`,
       "tableName": tableName,
-      "err": exn->Internal.prettifyExn,
+      "err": exn->Utils.prettifyExn,
     })
   }
 }
@@ -213,7 +213,7 @@ let createEntityRelationship = async (
     Logging.error({
       "msg": `EE808: There was an issue setting up ${relationshipType} relationship for the ${tableName} table in hasura - indexing may still work - but you may have issues querying the data in hasura.`,
       "tableName": tableName,
-      "err": exn->Internal.prettifyExn,
+      "err": exn->Utils.prettifyExn,
     })
   }
 }

package/src/Hasura.res.js

@@ -6,7 +6,6 @@ var Table = require("./db/Table.res.js");
 var Utils = require("./Utils.res.js");
 var Schema = require("./db/Schema.res.js");
 var Logging = require("./Logging.res.js");
-var Internal = require("./Internal.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var Caml_splice_call = require("rescript/lib/js/caml_splice_call.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
@@ -102,7 +101,7 @@ async function clearHasuraMetadata(endpoint, auth) {
     var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
     return Logging.error({
                 msg: "EE806: There was an issue clearing metadata in hasura - indexing may still work - but you may have issues querying the data in hasura.",
-                err: Internal.prettifyExn(exn)
+                err: Utils.prettifyExn(exn)
               });
   }
 }
@@ -138,7 +137,7 @@ async function trackTables(endpoint, auth, pgSchema, tableNames) {
     return Logging.error({
                 msg: "EE807: There was an issue tracking tables in hasura - indexing may still work - but you may have issues querying the data in hasura.",
                 tableNames: tableNames,
-                err: Internal.prettifyExn(exn)
+                err: Utils.prettifyExn(exn)
               });
   }
 }
@@ -174,7 +173,7 @@ async function createSelectPermissions(auth, endpoint, tableName, pgSchema, resp
     return Logging.error({
                 msg: "EE808: There was an issue setting up view permissions for the " + tableName + " table in hasura - indexing may still work - but you may have issues querying the data in hasura.",
                 tableName: tableName,
-                err: Internal.prettifyExn(exn)
+                err: Utils.prettifyExn(exn)
               });
   }
 }
@@ -199,7 +198,7 @@ async function createEntityRelationship(pgSchema, endpoint, auth, tableName, rel
     return Logging.error({
                 msg: "EE808: There was an issue setting up " + relationshipType + " relationship for the " + tableName + " table in hasura - indexing may still work - but you may have issues querying the data in hasura.",
                 tableName: tableName,
-                err: Internal.prettifyExn(exn)
+                err: Utils.prettifyExn(exn)
               });
   }
 }

package/src/Internal.res

@@ -238,10 +238,3 @@ let makeCacheTable = (~effectName) => {
 
 @genType.import(("./Types.ts", "Invalid"))
 type noEventFilters
-
-let prettifyExn = exn => {
-  switch exn->Js.Exn.anyToExnInternal {
-  | Js.Exn.Error(e) => e->(Utils.magic: Js.Exn.t => exn)
-  | exn => exn
-  }
-}

package/src/Internal.res.js

@@ -8,7 +8,6 @@ var Address = require("./Address.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
-var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
 var fuelSupplyParamsSchema = S$RescriptSchema.schema(function (s) {
       return {
@@ -44,19 +43,9 @@ function makeCacheTable(effectName) {
             ]);
 }
 
-function prettifyExn(exn) {
-  var e = Caml_js_exceptions.internalToOCamlException(exn);
-  if (e.RE_EXN_ID === Js_exn.$$Error) {
-    return e._1;
-  } else {
-    return e;
-  }
-}
-
 exports.fuelSupplyParamsSchema = fuelSupplyParamsSchema;
 exports.fuelTransferParamsSchema = fuelTransferParamsSchema;
 exports.makeEnumConfig = makeEnumConfig;
 exports.cacheTablePrefix = cacheTablePrefix;
 exports.makeCacheTable = makeCacheTable;
-exports.prettifyExn = prettifyExn;
 /* fuelSupplyParamsSchema Not a pure module */

package/src/LoadManager.res

@@ -71,7 +71,7 @@ let schedule = async loadManager => {
         await group.load(inputsToLoad)
       } catch {
       | exn => {
-          let exn = exn->Internal.prettifyExn
+          let exn = exn->Utils.prettifyExn
           currentInputKeys->Array.forEach(inputKey => {
             let call = calls->Js.Dict.unsafeGet(inputKey)
             call.reject(exn)

package/src/LoadManager.res.js

@@ -2,7 +2,6 @@
 'use strict';
 
 var Utils = require("./Utils.res.js");
-var Internal = require("./Internal.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
@@ -47,7 +46,7 @@ async function schedule(loadManager) {
           }
           catch (raw_exn){
             var exn = Caml_js_exceptions.internalToOCamlException(raw_exn);
-            var exn$1 = Internal.prettifyExn(exn);
+            var exn$1 = Utils.prettifyExn(exn);
             Belt_Array.forEach(currentInputKeys, (function (inputKey) {
                     var call = calls[inputKey];
                     call.reject(exn$1);

package/src/Logging.res

@@ -183,5 +183,5 @@ let getUserLogger = (eventItem): Envio.logger => {
   warn: (message: string, ~params=?) => eventItem->logForItem(#uwarn, message, ~params?),
   error: (message: string, ~params=?) => eventItem->logForItem(#uerror, message, ~params?),
   errorWithExn: (message: string, exn) =>
-    eventItem->logForItem(#uerror, message, ~params={"err": exn->Internal.prettifyExn}),
+    eventItem->logForItem(#uerror, message, ~params={"err": exn->Utils.prettifyExn}),
 }

package/src/Logging.res.js

@@ -3,10 +3,10 @@
 
 var Pino = require("./bindings/Pino.res.js");
 var Pino$1 = require("pino");
+var Utils = require("./Utils.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Js_dict = require("rescript/lib/js/js_dict.js");
 var Caml_obj = require("rescript/lib/js/caml_obj.js");
-var Internal = require("./Internal.res.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var EcsPinoFormat = require("@elastic/ecs-pino-format");
 
@@ -242,7 +242,7 @@ function getUserLogger(eventItem) {
             }),
           errorWithExn: (function (message, exn) {
               var params = {
-                err: Internal.prettifyExn(exn)
+                err: Utils.prettifyExn(exn)
               };
               getEventLogger(eventItem)["uerror"](params, message);
             })

package/src/PgStorage.res

@@ -325,9 +325,7 @@ let removeInvalidUtf8InPlace = entities =>
     })
   })
 
-let pgErrorMessageSchema = S.object(s =>
-  s.field("message", S.string)
-)
+let pgErrorMessageSchema = S.object(s => s.field("message", S.string))
 
 exception PgEncodingError({table: Table.table})
 
@@ -389,7 +387,7 @@ let setOrThrow = async (sql, ~items, ~table: Table.table, ~itemSchema, ~pgSchema
       raise(
         Persistence.StorageError({
           message: `Failed to insert items into table "${table.tableName}"`,
-          reason: exn->Internal.prettifyExn,
+          reason: exn->Utils.prettifyExn,
         }),
       )
     }
@@ -765,7 +763,7 @@ let make = (
         }
       }
     } catch {
-    | exn => Logging.errorWithExn(exn->Internal.prettifyExn, `Failed to dump cache.`)
+    | exn => Logging.errorWithExn(exn->Utils.prettifyExn, `Failed to dump cache.`)
     }
   }
 

package/src/PgStorage.res.js

@@ -277,7 +277,7 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
     throw {
           RE_EXN_ID: Persistence.StorageError,
           message: "Failed to insert items into table \"" + table.tableName + "\"",
-          reason: Internal.prettifyExn(exn),
+          reason: Utils.prettifyExn(exn),
           Error: new Error()
         };
   }
@@ -523,7 +523,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, onI
     }
     catch (raw_exn){
       var exn$1 = Caml_js_exceptions.internalToOCamlException(raw_exn);
-      return Logging.errorWithExn(Internal.prettifyExn(exn$1), "Failed to dump cache.");
+      return Logging.errorWithExn(Utils.prettifyExn(exn$1), "Failed to dump cache.");
     }
   };
   var restoreEffectCache = async function (withUpload) {

package/src/Time.res

@@ -17,7 +17,7 @@ let rec retryAsyncWithExponentialBackOff = async (
       let log = retryCount === 0 ? Logging.childTrace : Logging.childWarn
       logger->log({
         "msg": `Retrying query ${nextRetryCount->Belt.Int.toString}/${maxRetries->Belt.Int.toString} in ${backOffMillis->Belt.Int.toString}ms - waiting for correct result.`,
-        "err": exn->Internal.prettifyExn,
+        "err": exn->Utils.prettifyExn,
       })
       await resolvePromiseAfterDelay(~delayMilliseconds=backOffMillis)
 

package/src/Time.res.js

@@ -3,7 +3,6 @@
 
 var Utils = require("./Utils.res.js");
 var Logging = require("./Logging.res.js");
-var Internal = require("./Internal.res.js");
 var ErrorHandling = require("./ErrorHandling.res.js");
 var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
@@ -26,7 +25,7 @@ async function retryAsyncWithExponentialBackOff(backOffMillisOpt, multiplicative
       var log = retryCount === 0 ? Logging.childTrace : Logging.childWarn;
       log(logger, {
             msg: "Retrying query " + String(nextRetryCount) + "/" + String(maxRetries) + " in " + String(backOffMillis) + "ms - waiting for correct result.",
-            err: Internal.prettifyExn(exn)
+            err: Utils.prettifyExn(exn)
           });
       await Utils.delay(backOffMillis);
       return await retryAsyncWithExponentialBackOff(Math.imul(backOffMillis, multiplicative), multiplicative, nextRetryCount, maxRetries, logger, f);

package/src/Utils.res

@@ -584,3 +584,10 @@ module Hash = {
     }
   }
 }
+
+let prettifyExn = exn => {
+  switch exn->Js.Exn.anyToExnInternal {
+  | Js.Exn.Error(e) => e->(magic: Js.Exn.t => exn)
+  | exn => exn
+  }
+}

package/src/Utils.res.js

@@ -10,6 +10,7 @@ var Belt_Option = require("rescript/lib/js/belt_Option.js");
 var Caml_option = require("rescript/lib/js/caml_option.js");
 var Caml_splice_call = require("rescript/lib/js/caml_splice_call.js");
 var S$RescriptSchema = require("rescript-schema/src/S.res.js");
+var Caml_js_exceptions = require("rescript/lib/js/caml_js_exceptions.js");
 
 function delay(milliseconds) {
   return new Promise((function (resolve, param) {
@@ -530,6 +531,15 @@ var Hash = {
   makeOrThrow: makeOrThrow
 };
 
+function prettifyExn(exn) {
+  var e = Caml_js_exceptions.internalToOCamlException(exn);
+  if (e.RE_EXN_ID === Js_exn.$$Error) {
+    return e._1;
+  } else {
+    return e;
+  }
+}
+
 exports.delay = delay;
 exports.$$Object = $$Object;
 exports.$$Error = $$Error;
@@ -548,4 +558,5 @@ exports.$$WeakMap = $$WeakMap;
 exports.$$Map = $$Map;
 exports.$$Proxy = $$Proxy;
 exports.Hash = Hash;
+exports.prettifyExn = prettifyExn;
 /* variantTag Not a pure module */

package/src/db/EntityHistory.res

@@ -251,58 +251,57 @@ let fromTable = (table: table, ~pgSchema, ~schema: S.t<'entity>): t<'entity> =>
 
   let createInsertFnQuery = {
     `CREATE OR REPLACE FUNCTION ${insertFnName}(${historyRowArg} ${historyTablePath}, should_copy_current_entity BOOLEAN)
-      RETURNS void AS $$
-      DECLARE
-        v_previous_record RECORD;
-        v_origin_record RECORD;
-      BEGIN
-        -- Check if previous values are not provided
-        IF ${previousHistoryFieldsAreNullStr} THEN
-          -- Find the most recent record for the same id
-          SELECT ${currentChangeFieldNamesCommaSeparated} INTO v_previous_record
-          FROM ${historyTablePath}
-          WHERE ${id} = ${historyRowArg}.${id}
-          ORDER BY ${currentChangeFieldNames
+RETURNS void AS $$
+DECLARE
+  v_previous_record RECORD;
+  v_origin_record RECORD;
+BEGIN
+  -- Check if previous values are not provided
+  IF ${previousHistoryFieldsAreNullStr} THEN
+    -- Find the most recent record for the same id
+    SELECT ${currentChangeFieldNamesCommaSeparated} INTO v_previous_record
+    FROM ${historyTablePath}
+    WHERE ${id} = ${historyRowArg}.${id}
+    ORDER BY ${currentChangeFieldNames
       ->Belt.Array.map(fieldName => fieldName ++ " DESC")
       ->Js.Array2.joinWith(", ")}
-          LIMIT 1;
+    LIMIT 1;
 
-          -- If a previous record exists, use its values
-          IF FOUND THEN
-            ${Belt.Array.zip(currentChangeFieldNames, previousChangeFieldNames)
+    -- If a previous record exists, use its values
+    IF FOUND THEN
+      ${Belt.Array.zip(currentChangeFieldNames, previousChangeFieldNames)
       ->Belt.Array.map(((currentFieldName, previousFieldName)) => {
         `${historyRowArg}.${previousFieldName} := v_previous_record.${currentFieldName};`
       })
       ->Js.Array2.joinWith(" ")}
-            ElSIF should_copy_current_entity THEN
-            -- Check if a value for the id exists in the origin table and if so, insert a history row for it.
-            SELECT ${dataFieldNamesCommaSeparated} FROM ${originTablePath} WHERE id = ${historyRowArg}.${id} INTO v_origin_record;
-            IF FOUND THEN
-              INSERT INTO ${historyTablePath} (${currentChangeFieldNamesCommaSeparated}, ${dataFieldNamesCommaSeparated}, "${actionFieldName}")
-              -- SET the current change data fields to 0 since we don't know what they were
-              -- and it doesn't matter provided they are less than any new values
-              VALUES (${currentChangeFieldNames
+      ElSIF should_copy_current_entity THEN
+      -- Check if a value for the id exists in the origin table and if so, insert a history row for it.
+      SELECT ${dataFieldNamesCommaSeparated} FROM ${originTablePath} WHERE id = ${historyRowArg}.${id} INTO v_origin_record;
+      IF FOUND THEN
+        INSERT INTO ${historyTablePath} (${currentChangeFieldNamesCommaSeparated}, ${dataFieldNamesCommaSeparated}, "${actionFieldName}")
+        -- SET the current change data fields to 0 since we don't know what they were
+        -- and it doesn't matter provided they are less than any new values
+        VALUES (${currentChangeFieldNames
       ->Belt.Array.map(_ => "0")
       ->Js.Array2.joinWith(", ")}, ${dataFieldNames
       ->Belt.Array.map(fieldName => `v_origin_record."${fieldName}"`)
       ->Js.Array2.joinWith(", ")}, 'SET');
 
-              ${previousChangeFieldNames
+        ${previousChangeFieldNames
       ->Belt.Array.map(previousFieldName => {
         `${historyRowArg}.${previousFieldName} := 0;`
       })
       ->Js.Array2.joinWith(" ")}
-            END IF;
-          END IF;
-        END IF;
+      END IF;
+    END IF;
+  END IF;
 
-        INSERT INTO ${historyTablePath} (${allFieldNamesDoubleQuoted->Js.Array2.joinWith(", ")})
-        VALUES (${allFieldNamesDoubleQuoted
+  INSERT INTO ${historyTablePath} (${allFieldNamesDoubleQuoted->Js.Array2.joinWith(", ")})
+  VALUES (${allFieldNamesDoubleQuoted
       ->Belt.Array.map(fieldName => `${historyRowArg}.${fieldName}`)
       ->Js.Array2.joinWith(", ")});
-      END;
-      $$ LANGUAGE plpgsql;
-      `
+END;
+$$ LANGUAGE plpgsql;`
   }
 
   let insertFnString = `(sql, rowArgs, shouldCopyCurrentEntity) =>
@@ -318,3 +317,68 @@ let fromTable = (table: table, ~pgSchema, ~schema: S.t<'entity>): t<'entity> =>
 
   {table, createInsertFnQuery, schema, schemaRows: S.array(schema), insertFn}
 }
+
+type safeReorgBlocks = {
+  chainIds: array<int>,
+  blockNumbers: array<int>,
+}
+
+// We want to keep only the minimum history needed to survive chain reorgs and delete everything older.
+// Each chain gives us a "safe block": we assume reorgs will never happen at that block.
+//
+// What we keep per entity id:
+// - The latest history row at or before the safe block (the "anchor"). This is the last state that could
+//   ever be relevant during a rollback.
+// - If there are history rows in reorg threshold (after the safe block), we keep the anchor and delete all older rows.
+// - If there are no history rows in reorg threshold (after the safe block), even the anchor is redundant, so we delete it too.
+//
+// Why this is safe:
+// - Rollbacks will not cross the safe block, so rows older than the anchor can never be referenced again.
+// - If nothing changed in reorg threshold (after the safe block), the current state for that id can be reconstructed from the
+//   origin table; we do not need a pre-safe anchor for it.
+//
+// Performance notes:
+// - Multi-chain batching: inputs are expanded with unnest, letting one prepared statement prune many chains and
+//   enabling the planner to use indexes per chain_id efficiently.
+// - Minimal row touches: we only compute keep_serial per id and delete strictly older rows; this reduces write
+//   amplification and vacuum pressure compared to broad time-based purges.
+// - Contention-awareness: the DELETE joins on ids first, narrowing target rows early to limit locking and buffer churn.
+let makePruneStaleEntityHistoryQuery = (~entityName, ~pgSchema) => {
+  let historyTableName = entityName ++ "_history"
+  let historyTableRef = `"${pgSchema}"."${historyTableName}"`
+
+  `WITH safe AS (
+  SELECT s.chain_id, s.block_number
+  FROM unnest($1::int[], $2::bigint[]) AS s(chain_id, block_number)
+),
+max_before_safe AS (
+  SELECT t.id, MAX(t.serial) AS keep_serial
+  FROM ${historyTableRef} t
+  JOIN safe s
+    ON s.chain_id = t.entity_history_chain_id
+   AND t.entity_history_block_number <= s.block_number
+  GROUP BY t.id
+),
+post_safe AS (
+  SELECT DISTINCT t.id
+  FROM ${historyTableRef} t
+  JOIN safe s
+    ON s.chain_id = t.entity_history_chain_id
+   AND t.entity_history_block_number > s.block_number
+)
+DELETE FROM ${historyTableRef} d
+USING max_before_safe m
+LEFT JOIN post_safe p ON p.id = m.id
+WHERE d.id = m.id
+  AND (
+    d.serial < m.keep_serial
+    OR (p.id IS NULL AND d.serial = m.keep_serial)
+  );`
+}
+
+let pruneStaleEntityHistory = (sql, ~entityName, ~pgSchema, ~safeReorgBlocks): promise<unit> => {
+  sql->Postgres.preparedUnsafe(
+    makePruneStaleEntityHistoryQuery(~entityName, ~pgSchema),
+    (safeReorgBlocks.chainIds, safeReorgBlocks.blockNumbers)->Utils.magic,
+  )
+}

package/src/db/EntityHistory.res.js

@@ -252,19 +252,19 @@ function fromTable(table, pgSchema, schema) {
           ]), (function (fieldName) {
           return "\"" + fieldName + "\"";
         }));
-  var createInsertFnQuery = "CREATE OR REPLACE FUNCTION " + insertFnName + "(" + historyRowArg + " " + historyTablePath + ", should_copy_current_entity BOOLEAN)\n      RETURNS void AS $$\n      DECLARE\n        v_previous_record RECORD;\n        v_origin_record RECORD;\n      BEGIN\n        -- Check if previous values are not provided\n        IF " + previousHistoryFieldsAreNullStr + " THEN\n          -- Find the most recent record for the same id\n          SELECT " + currentChangeFieldNamesCommaSeparated + " INTO v_previous_record\n          FROM " + historyTablePath + "\n          WHERE " + id + " = " + historyRowArg + "." + id + "\n          ORDER BY " + Belt_Array.map(currentChangeFieldNames, (function (fieldName) {
+  var createInsertFnQuery = "CREATE OR REPLACE FUNCTION " + insertFnName + "(" + historyRowArg + " " + historyTablePath + ", should_copy_current_entity BOOLEAN)\nRETURNS void AS $$\nDECLARE\n  v_previous_record RECORD;\n  v_origin_record RECORD;\nBEGIN\n  -- Check if previous values are not provided\n  IF " + previousHistoryFieldsAreNullStr + " THEN\n    -- Find the most recent record for the same id\n    SELECT " + currentChangeFieldNamesCommaSeparated + " INTO v_previous_record\n    FROM " + historyTablePath + "\n    WHERE " + id + " = " + historyRowArg + "." + id + "\n    ORDER BY " + Belt_Array.map(currentChangeFieldNames, (function (fieldName) {
             return fieldName + " DESC";
-          })).join(", ") + "\n          LIMIT 1;\n\n          -- If a previous record exists, use its values\n          IF FOUND THEN\n            " + Belt_Array.map(Belt_Array.zip(currentChangeFieldNames, previousChangeFieldNames), (function (param) {
+          })).join(", ") + "\n    LIMIT 1;\n\n    -- If a previous record exists, use its values\n    IF FOUND THEN\n      " + Belt_Array.map(Belt_Array.zip(currentChangeFieldNames, previousChangeFieldNames), (function (param) {
             return historyRowArg + "." + param[1] + " := v_previous_record." + param[0] + ";";
-          })).join(" ") + "\n            ElSIF should_copy_current_entity THEN\n            -- Check if a value for the id exists in the origin table and if so, insert a history row for it.\n            SELECT " + dataFieldNamesCommaSeparated + " FROM " + originTablePath + " WHERE id = " + historyRowArg + "." + id + " INTO v_origin_record;\n            IF FOUND THEN\n              INSERT INTO " + historyTablePath + " (" + currentChangeFieldNamesCommaSeparated + ", " + dataFieldNamesCommaSeparated + ", \"" + actionFieldName + "\")\n              -- SET the current change data fields to 0 since we don't know what they were\n              -- and it doesn't matter provided they are less than any new values\n              VALUES (" + Belt_Array.map(currentChangeFieldNames, (function (param) {
+          })).join(" ") + "\n      ElSIF should_copy_current_entity THEN\n      -- Check if a value for the id exists in the origin table and if so, insert a history row for it.\n      SELECT " + dataFieldNamesCommaSeparated + " FROM " + originTablePath + " WHERE id = " + historyRowArg + "." + id + " INTO v_origin_record;\n      IF FOUND THEN\n        INSERT INTO " + historyTablePath + " (" + currentChangeFieldNamesCommaSeparated + ", " + dataFieldNamesCommaSeparated + ", \"" + actionFieldName + "\")\n        -- SET the current change data fields to 0 since we don't know what they were\n        -- and it doesn't matter provided they are less than any new values\n        VALUES (" + Belt_Array.map(currentChangeFieldNames, (function (param) {
             return "0";
           })).join(", ") + ", " + Belt_Array.map(dataFieldNames, (function (fieldName) {
             return "v_origin_record.\"" + fieldName + "\"";
-          })).join(", ") + ", 'SET');\n\n              " + Belt_Array.map(previousChangeFieldNames, (function (previousFieldName) {
+          })).join(", ") + ", 'SET');\n\n        " + Belt_Array.map(previousChangeFieldNames, (function (previousFieldName) {
             return historyRowArg + "." + previousFieldName + " := 0;";
-          })).join(" ") + "\n            END IF;\n          END IF;\n        END IF;\n\n        INSERT INTO " + historyTablePath + " (" + allFieldNamesDoubleQuoted.join(", ") + ")\n        VALUES (" + Belt_Array.map(allFieldNamesDoubleQuoted, (function (fieldName) {
+          })).join(" ") + "\n      END IF;\n    END IF;\n  END IF;\n\n  INSERT INTO " + historyTablePath + " (" + allFieldNamesDoubleQuoted.join(", ") + ")\n  VALUES (" + Belt_Array.map(allFieldNamesDoubleQuoted, (function (fieldName) {
             return historyRowArg + "." + fieldName;
-          })).join(", ") + ");\n      END;\n      $$ LANGUAGE plpgsql;\n      ";
+          })).join(", ") + ");\nEND;\n$$ LANGUAGE plpgsql;";
   var insertFnString = "(sql, rowArgs, shouldCopyCurrentEntity) =>\n      sql\`select " + insertFnName + "(ROW(" + Belt_Array.map(allFieldNamesDoubleQuoted, (function (fieldNameDoubleQuoted) {
             return "\${rowArgs[" + fieldNameDoubleQuoted + "]\}";
           })).join(", ") + ", NULL),  --NULL argument for SERIAL field\n    \${shouldCopyCurrentEntity});\`";
@@ -279,10 +279,25 @@ function fromTable(table, pgSchema, schema) {
         };
 }
 
+function makePruneStaleEntityHistoryQuery(entityName, pgSchema) {
+  var historyTableName = entityName + "_history";
+  var historyTableRef = "\"" + pgSchema + "\".\"" + historyTableName + "\"";
+  return "WITH safe AS (\n  SELECT s.chain_id, s.block_number\n  FROM unnest($1::int[], $2::bigint[]) AS s(chain_id, block_number)\n),\nmax_before_safe AS (\n  SELECT t.id, MAX(t.serial) AS keep_serial\n  FROM " + historyTableRef + " t\n  JOIN safe s\n    ON s.chain_id = t.entity_history_chain_id\n   AND t.entity_history_block_number <= s.block_number\n  GROUP BY t.id\n),\npost_safe AS (\n  SELECT DISTINCT t.id\n  FROM " + historyTableRef + " t\n  JOIN safe s\n    ON s.chain_id = t.entity_history_chain_id\n   AND t.entity_history_block_number > s.block_number\n)\nDELETE FROM " + historyTableRef + " d\nUSING max_before_safe m\nLEFT JOIN post_safe p ON p.id = m.id\nWHERE d.id = m.id\n  AND (\n    d.serial < m.keep_serial\n    OR (p.id IS NULL AND d.serial = m.keep_serial)\n  );";
+}
+
+function pruneStaleEntityHistory(sql, entityName, pgSchema, safeReorgBlocks) {
+  return sql.unsafe(makePruneStaleEntityHistoryQuery(entityName, pgSchema), [
+              safeReorgBlocks.chainIds,
+              safeReorgBlocks.blockNumbers
+            ], {prepare: true});
+}
+
 exports.RowAction = RowAction;
 exports.entityIdOnlySchema = entityIdOnlySchema;
 exports.previousHistoryFieldsSchema = previousHistoryFieldsSchema;
 exports.currentHistoryFieldsSchema = currentHistoryFieldsSchema;
 exports.makeHistoryRowSchema = makeHistoryRowSchema;
 exports.fromTable = fromTable;
+exports.makePruneStaleEntityHistoryQuery = makePruneStaleEntityHistoryQuery;
+exports.pruneStaleEntityHistory = pruneStaleEntityHistory;
 /* schema Not a pure module */

package/src/db/Table.res

@@ -240,6 +240,7 @@ let toSqlParams = (table: table, ~schema, ~pgSchema) => {
           switch field {
           | Field(f) =>
             switch f.fieldType {
+            // The case for `BigDecimal! @config(precision: 10, scale: 8)`
             | Custom(fieldType) if fieldType->Js.String2.startsWith("NUMERIC(") => fieldType
             | Custom(fieldType) => `${(Text :> string)}[]::"${pgSchema}".${(fieldType :> string)}`
             | Boolean => `${(Integer :> string)}[]::${(f.fieldType :> string)}`

package/src/sources/SourceManager.res

@@ -198,7 +198,7 @@ let getSourceNewHeight = async (
       logger->Logging.childTrace({
         "msg": `Height retrieval from ${source.name} source failed. Retrying in ${retryInterval->Int.toString}ms.`,
         "source": source.name,
-        "err": exn->Internal.prettifyExn,
+        "err": exn->Utils.prettifyExn,
       })
       retry := retry.contents + 1
       await Utils.delay(retryInterval)
@@ -410,7 +410,7 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
             | FailedParsingItems({exn, message, blockNumber, logIndex}) =>
               logger->Logging.childError({
                 "msg": message,
-                "err": exn->Internal.prettifyExn,
+                "err": exn->Utils.prettifyExn,
                 "blockNumber": blockNumber,
                 "logIndex": logIndex,
               })
@@ -469,7 +469,7 @@ let executeQuery = async (sourceManager: t, ~query: FetchState.query, ~currentBl
           "toBlock": attemptedToBlock,
           "backOffMilliseconds": backoffMillis,
           "retry": retry,
-          "err": exn->Internal.prettifyExn,
+          "err": exn->Utils.prettifyExn,
         })
 
         let shouldSwitch = nextSource !== source

package/src/sources/SourceManager.res.js

@@ -7,7 +7,6 @@ var Hrtime = require("../bindings/Hrtime.res.js");
 var Js_exn = require("rescript/lib/js/js_exn.js");
 var Source = require("./Source.res.js");
 var Logging = require("../Logging.res.js");
-var Internal = require("../Internal.res.js");
 var Belt_Array = require("rescript/lib/js/belt_Array.js");
 var FetchState = require("../FetchState.res.js");
 var Prometheus = require("../Prometheus.res.js");
@@ -142,7 +141,7 @@ async function getSourceNewHeight(sourceManager, source, currentBlockHeight, sta
       Logging.childTrace(logger, {
             msg: "Height retrieval from " + source.name + " source failed. Retrying in " + String(retryInterval) + "ms.",
             source: source.name,
-            err: Internal.prettifyExn(exn)
+            err: Utils.prettifyExn(exn)
           });
       retry = retry + 1 | 0;
       await Utils.delay(retryInterval);
@@ -298,7 +297,7 @@ async function executeQuery(sourceManager, query, currentBlockHeight) {
                   toBlock: attemptedToBlock,
                   backOffMilliseconds: backoffMillis,
                   retry: retry,
-                  err: Internal.prettifyExn(error$1.exn)
+                  err: Utils.prettifyExn(error$1.exn)
                 });
             var shouldSwitch = nextSource !== source;
             if (shouldSwitch) {
@@ -332,7 +331,7 @@ async function executeQuery(sourceManager, query, currentBlockHeight) {
             if (exit === 1) {
               Logging.childError(logger, {
                     msg: error$1.message,
-                    err: Internal.prettifyExn(error$1.exn),
+                    err: Utils.prettifyExn(error$1.exn),
                     blockNumber: error$1.blockNumber,
                     logIndex: error$1.logIndex
                   });