envio 2.27.5 → 2.28.0-alpha.1

package/src/db/EntityHistory.res

@@ -141,7 +141,7 @@ let makeHistoryRowSchema: S.t<'entity> => S.t<historyRow<'entity>> = entitySchem
 
 type t<'entity> = {
   table: table,
-  createInsertFnQuery: string,
+  makeInsertFnQuery: (~pgSchema: string) => string,
   schema: S.t<historyRow<'entity>>,
   // Used for parsing
   schemaRows: S.t<array<historyRow<'entity>>>,
@@ -153,7 +153,7 @@ type entityInternal
 external castInternal: t<'entity> => t<entityInternal> = "%identity"
 external eval: string => 'a = "eval"
 
-let fromTable = (table: table, ~pgSchema, ~schema: S.t<'entity>): t<'entity> => {
+let fromTable = (table: table, ~schema: S.t<'entity>): t<'entity> => {
   let entity_history_block_timestamp = "entity_history_block_timestamp"
   let entity_history_chain_id = "entity_history_chain_id"
   let entity_history_block_number = "entity_history_block_number"
@@ -227,19 +227,6 @@ let fromTable = (table: table, ~pgSchema, ~schema: S.t<'entity>): t<'entity> =>
   )
 
   let insertFnName = `"insert_${table.tableName}"`
-  let historyRowArg = "history_row"
-  let historyTablePath = `"${pgSchema}"."${historyTableName}"`
-  let originTablePath = `"${pgSchema}"."${originTableName}"`
-
-  let previousHistoryFieldsAreNullStr =
-    previousChangeFieldNames
-    ->Belt.Array.map(fieldName => `${historyRowArg}.${fieldName} IS NULL`)
-    ->Js.Array2.joinWith(" OR ")
-
-  let currentChangeFieldNamesCommaSeparated = currentChangeFieldNames->Js.Array2.joinWith(", ")
-
-  let dataFieldNamesDoubleQuoted = dataFieldNames->Belt.Array.map(fieldName => `"${fieldName}"`)
-  let dataFieldNamesCommaSeparated = dataFieldNamesDoubleQuoted->Js.Array2.joinWith(", ")
 
   let allFieldNamesDoubleQuoted =
     Belt.Array.concatMany([
@@ -249,60 +236,73 @@ let fromTable = (table: table, ~pgSchema, ~schema: S.t<'entity>): t<'entity> =>
       [actionFieldName],
     ])->Belt.Array.map(fieldName => `"${fieldName}"`)
 
-  let createInsertFnQuery = {
+  let makeInsertFnQuery = (~pgSchema) => {
+    let historyRowArg = "history_row"
+    let historyTablePath = `"${pgSchema}"."${historyTableName}"`
+    let originTablePath = `"${pgSchema}"."${originTableName}"`
+
+    let previousHistoryFieldsAreNullStr =
+      previousChangeFieldNames
+      ->Belt.Array.map(fieldName => `${historyRowArg}.${fieldName} IS NULL`)
+      ->Js.Array2.joinWith(" OR ")
+
+    let currentChangeFieldNamesCommaSeparated = currentChangeFieldNames->Js.Array2.joinWith(", ")
+
+    let dataFieldNamesDoubleQuoted = dataFieldNames->Belt.Array.map(fieldName => `"${fieldName}"`)
+    let dataFieldNamesCommaSeparated = dataFieldNamesDoubleQuoted->Js.Array2.joinWith(", ")
+
     `CREATE OR REPLACE FUNCTION ${insertFnName}(${historyRowArg} ${historyTablePath}, should_copy_current_entity BOOLEAN)
-      RETURNS void AS $$
-      DECLARE
-        v_previous_record RECORD;
-        v_origin_record RECORD;
-      BEGIN
-        -- Check if previous values are not provided
-        IF ${previousHistoryFieldsAreNullStr} THEN
-          -- Find the most recent record for the same id
-          SELECT ${currentChangeFieldNamesCommaSeparated} INTO v_previous_record
-          FROM ${historyTablePath}
-          WHERE ${id} = ${historyRowArg}.${id}
-          ORDER BY ${currentChangeFieldNames
+RETURNS void AS $$
+DECLARE
+  v_previous_record RECORD;
+  v_origin_record RECORD;
+BEGIN
+  -- Check if previous values are not provided
+  IF ${previousHistoryFieldsAreNullStr} THEN
+    -- Find the most recent record for the same id
+    SELECT ${currentChangeFieldNamesCommaSeparated} INTO v_previous_record
+    FROM ${historyTablePath}
+    WHERE ${id} = ${historyRowArg}.${id}
+    ORDER BY ${currentChangeFieldNames
       ->Belt.Array.map(fieldName => fieldName ++ " DESC")
       ->Js.Array2.joinWith(", ")}
-          LIMIT 1;
+    LIMIT 1;
 
-          -- If a previous record exists, use its values
-          IF FOUND THEN
-            ${Belt.Array.zip(currentChangeFieldNames, previousChangeFieldNames)
+    -- If a previous record exists, use its values
+    IF FOUND THEN
+      ${Belt.Array.zip(currentChangeFieldNames, previousChangeFieldNames)
       ->Belt.Array.map(((currentFieldName, previousFieldName)) => {
         `${historyRowArg}.${previousFieldName} := v_previous_record.${currentFieldName};`
       })
       ->Js.Array2.joinWith(" ")}
-            ElSIF should_copy_current_entity THEN
-            -- Check if a value for the id exists in the origin table and if so, insert a history row for it.
-            SELECT ${dataFieldNamesCommaSeparated} FROM ${originTablePath} WHERE id = ${historyRowArg}.${id} INTO v_origin_record;
-            IF FOUND THEN
-              INSERT INTO ${historyTablePath} (${currentChangeFieldNamesCommaSeparated}, ${dataFieldNamesCommaSeparated}, "${actionFieldName}")
-              -- SET the current change data fields to 0 since we don't know what they were
-              -- and it doesn't matter provided they are less than any new values
-              VALUES (${currentChangeFieldNames
+      ElSIF should_copy_current_entity THEN
+      -- Check if a value for the id exists in the origin table and if so, insert a history row for it.
+      SELECT ${dataFieldNamesCommaSeparated} FROM ${originTablePath} WHERE id = ${historyRowArg}.${id} INTO v_origin_record;
+      IF FOUND THEN
+        INSERT INTO ${historyTablePath} (${currentChangeFieldNamesCommaSeparated}, ${dataFieldNamesCommaSeparated}, "${actionFieldName}")
+        -- SET the current change data fields to 0 since we don't know what they were
+        -- and it doesn't matter provided they are less than any new values
+        VALUES (${currentChangeFieldNames
       ->Belt.Array.map(_ => "0")
       ->Js.Array2.joinWith(", ")}, ${dataFieldNames
       ->Belt.Array.map(fieldName => `v_origin_record."${fieldName}"`)
       ->Js.Array2.joinWith(", ")}, 'SET');
 
-              ${previousChangeFieldNames
+        ${previousChangeFieldNames
       ->Belt.Array.map(previousFieldName => {
         `${historyRowArg}.${previousFieldName} := 0;`
       })
       ->Js.Array2.joinWith(" ")}
-            END IF;
-          END IF;
-        END IF;
+      END IF;
+    END IF;
+  END IF;
 
-        INSERT INTO ${historyTablePath} (${allFieldNamesDoubleQuoted->Js.Array2.joinWith(", ")})
-        VALUES (${allFieldNamesDoubleQuoted
+  INSERT INTO ${historyTablePath} (${allFieldNamesDoubleQuoted->Js.Array2.joinWith(", ")})
+  VALUES (${allFieldNamesDoubleQuoted
       ->Belt.Array.map(fieldName => `${historyRowArg}.${fieldName}`)
       ->Js.Array2.joinWith(", ")});
-      END;
-      $$ LANGUAGE plpgsql;
-      `
+END;
+$$ LANGUAGE plpgsql;`
   }
 
   let insertFnString = `(sql, rowArgs, shouldCopyCurrentEntity) =>
@@ -316,5 +316,70 @@ let fromTable = (table: table, ~pgSchema, ~schema: S.t<'entity>): t<'entity> =>
 
   let schema = makeHistoryRowSchema(schema)
 
-  {table, createInsertFnQuery, schema, schemaRows: S.array(schema), insertFn}
+  {table, makeInsertFnQuery, schema, schemaRows: S.array(schema), insertFn}
+}
+
+type safeReorgBlocks = {
+  chainIds: array<int>,
+  blockNumbers: array<int>,
+}
+
+// We want to keep only the minimum history needed to survive chain reorgs and delete everything older.
+// Each chain gives us a "safe block": we assume reorgs will never happen at that block.
+//
+// What we keep per entity id:
+// - The latest history row at or before the safe block (the "anchor"). This is the last state that could
+//   ever be relevant during a rollback.
+// - If there are history rows in reorg threshold (after the safe block), we keep the anchor and delete all older rows.
+// - If there are no history rows in reorg threshold (after the safe block), even the anchor is redundant, so we delete it too.
+//
+// Why this is safe:
+// - Rollbacks will not cross the safe block, so rows older than the anchor can never be referenced again.
+// - If nothing changed in reorg threshold (after the safe block), the current state for that id can be reconstructed from the
+//   origin table; we do not need a pre-safe anchor for it.
+//
+// Performance notes:
+// - Multi-chain batching: inputs are expanded with unnest, letting one prepared statement prune many chains and
+//   enabling the planner to use indexes per chain_id efficiently.
+// - Minimal row touches: we only compute keep_serial per id and delete strictly older rows; this reduces write
+//   amplification and vacuum pressure compared to broad time-based purges.
+// - Contention-awareness: the DELETE joins on ids first, narrowing target rows early to limit locking and buffer churn.
+let makePruneStaleEntityHistoryQuery = (~entityName, ~pgSchema) => {
+  let historyTableName = entityName ++ "_history"
+  let historyTableRef = `"${pgSchema}"."${historyTableName}"`
+
+  `WITH safe AS (
+  SELECT s.chain_id, s.block_number
+  FROM unnest($1::int[], $2::bigint[]) AS s(chain_id, block_number)
+),
+max_before_safe AS (
+  SELECT t.id, MAX(t.serial) AS keep_serial
+  FROM ${historyTableRef} t
+  JOIN safe s
+    ON s.chain_id = t.entity_history_chain_id
+   AND t.entity_history_block_number <= s.block_number
+  GROUP BY t.id
+),
+post_safe AS (
+  SELECT DISTINCT t.id
+  FROM ${historyTableRef} t
+  JOIN safe s
+    ON s.chain_id = t.entity_history_chain_id
+   AND t.entity_history_block_number > s.block_number
+)
+DELETE FROM ${historyTableRef} d
+USING max_before_safe m
+LEFT JOIN post_safe p ON p.id = m.id
+WHERE d.id = m.id
+  AND (
+    d.serial < m.keep_serial
+    OR (p.id IS NULL AND d.serial = m.keep_serial)
+  );`
+}
+
+let pruneStaleEntityHistory = (sql, ~entityName, ~pgSchema, ~safeReorgBlocks): promise<unit> => {
+  sql->Postgres.preparedUnsafe(
+    makePruneStaleEntityHistoryQuery(~entityName, ~pgSchema),
+    (safeReorgBlocks.chainIds, safeReorgBlocks.blockNumbers)->Utils.magic,
+  )
 }

package/src/db/EntityHistory.res.js

@@ -157,7 +157,7 @@ function makeHistoryRowSchema(entitySchema) {
               }));
 }
 
-function fromTable(table, pgSchema, schema) {
+function fromTable(table, schema) {
   var currentChangeFieldNames = [
     "entity_history_block_timestamp",
     "entity_history_chain_id",
@@ -233,17 +233,6 @@ function fromTable(table, pgSchema, schema) {
             ]
           ]));
   var insertFnName = "\"insert_" + table$1.tableName + "\"";
-  var historyRowArg = "history_row";
-  var historyTablePath = "\"" + pgSchema + "\".\"" + historyTableName + "\"";
-  var originTablePath = "\"" + pgSchema + "\".\"" + originTableName + "\"";
-  var previousHistoryFieldsAreNullStr = Belt_Array.map(previousChangeFieldNames, (function (fieldName) {
-            return historyRowArg + "." + fieldName + " IS NULL";
-          })).join(" OR ");
-  var currentChangeFieldNamesCommaSeparated = currentChangeFieldNames.join(", ");
-  var dataFieldNamesDoubleQuoted = Belt_Array.map(dataFieldNames, (function (fieldName) {
-          return "\"" + fieldName + "\"";
-        }));
-  var dataFieldNamesCommaSeparated = dataFieldNamesDoubleQuoted.join(", ");
   var allFieldNamesDoubleQuoted = Belt_Array.map(Belt_Array.concatMany([
             currentChangeFieldNames,
             previousChangeFieldNames,
@@ -252,19 +241,32 @@ function fromTable(table, pgSchema, schema) {
           ]), (function (fieldName) {
           return "\"" + fieldName + "\"";
         }));
-  var createInsertFnQuery = "CREATE OR REPLACE FUNCTION " + insertFnName + "(" + historyRowArg + " " + historyTablePath + ", should_copy_current_entity BOOLEAN)\n      RETURNS void AS $$\n      DECLARE\n        v_previous_record RECORD;\n        v_origin_record RECORD;\n      BEGIN\n        -- Check if previous values are not provided\n        IF " + previousHistoryFieldsAreNullStr + " THEN\n          -- Find the most recent record for the same id\n          SELECT " + currentChangeFieldNamesCommaSeparated + " INTO v_previous_record\n          FROM " + historyTablePath + "\n          WHERE " + id + " = " + historyRowArg + "." + id + "\n          ORDER BY " + Belt_Array.map(currentChangeFieldNames, (function (fieldName) {
-            return fieldName + " DESC";
-          })).join(", ") + "\n          LIMIT 1;\n\n          -- If a previous record exists, use its values\n          IF FOUND THEN\n            " + Belt_Array.map(Belt_Array.zip(currentChangeFieldNames, previousChangeFieldNames), (function (param) {
-            return historyRowArg + "." + param[1] + " := v_previous_record." + param[0] + ";";
-          })).join(" ") + "\n            ElSIF should_copy_current_entity THEN\n            -- Check if a value for the id exists in the origin table and if so, insert a history row for it.\n            SELECT " + dataFieldNamesCommaSeparated + " FROM " + originTablePath + " WHERE id = " + historyRowArg + "." + id + " INTO v_origin_record;\n            IF FOUND THEN\n              INSERT INTO " + historyTablePath + " (" + currentChangeFieldNamesCommaSeparated + ", " + dataFieldNamesCommaSeparated + ", \"" + actionFieldName + "\")\n              -- SET the current change data fields to 0 since we don't know what they were\n              -- and it doesn't matter provided they are less than any new values\n              VALUES (" + Belt_Array.map(currentChangeFieldNames, (function (param) {
-            return "0";
-          })).join(", ") + ", " + Belt_Array.map(dataFieldNames, (function (fieldName) {
-            return "v_origin_record.\"" + fieldName + "\"";
-          })).join(", ") + ", 'SET');\n\n              " + Belt_Array.map(previousChangeFieldNames, (function (previousFieldName) {
-            return historyRowArg + "." + previousFieldName + " := 0;";
-          })).join(" ") + "\n            END IF;\n          END IF;\n        END IF;\n\n        INSERT INTO " + historyTablePath + " (" + allFieldNamesDoubleQuoted.join(", ") + ")\n        VALUES (" + Belt_Array.map(allFieldNamesDoubleQuoted, (function (fieldName) {
-            return historyRowArg + "." + fieldName;
-          })).join(", ") + ");\n      END;\n      $$ LANGUAGE plpgsql;\n      ";
+  var makeInsertFnQuery = function (pgSchema) {
+    var historyRowArg = "history_row";
+    var historyTablePath = "\"" + pgSchema + "\".\"" + historyTableName + "\"";
+    var originTablePath = "\"" + pgSchema + "\".\"" + originTableName + "\"";
+    var previousHistoryFieldsAreNullStr = Belt_Array.map(previousChangeFieldNames, (function (fieldName) {
+              return historyRowArg + "." + fieldName + " IS NULL";
+            })).join(" OR ");
+    var currentChangeFieldNamesCommaSeparated = currentChangeFieldNames.join(", ");
+    var dataFieldNamesDoubleQuoted = Belt_Array.map(dataFieldNames, (function (fieldName) {
+            return "\"" + fieldName + "\"";
+          }));
+    var dataFieldNamesCommaSeparated = dataFieldNamesDoubleQuoted.join(", ");
+    return "CREATE OR REPLACE FUNCTION " + insertFnName + "(" + historyRowArg + " " + historyTablePath + ", should_copy_current_entity BOOLEAN)\nRETURNS void AS $$\nDECLARE\n  v_previous_record RECORD;\n  v_origin_record RECORD;\nBEGIN\n  -- Check if previous values are not provided\n  IF " + previousHistoryFieldsAreNullStr + " THEN\n    -- Find the most recent record for the same id\n    SELECT " + currentChangeFieldNamesCommaSeparated + " INTO v_previous_record\n    FROM " + historyTablePath + "\n    WHERE " + id + " = " + historyRowArg + "." + id + "\n    ORDER BY " + Belt_Array.map(currentChangeFieldNames, (function (fieldName) {
+                    return fieldName + " DESC";
+                  })).join(", ") + "\n    LIMIT 1;\n\n    -- If a previous record exists, use its values\n    IF FOUND THEN\n      " + Belt_Array.map(Belt_Array.zip(currentChangeFieldNames, previousChangeFieldNames), (function (param) {
+                    return historyRowArg + "." + param[1] + " := v_previous_record." + param[0] + ";";
+                  })).join(" ") + "\n      ElSIF should_copy_current_entity THEN\n      -- Check if a value for the id exists in the origin table and if so, insert a history row for it.\n      SELECT " + dataFieldNamesCommaSeparated + " FROM " + originTablePath + " WHERE id = " + historyRowArg + "." + id + " INTO v_origin_record;\n      IF FOUND THEN\n        INSERT INTO " + historyTablePath + " (" + currentChangeFieldNamesCommaSeparated + ", " + dataFieldNamesCommaSeparated + ", \"" + actionFieldName + "\")\n        -- SET the current change data fields to 0 since we don't know what they were\n        -- and it doesn't matter provided they are less than any new values\n        VALUES (" + Belt_Array.map(currentChangeFieldNames, (function (param) {
+                    return "0";
+                  })).join(", ") + ", " + Belt_Array.map(dataFieldNames, (function (fieldName) {
+                    return "v_origin_record.\"" + fieldName + "\"";
+                  })).join(", ") + ", 'SET');\n\n        " + Belt_Array.map(previousChangeFieldNames, (function (previousFieldName) {
+                    return historyRowArg + "." + previousFieldName + " := 0;";
+                  })).join(" ") + "\n      END IF;\n    END IF;\n  END IF;\n\n  INSERT INTO " + historyTablePath + " (" + allFieldNamesDoubleQuoted.join(", ") + ")\n  VALUES (" + Belt_Array.map(allFieldNamesDoubleQuoted, (function (fieldName) {
+                    return historyRowArg + "." + fieldName;
+                  })).join(", ") + ");\nEND;\n$$ LANGUAGE plpgsql;";
+  };
   var insertFnString = "(sql, rowArgs, shouldCopyCurrentEntity) =>\n      sql\`select " + insertFnName + "(ROW(" + Belt_Array.map(allFieldNamesDoubleQuoted, (function (fieldNameDoubleQuoted) {
             return "\${rowArgs[" + fieldNameDoubleQuoted + "]\}";
           })).join(", ") + ", NULL),  --NULL argument for SERIAL field\n    \${shouldCopyCurrentEntity});\`";
@@ -272,17 +274,32 @@ function fromTable(table, pgSchema, schema) {
   var schema$1 = makeHistoryRowSchema(schema);
   return {
           table: table$1,
-          createInsertFnQuery: createInsertFnQuery,
+          makeInsertFnQuery: makeInsertFnQuery,
           schema: schema$1,
           schemaRows: S$RescriptSchema.array(schema$1),
           insertFn: insertFn
         };
 }
 
+function makePruneStaleEntityHistoryQuery(entityName, pgSchema) {
+  var historyTableName = entityName + "_history";
+  var historyTableRef = "\"" + pgSchema + "\".\"" + historyTableName + "\"";
+  return "WITH safe AS (\n  SELECT s.chain_id, s.block_number\n  FROM unnest($1::int[], $2::bigint[]) AS s(chain_id, block_number)\n),\nmax_before_safe AS (\n  SELECT t.id, MAX(t.serial) AS keep_serial\n  FROM " + historyTableRef + " t\n  JOIN safe s\n    ON s.chain_id = t.entity_history_chain_id\n   AND t.entity_history_block_number <= s.block_number\n  GROUP BY t.id\n),\npost_safe AS (\n  SELECT DISTINCT t.id\n  FROM " + historyTableRef + " t\n  JOIN safe s\n    ON s.chain_id = t.entity_history_chain_id\n   AND t.entity_history_block_number > s.block_number\n)\nDELETE FROM " + historyTableRef + " d\nUSING max_before_safe m\nLEFT JOIN post_safe p ON p.id = m.id\nWHERE d.id = m.id\n  AND (\n    d.serial < m.keep_serial\n    OR (p.id IS NULL AND d.serial = m.keep_serial)\n  );";
+}
+
+function pruneStaleEntityHistory(sql, entityName, pgSchema, safeReorgBlocks) {
+  return sql.unsafe(makePruneStaleEntityHistoryQuery(entityName, pgSchema), [
+              safeReorgBlocks.chainIds,
+              safeReorgBlocks.blockNumbers
+            ], {prepare: true});
+}
+
 exports.RowAction = RowAction;
 exports.entityIdOnlySchema = entityIdOnlySchema;
 exports.previousHistoryFieldsSchema = previousHistoryFieldsSchema;
 exports.currentHistoryFieldsSchema = currentHistoryFieldsSchema;
 exports.makeHistoryRowSchema = makeHistoryRowSchema;
 exports.fromTable = fromTable;
+exports.makePruneStaleEntityHistoryQuery = makePruneStaleEntityHistoryQuery;
+exports.pruneStaleEntityHistory = pruneStaleEntityHistory;
 /* schema Not a pure module */

package/src/db/InternalTable.gen.ts

@@ -0,0 +1,43 @@
+/* TypeScript file generated from InternalTable.res by genType. */
+
+/* eslint-disable */
+/* tslint:disable */
+
+import type {Json_t as Js_Json_t} from '../../src/Js.shim';
+
+import type {t as Address_t} from '../../src/Address.gen';
+
+export type EventSyncState_t = {
+  readonly chain_id: number; 
+  readonly block_number: number; 
+  readonly log_index: number; 
+  readonly block_timestamp: number
+};
+
+export type RawEvents_t = {
+  readonly chain_id: number; 
+  readonly event_id: bigint; 
+  readonly event_name: string; 
+  readonly contract_name: string; 
+  readonly block_number: number; 
+  readonly log_index: number; 
+  readonly src_address: Address_t; 
+  readonly block_hash: string; 
+  readonly block_timestamp: number; 
+  readonly block_fields: Js_Json_t; 
+  readonly transaction_fields: Js_Json_t; 
+  readonly params: Js_Json_t
+};
+
+export type DynamicContractRegistry_t = {
+  readonly id: string; 
+  readonly chain_id: number; 
+  readonly registering_event_block_number: number; 
+  readonly registering_event_log_index: number; 
+  readonly registering_event_block_timestamp: number; 
+  readonly registering_event_contract_name: string; 
+  readonly registering_event_name: string; 
+  readonly registering_event_src_address: Address_t; 
+  readonly contract_address: Address_t; 
+  readonly contract_name: string
+};