envio 3.11.0 → 3.12.0

package/src/PgStorage.res.mjs

@@ -12,6 +12,7 @@ import * as Schema from "./db/Schema.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as EntityId from "./EntityId.res.mjs";
+import * as Frontier from "./db/Frontier.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import Postgres from "postgres";
 import * as ChainState from "./ChainState.res.mjs";
@@ -22,6 +23,7 @@ import * as IndexCatalog from "./db/IndexCatalog.res.mjs";
 import * as IndexManager from "./db/IndexManager.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
+import * as HistoryPolicy from "./HistoryPolicy.res.mjs";
 import * as InternalTable from "./db/InternalTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Child_process from "child_process";
@@ -29,10 +31,10 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
 import * as ContractMapping from "./ContractMapping.res.mjs";
 import * as IndexDefinition from "./db/IndexDefinition.res.mjs";
-import * as CheckpointBounds from "./db/CheckpointBounds.res.mjs";
 import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 function makeClient() {
@@ -57,7 +59,25 @@ function formatSeconds(timeRef) {
 
 let slowOnLargeDatabaseNotice = "This can take a long time on a large database.";
 
-function getSchemaIndexes(entities) {
+function partitionTableName(entityConfig, chainId) {
+  let chainIdStr = ChainId.toString(chainId);
+  return Table.fitPgTableName(entityConfig.table.tableName + `$` + chainIdStr, `$` + entityConfig.index.toString() + `$` + chainIdStr);
+}
+
+function indexTableNames(entityConfig, partitionChainIds) {
+  let match = Table.getChainIdField(entityConfig.table);
+  if (match !== undefined) {
+    if (partitionChainIds !== undefined) {
+      return partitionChainIds.map(chainId => partitionTableName(entityConfig, chainId));
+    } else {
+      return [entityConfig.table.tableName];
+    }
+  } else {
+    return [entityConfig.table.tableName];
+  }
+}
+
+function getSchemaIndexes(entities, partitionChainIds) {
   let derivedSchema = Schema.make(entities.map(e => e.table));
   let all = [];
   entities.forEach(param => {
@@ -76,6 +96,10 @@ function getSchemaIndexes(entities) {
     });
   });
   let seen = new Set();
+  let entityByTableName = {};
+  entities.forEach(entityConfig => {
+    entityByTableName[entityConfig.table.tableName] = entityConfig;
+  });
   return all.filter(definition => {
     let key = IndexDefinition.key(definition);
     if (seen.has(key)) {
@@ -84,7 +108,11 @@ function getSchemaIndexes(entities) {
       seen.add(key);
       return true;
     }
-  });
+  }).flatMap(definition => indexTableNames(Stdlib_Option.getOrThrow(entityByTableName[definition.tableName], undefined), partitionChainIds).map(tableName => ({
+    tableName: tableName,
+    columns: definition.columns,
+    method: definition.method
+  })));
 }
 
 function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText, chainIdModeOpt, partitionByColumn) {
@@ -108,11 +136,6 @@ function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText, chainIdMode
   ) + `;`;
 }
 
-function partitionTableName(entityConfig, chainId) {
-  let chainIdStr = ChainId.toString(chainId);
-  return Table.fitPgTableName(entityConfig.table.tableName + `$` + chainIdStr, `$` + entityConfig.index.toString() + `$` + chainIdStr);
-}
-
 let rowSchemaCache = new WeakMap();
 
 function getRowSchema(entityConfig) {
@@ -228,7 +251,7 @@ function makeCreateEntityTableQueries(entityConfig, pgSchema, isNumericArrayAsTe
   ).concat([createTable(getEntityHistory(entityConfig).table, undefined)]);
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt, deferSchemaIndexesOpt, chainIdModeOpt) {
+function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, checkpointSequence, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt, deferSchemaIndexesOpt, chainIdModeOpt) {
   let chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
   let entities = entitiesOpt !== undefined ? entitiesOpt : [];
   let enums = enumsOpt !== undefined ? enumsOpt : [];
@@ -240,12 +263,12 @@ function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfi
     InternalTable.EnvioInfo.table,
     InternalTable.EnvioContracts.table,
     InternalTable.EnvioAddresses.table,
-    InternalTable.Checkpoints.table,
+    InternalTable.Checkpoints.tableFor(checkpointSequence),
     InternalTable.RawEvents.table
   ];
   let chainIds = chainConfigs.map(chainConfig => chainConfig.id);
   let tableQueries = generalTables.map(table => makeCreateTableQuery(table, pgSchema, isHasuraEnabled, chainIdMode, undefined)).concat(entities.flatMap(entityConfig => makeCreateEntityTableQueries(entityConfig, pgSchema, isHasuraEnabled, chainIdMode, chainIds)));
-  let schemaIndexes = getSchemaIndexes(entities);
+  let schemaIndexes = getSchemaIndexes(entities, undefined);
   let query = {
     contents: (
       isEmptyPgSchema && pgSchema === "public" ? `CREATE SCHEMA IF NOT EXISTS "` + pgSchema + `";\n` : `DROP SCHEMA IF EXISTS "` + pgSchema + `" CASCADE;
@@ -687,10 +710,29 @@ function executeSet(sql, items, dbFunction) {
   }
 }
 
-async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, updatedEntities, registeredAddresses, sinkPromise, chainMetaData, escapeTables) {
+function pickCheckpoints(column, picked) {
+  if (typeof picked !== "object") {
+    return column;
+  } else {
+    return picked._0.map(index => column[index]);
+  }
+}
+
+async function writeBatch(sql, batch, pgSchema, rollback, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, updatedEntities, registeredAddresses, sinkPromise, chainMetaData, escapeTables) {
   try {
     let chainIdMode = config.chainIdMode;
-    let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold, undefined);
+    let indexes = Stdlib_Array.filterMapWithIndex(batch.checkpointChainIds, (chainId, index) => {
+      if (HistoryPolicy.forChain(batch.history, chainId)) {
+        return index;
+      }
+    });
+    let pickedCheckpoints = Utils.$$Array.isEmpty(indexes) ? undefined : (
+        indexes.length === batch.checkpointIds.length ? "AllCheckpoints" : ({
+            TAG: "CheckpointIndexes",
+            _0: indexes
+          })
+      );
+    let writtenFrontier = Persistence.writtenFrontier(batch, rollback);
     let specificError = {
       contents: undefined
     };
@@ -725,6 +767,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
       }
     };
     let setEntities = updatedEntities.map(param => {
+      let shouldSaveHistory = param.shouldSaveHistory;
       let changes = param.changes;
       let scope = param.scope;
       let entityConfig = param.entityConfig;
@@ -732,7 +775,6 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
       let idsToDelete = [];
       let scopeChainId;
       scopeChainId = scope === "crossChain" ? undefined : Primitive_option.some(scope);
-      let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold, scopeChainId);
       let match = Table.getChainIdField(entityConfig.table);
       let changes$1;
       if (match !== undefined && scopeChainId !== undefined) {
@@ -756,7 +798,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
       let chainIdParams = match$1 !== undefined ? (
           scopeChainId !== undefined ? [Primitive_option.valFromOption(scopeChainId)] : []
         ) : [];
-      let diffCheckpointId = Stdlib_Option.map(rollback, r => r.diffCheckpointId);
+      let diffCheckpointId = Stdlib_Option.flatMap(rollback, r => CheckpointSequence.findForScope(config.checkpointSequence, r.diffFrontier, scope));
       let batchSetUpdates = [];
       let batchDeleteEntityIds = [];
       let batchDeleteCheckpointIds = [];
@@ -872,6 +914,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
             sql => InternalTable.Chains.setProgressedChains(sql, pgSchema, Utils.Dict.mapValuesToArray(batch.progressedChainsById, chainAfterBatch => ({
               chainId: chainAfterBatch.fetchState.chainId,
               progressBlockNumber: chainAfterBatch.progressBlockNumber,
+              progressBlockTime: chainAfterBatch.progressBlockTime,
               sourceBlockNumber: chainAfterBatch.sourceBlockNumber,
               totalEventsProcessed: chainAfterBatch.totalEventsProcessed
             }))),
@@ -883,8 +926,11 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
           if (Utils.$$Array.notEmpty(registeredAddresses)) {
             setOperations.push(sql => InternalTable.EnvioAddresses.insert(sql, pgSchema, registeredAddresses.map(staged => staged.row), chainIdMode));
           }
-          if (shouldSaveHistory) {
-            setOperations.push(sql => InternalTable.Checkpoints.insert(sql, pgSchema, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointBlockHashes, batch.checkpointEventsProcessed, chainIdMode));
+          if (!Utils.Dict.isEmpty(writtenFrontier)) {
+            setOperations.push(sql => InternalTable.Chains.setCheckpointFrontier(sql, pgSchema, writtenFrontier, chainIdMode));
+          }
+          if (pickedCheckpoints !== undefined) {
+            setOperations.push(sql => InternalTable.Checkpoints.insert(sql, pgSchema, pickCheckpoints(batch.checkpointIds, pickedCheckpoints), pickCheckpoints(batch.checkpointChainIds, pickedCheckpoints), pickCheckpoints(batch.checkpointBlockNumbers, pickedCheckpoints), pickCheckpoints(batch.checkpointBlockHashes, pickedCheckpoints), pickCheckpoints(batch.checkpointEventsProcessed, pickedCheckpoints), chainIdMode));
           }
           await Promise.all(setOperations.map(dbFunc => dbFunc(sql)));
           if (sinkPromise === undefined) {
@@ -913,7 +959,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
     if (exn$1.RE_EXN_ID === PgEncodingError) {
       let escapeTables$1 = escapeTables !== undefined ? Primitive_option.valFromOption(escapeTables) : new Set();
       escapeTables$1.add(exn$1.table);
-      return await writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, updatedEntities, registeredAddresses, sinkPromise, chainMetaData, Primitive_option.some(escapeTables$1));
+      return await writeBatch(sql, batch, pgSchema, rollback, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, updatedEntities, registeredAddresses, sinkPromise, chainMetaData, Primitive_option.some(escapeTables$1));
     }
     throw exn$1;
   }
@@ -943,7 +989,7 @@ function makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema, floors) {
   let keyColumns = rollbackKeyColumns(entityConfig);
   let keyColumnsCommaSeparated = keyColumns.map(c => tableRef + `."` + c + `"`).join(", ");
   let keyMatch = keyColumns.map(c => `h."` + c + `" = ` + tableRef + `."` + c + `"`).join(" AND ");
-  let bounds = CheckpointBounds.sql(floors.floors, Table.getPgChainIdColumn(entityConfig.table), tableRef);
+  let bounds = CheckpointSequence.sql(floors.checkpointBounds, Table.getPgChainIdColumn(entityConfig.table), tableRef);
   return `SELECT DISTINCT ON (` + keyColumnsCommaSeparated + `) ` + dataFieldsCommaSeparated + `, ` + tableRef + `."` + EntityHistory.changeFieldName + `"
   FROM "` + pgSchema + `"."` + historyTableName + `"` + bounds.join + `
   WHERE ` + tableRef + `."` + EntityHistory.checkpointIdFieldName + `" <= ` + bounds.checkpointId + `
@@ -961,7 +1007,7 @@ function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema, floors) {
   let tableRef = `"` + historyTableName + `"`;
   let keyColumns = rollbackKeyColumns(entityConfig);
   let keyMatch = keyColumns.map(c => `h."` + c + `" = ` + tableRef + `."` + c + `"`).join(" AND ");
-  let bounds = CheckpointBounds.sql(floors.floors, Table.getPgChainIdColumn(entityConfig.table), tableRef);
+  let bounds = CheckpointSequence.sql(floors.checkpointBounds, Table.getPgChainIdColumn(entityConfig.table), tableRef);
   return `SELECT DISTINCT ` + keyColumns.map(c => tableRef + `."` + c + `"`).join(", ") + `
   FROM "` + pgSchema + `"."` + historyTableName + `"` + bounds.join + `
   WHERE ` + tableRef + `."` + EntityHistory.checkpointIdFieldName + `" > ` + bounds.checkpointId + `
@@ -982,8 +1028,9 @@ let rollbackChainIdSchema = Utils.$$WeakMap.memoize(table => Stdlib_Option.map(T
 
 let rollbackRemovedIdSchema = Utils.$$WeakMap.memoize(table => S$RescriptSchema.object(s => s.f(Table.idFieldName, Table.getIdSchema(table))));
 
-function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, chainIdModeOpt, ecosystem, sink, onInitialize, onNewTables) {
+function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, chainIdModeOpt, ecosystem, sink, isolatedOpt, onInitialize) {
   let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
+  let isolated = isolatedOpt !== undefined ? isolatedOpt : false;
   let containerName = "envio-postgres";
   let psqlExecOptions_env = Object.fromEntries([
     [
@@ -1107,9 +1154,6 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       }
     }
     let cacheTableInfo = await queryCacheTableInfo();
-    if (withUpload && Utils.$$Array.notEmpty(cacheTableInfo) && onNewTables !== undefined) {
-      await onNewTables(cacheTableInfo.map(info => info.table_name));
-    }
     let cache = {};
     cacheTableInfo.forEach(param => {
       let tableName = param.table_name;
@@ -1138,7 +1182,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     if (sink !== undefined) {
       await sink.initialize(entities);
     }
-    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames), true, chainIdMode);
+    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, CheckpointSequence.fromEntities(entities), chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames), true, chainIdMode);
     let rowsByChain = chainConfigs.map(chainConfig => ChainState.configStorageRows(chainConfig, ecosystem, contractMapping));
     let configAddressRows = rowsByChain.flat();
     await sql.begin(async sql => {
@@ -1165,13 +1209,14 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
         endBlock: chainConfig.endBlock,
         maxReorgDepth: chainConfig.maxReorgDepth,
         progressBlockNumber: -1,
+        progressBlockTime: undefined,
         numEventsProcessed: 0,
         firstEventBlockNumber: undefined,
         timestampCaughtUpToHeadOrEndblock: undefined,
         addressRows: AddressRows.seedRowsOf(rowsByChain[idx]),
         sourceBlockNumber: 0
       })),
-      checkpointId: InternalTable.Checkpoints.initialCheckpointId,
+      checkpointFrontier: Frontier.empty(),
       reorgCheckpoints: []
     };
   };
@@ -1246,10 +1291,23 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     }
     return IndexManager.resync(indexManager, name, rows);
   };
-  let ensureQueryIndexes = async (table, filters) => {
-    let columns = filterColumns(table, filters);
+  let partitionChainIds = chainIds => {
+    if (isolated) {
+      return Primitive_option.some(chainIds);
+    }
+  };
+  let queryTableName = (entityConfig, scope) => {
+    if (scope === "crossChain") {
+      return entityConfig.table.tableName;
+    } else {
+      return indexTableNames(entityConfig, partitionChainIds([scope]))[0];
+    }
+  };
+  let ensureQueryIndexes = async (entityConfig, scope, filters) => {
+    let tableName = queryTableName(entityConfig, scope);
+    let columns = filterColumns(entityConfig.table, filters);
     await Promise.all(columns.map(column => {
-      let definition = IndexDefinition.single(table.tableName, column);
+      let definition = IndexDefinition.single(tableName, column);
       return Stdlib_Promise.$$catch(IndexManager.ensure(indexManager, definition, "LeadingColumns", async () => {
         let prepared = IndexManager.prepare(indexManager, definition, "LeadingColumns", pgSchema);
         if (prepared === undefined) {
@@ -1258,7 +1316,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
         let verb = prepared.isRebuild ? "Rebuilding unusable index" : "Creating index";
         Logging.info({
           storage: storageName,
-          msg: verb + ` "` + prepared.name + `" to serve a getWhere query on "` + table.tableName + `". Writes to the table are paused until it completes. ` + slowOnLargeDatabaseNotice
+          msg: verb + ` "` + prepared.name + `" to serve a getWhere query on "` + tableName + `". Writes to the table are paused until it completes. ` + slowOnLargeDatabaseNotice
         });
         let timeRef = Performance.now();
         let entry = await runAndVerify(sql, prepared);
@@ -1270,15 +1328,15 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       }), async exn => {
         Logging.warn({
           storage: storageName,
-          msg: `Failed to create an index on "` + table.tableName + `"("` + column + `") for a getWhere query. The query runs without it.`,
+          msg: `Failed to create an index on "` + tableName + `"("` + column + `") for a getWhere query. The query runs without it.`,
           err: Utils.prettifyExn(exn)
         });
         return await resyncIndex(IndexDefinition.name(definition));
       });
     }));
   };
-  let ensureSchemaIndexes = async entities => {
-    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres));
+  let ensureSchemaIndexes = async (entities, chainIds) => {
+    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres), partitionChainIds(chainIds));
     await Promise.all(schemaIndexes.map(definition => Stdlib_Promise.$$catch(IndexManager.ensure(indexManager, definition, "Exact", async () => {
       let prepared = IndexManager.prepare(indexManager, definition, "Exact", pgSchema);
       if (prepared === undefined) {
@@ -1306,7 +1364,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     })));
   };
   let finalizeBackfill = async (entities, chainIds, readyAt) => {
-    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres));
+    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres), partitionChainIds(chainIds));
     let missing = Stdlib_Array.filterMap(schemaIndexes, definition => IndexManager.prepare(indexManager, definition, "Exact", pgSchema));
     let rebuilt = missing.filter(prepared => prepared.isRebuild);
     if (rebuilt.length !== 0) {
@@ -1362,9 +1420,6 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
   let setEffectCacheOrThrow = async (table, itemSchema, items, initialize) => {
     if (initialize) {
       await sql.unsafe(makeCreateTableQuery(table, pgSchema, false, undefined, undefined));
-      if (onNewTables !== undefined) {
-        await onNewTables([table.tableName]);
-      }
     }
     return await setOrThrow$1(items, table, itemSchema);
   };
@@ -1415,22 +1470,31 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       return Logging.errorWithExn(Utils.prettifyExn(exn$1), `Failed to dump cache.`);
     }
   };
-  let resumeInitialState = async (entities, throwIfIncompatible) => {
+  let resumeInitialState = async (entities, chainIds, throwIfIncompatible) => {
     let match = await Promise.all([
       restoreEffectCache(false),
-      InternalTable.Chains.getInitialState(sql, pgSchema).then(rawInitialStates => rawInitialStates.map(rawInitialState => ({
-        id: rawInitialState.id,
-        startBlock: rawInitialState.startBlock,
-        endBlock: Primitive_option.fromNull(rawInitialState.endBlock),
-        maxReorgDepth: rawInitialState.maxReorgDepth,
-        progressBlockNumber: rawInitialState.progressBlockNumber,
-        numEventsProcessed: rawInitialState.numEventsProcessed,
-        firstEventBlockNumber: Primitive_option.fromNull(rawInitialState.firstEventBlockNumber),
-        timestampCaughtUpToHeadOrEndblock: Primitive_option.fromNull(rawInitialState.timestampCaughtUpToHeadOrEndblock),
-        addressRows: rawInitialState.addressRows,
-        sourceBlockNumber: rawInitialState.sourceBlockNumber
-      }))),
-      sql.unsafe(InternalTable.Checkpoints.makeCommitedCheckpointIdQuery(pgSchema)),
+      InternalTable.Chains.getInitialState(sql, pgSchema).then(rawInitialStates => {
+        let rawInitialStates$1 = rawInitialStates.filter(rawInitialState => chainIds.includes(rawInitialState.id));
+        return [
+          rawInitialStates$1.map(rawInitialState => ({
+            id: rawInitialState.id,
+            startBlock: rawInitialState.startBlock,
+            endBlock: Primitive_option.fromNull(rawInitialState.endBlock),
+            maxReorgDepth: rawInitialState.maxReorgDepth,
+            progressBlockNumber: rawInitialState.progressBlockNumber,
+            progressBlockTime: InternalTable.Chains.blockTimeFromDb(rawInitialState.progressBlockTime),
+            numEventsProcessed: rawInitialState.numEventsProcessed,
+            firstEventBlockNumber: Primitive_option.fromNull(rawInitialState.firstEventBlockNumber),
+            timestampCaughtUpToHeadOrEndblock: Primitive_option.fromNull(rawInitialState.timestampCaughtUpToHeadOrEndblock),
+            addressRows: rawInitialState.addressRows,
+            sourceBlockNumber: rawInitialState.sourceBlockNumber
+          })),
+          Frontier.fromEntries(rawInitialStates$1.map(rawInitialState => [
+            rawInitialState.id,
+            BigInt(rawInitialState.checkpointId)
+          ]))
+        ];
+      }),
       sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema)),
       Promise.all([
         InternalTable.EnvioInfo.read(sql, pgSchema),
@@ -1451,21 +1515,22 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
         }
       })
     ]);
-    let match$1 = match[4];
+    let match$1 = match[3];
     let storedContractMapping = match$1[1];
     let storedEnvioInfo = match$1[0];
-    let chains = match[1];
+    let match$2 = match[1];
+    let checkpointFrontier = match$2[1];
+    let chains = match$2[0];
     throwIfIncompatible(storedEnvioInfo, storedContractMapping);
     await reloadIndexCatalog();
-    let checkpointId = BigInt(match[2][0].id);
-    let reorgCheckpoints = match[3].map(raw => ({
+    let reorgCheckpoints = match[2].map(raw => ({
       id: BigInt(raw.id),
       chain_id: ChainId.normalizeOrThrow(raw.chain_id),
       block_number: raw.block_number,
       block_hash: raw.block_hash
     }));
     if (sink !== undefined) {
-      await sink.resume(checkpointId, chains, entities);
+      await sink.resume(checkpointFrontier, chains, entities);
     }
     return {
       cleanRun: false,
@@ -1473,7 +1538,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       envioInfo: storedEnvioInfo,
       cache: match[0],
       chains: chains,
-      checkpointId: checkpointId,
+      checkpointFrontier: checkpointFrontier,
       reorgCheckpoints: reorgCheckpoints
     };
   };
@@ -1487,7 +1552,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
   let getRollbackTargetCheckpoint = (reorgChainId, lastKnownValidBlockNumber) => InternalTable.Checkpoints.getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValidBlockNumber);
   let getRollbackProgressDiff = floors => InternalTable.Checkpoints.getRollbackProgressDiff(sql, pgSchema, floors);
   let getRollbackData = async (entityConfig, floors) => {
-    let params = CheckpointBounds.params(floors.floors);
+    let params = CheckpointSequence.params(floors.checkpointBounds);
     let match = await Promise.all([
       sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema, floors), params, {prepare: true}),
       sql.unsafe(makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema, floors), params, {prepare: true})
@@ -1521,7 +1586,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       S$RescriptSchema.parseOrThrow(restoredEntitiesResult, Table.pgRowsSchema(entityConfig.table))
     ];
   };
-  let writeBatchMethod = async (batch, rollback, isInReorgThreshold, config, allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, onWrite) => {
+  let writeBatchMethod = async (batch, rollback, config, allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, onWrite) => {
     let pgUpdates = [];
     let chUpdates = [];
     for (let i = 0, i_finish = updatedEntities.length; i < i_finish; ++i) {
@@ -1537,14 +1602,14 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     let sinkPromise;
     if (sink !== undefined) {
       let timerRef = Performance.now();
-      sinkPromise = sink.writeBatch(batch, chUpdates).then(() => {
+      sinkPromise = sink.writeBatch(batch, rollback !== undefined ? rollback.diffCheckpoints : [], chUpdates).then(() => {
         onWrite(sink.name, Performance.secondsSince(timerRef));
       }).catch(exn => exn);
     } else {
       sinkPromise = undefined;
     }
     let primaryTimerRef = Performance.now();
-    await writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, pgUpdates, registeredAddresses, sinkPromise, chainMetaData, undefined);
+    await writeBatch(sql, batch, pgSchema, rollback, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, pgUpdates, registeredAddresses, sinkPromise, chainMetaData, undefined);
     return onWrite(storageName, Performance.secondsSince(primaryTimerRef));
   };
   let close = () => sql.end();
@@ -1596,21 +1661,14 @@ function makeStorageFromEnv(config, sqlOpt, pgSchemaOpt, isHasuraEnabledOpt) {
     if (missing.length !== 0) {
       Stdlib_JsError.throwWithMessage(`ClickHouse storage is enabled but required env vars are not set: ` + missing.join(", ") + `. Please set them, disable clickhouse in the \`storage\` config, or run \`envio dev\` for a pre-configured local ClickHouse.`);
     }
-    tmp = Sink.makeClickHouse(host, database, username, password, config.chainIdMode);
+    tmp = Sink.makeClickHouse(host, database, username, password, config.checkpointSequence, config.chainIdMode);
   } else {
     tmp = undefined;
   }
-  return make(sql, Env.Db.host, pgSchema, Env.Db.port, Env.Db.user, Env.Db.database, Env.Db.password, isHasuraEnabled, config.chainIdMode, config.ecosystem.name, tmp, isHasuraEnabled ? () => Stdlib_Promise.$$catch(Hasura.trackDatabase(Env.Hasura.graphqlEndpoint, {
+  return make(sql, Env.Db.host, pgSchema, Env.Db.port, Env.Db.user, Env.Db.database, Env.Db.password, isHasuraEnabled, config.chainIdMode, config.ecosystem.name, tmp, config.isolated, isHasuraEnabled ? () => Stdlib_Promise.$$catch(Hasura.trackDatabase(Env.Hasura.graphqlEndpoint, {
       role: Env.Hasura.role,
       secret: Env.Hasura.secret
-    }, pgSchema, Config.getPgUserEntities(config), Env.Hasura.aggregateEntities, Env.Hasura.responseLimit, Schema.make(config.userEntities.map(e => e.table))), err => Promise.resolve(Logging.errorWithExn(Utils.prettifyExn(err), `Error tracking tables`))) : undefined, isHasuraEnabled ? tableNames => Stdlib_Promise.$$catch(Hasura.trackTables(Env.Hasura.graphqlEndpoint, {
-      role: Env.Hasura.role,
-      secret: Env.Hasura.secret
-    }, pgSchema, tableNames.map(tableName => ({
-      tableName: tableName,
-      description: undefined,
-      columnConfigs: {}
-    }))), err => Promise.resolve(Logging.errorWithExn(Utils.prettifyExn(err), `Error tracking new tables`))) : undefined);
+    }, pgSchema, Config.getPgUserEntities(config), Env.Hasura.aggregateEntities, Env.Hasura.responseLimit, Schema.make(config.userEntities.map(e => e.table))), err => Promise.resolve(Logging.errorWithExn(Utils.prettifyExn(err), `Error tracking tables`))) : undefined);
 }
 
 function makePersistenceFromConfig(config, storageOpt) {
@@ -1624,9 +1682,10 @@ export {
   makeClient,
   formatSeconds,
   slowOnLargeDatabaseNotice,
+  partitionTableName,
+  indexTableNames,
   getSchemaIndexes,
   makeCreateTableQuery,
-  partitionTableName,
   rowSchemaCache,
   getRowSchema,
   entityHistoryCache,
@@ -1658,6 +1717,7 @@ export {
   deleteByIdsOrThrow,
   makeInsertDeleteUpdatesQuery,
   executeSet,
+  pickCheckpoints,
   writeBatch,
   rollbackKeyColumns,
   makeGetRollbackPreTargetRowsQuery,

package/src/PruneStaleHistory.res

@@ -12,7 +12,7 @@ let maxEntitiesPerWrite = 5
 let forcedIntervalMultiplier = 5.
 
 type targets = {
-  safeCheckpoints: CheckpointBounds.t,
+  safeCheckpoints: CheckpointSequence.checkpointBoundsByChain,
   concurrent: array<Internal.entityConfig>,
   forced: array<Internal.entityConfig>,
 }
@@ -28,25 +28,19 @@ type targets = {
 // every chain back to one checkpoint, which can sit below another chain's safe
 // point, so the lowest is the only bound every chain agrees on.
 let selectSafeCheckpoints = (state: IndexerState.t) => {
+  let sequence = (state->IndexerState.config).checkpointSequence
   let byChain = state->IndexerState.getSafeCheckpointIdByChain
   let safe =
     byChain->Array.filterMap(((chainId, checkpointId)) =>
       checkpointId->Option.map(checkpointId => (chainId, checkpointId))
     )
-  if state->IndexerState.config->Config.isIsolatedMultichain {
-    safe->Utils.Array.notEmpty ? Some(CheckpointBounds.PerChain(safe)) : None
-  } else if safe->Array.length === byChain->Array.length {
-    let (_, lowest) = safe->Array.getUnsafe(0)
-    Some(
-      CheckpointBounds.EveryChain(
-        safe->Array.reduce(lowest, (lowest, (_, checkpointId)) =>
-          Pervasives.min(lowest, checkpointId)
-        ),
-      ),
-    )
-  } else {
-    None
+  // Under one shared sequence the bound is the lowest of them, which only
+  // exists once every chain has one.
+  let hasBound = switch sequence {
+  | PerChain => safe->Utils.Array.notEmpty
+  | SharedAcrossChains => safe->Array.length === byChain->Array.length
   }
+  hasBound ? Some({CheckpointSequence.sequence, byChain: Frontier.fromEntries(safe)}) : None
 }
 
 let selectFrom = (
@@ -103,7 +97,7 @@ let selectFrom = (
 
 let select = (state: IndexerState.t, ~writtenEntityNames, ~isRollback) => {
   let config = state->IndexerState.config
-  if config->Config.shouldPruneHistory(~isInReorgThreshold=state->IndexerState.isInReorgThreshold) {
+  if config->HistoryPolicy.mayPrune(~shouldSaveHistory=state->IndexerState.shouldSaveHistory) {
     state
     ->selectSafeCheckpoints
     ->Option.map(safeCheckpoints =>

package/src/PruneStaleHistory.res.mjs

@@ -3,16 +3,17 @@
 import * as Env from "./Env.res.mjs";
 import * as Table from "./db/Table.res.mjs";
 import * as Utils from "./Utils.res.mjs";
-import * as Config from "./Config.res.mjs";
 import * as Logging from "./Logging.res.mjs";
+import * as Frontier from "./db/Frontier.res.mjs";
 import * as Performance from "./bindings/Performance.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as HistoryPolicy from "./HistoryPolicy.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
-import * as Primitive_bigint from "@rescript/runtime/lib/es6/Primitive_bigint.js";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 function selectSafeCheckpoints(state) {
+  let sequence = IndexerState.config(state).checkpointSequence;
   let byChain = IndexerState.getSafeCheckpointIdByChain(state);
   let safe = Stdlib_Array.filterMap(byChain, param => {
     let chainId = param[0];
@@ -21,24 +22,14 @@ function selectSafeCheckpoints(state) {
       checkpointId
     ]);
   });
-  if (Config.isIsolatedMultichain(IndexerState.config(state))) {
-    if (Utils.$$Array.notEmpty(safe)) {
-      return {
-        TAG: "PerChain",
-        _0: safe
-      };
-    } else {
-      return;
-    }
+  let hasBound;
+  hasBound = sequence === "SharedAcrossChains" ? safe.length === byChain.length : Utils.$$Array.notEmpty(safe);
+  if (hasBound) {
+    return {
+      sequence: sequence,
+      byChain: Frontier.fromEntries(safe)
+    };
   }
-  if (safe.length !== byChain.length) {
-    return;
-  }
-  let match = safe[0];
-  return {
-    TAG: "EveryChain",
-    _0: Stdlib_Array.reduce(safe, match[1], (lowest, param) => Primitive_bigint.min(lowest, param[1]))
-  };
 }
 
 function selectFrom(allEntities, lastPrunedAtMillis, writtenEntityNames, isRollback, nowMillis, intervalMillis, safeCheckpoints) {
@@ -83,7 +74,7 @@ function selectFrom(allEntities, lastPrunedAtMillis, writtenEntityNames, isRollb
 
 function select(state, writtenEntityNames, isRollback) {
   let config = IndexerState.config(state);
-  if (Config.shouldPruneHistory(config, IndexerState.isInReorgThreshold(state))) {
+  if (HistoryPolicy.mayPrune(config, IndexerState.shouldSaveHistory(state))) {
     return Stdlib_Option.map(selectSafeCheckpoints(state), safeCheckpoints => selectFrom(IndexerState.persistence(state).allEntities, IndexerState.lastPrunedAtMillis(state), writtenEntityNames, isRollback, Date.now(), Env.ThrottleWrites.pruneStaleDataIntervalMillis, safeCheckpoints));
   }
 }

package/src/Rollback.res

@@ -132,19 +132,14 @@ and executeRollback = async (
   // flush above leaves a diff pending only when no batch has come along to
   // carry it.
   let floors = {
-    let next = if state->IndexerState.config->Config.isIsolatedMultichain {
-      RollbackFloors.isolated(
-        ~chainId=reorgChain,
-        ~floorCheckpointId=rollbackTargetCheckpointId,
-        ~forkBlockNumber=rollbackTargetBlockNumber,
-      )
-    } else {
-      RollbackFloors.global(
-        ~floorCheckpointId=rollbackTargetCheckpointId,
-        ~reorgChainId=reorgChain,
-        ~forkBlockNumber=rollbackTargetBlockNumber,
-      )
-    }
+    let config = state->IndexerState.config
+    let next = RollbackFloors.make(
+      ~sequence=config.checkpointSequence,
+      ~chainIds=config.chainMap->ChainMap.keys,
+      ~reorgChainId=reorgChain,
+      ~floorCheckpointId=rollbackTargetCheckpointId,
+      ~forkBlockNumber=rollbackTargetBlockNumber,
+    )
     switch state->IndexerState.pendingRollback {
     | None => next
     | Some({floors: pending}) => RollbackFloors.merge(pending, next)
@@ -205,6 +200,7 @@ and executeRollback = async (
       ->Array.push({
         chainId,
         progressBlockNumber: toBlock,
+        progressBlockTime: cs->ChainState.committedProgressBlockTime,
         sourceBlockNumber: cs->ChainState.knownHeight,
         totalEventsProcessed: cs->ChainState.numEventsProcessed,
       })
@@ -221,7 +217,6 @@ and executeRollback = async (
 
   let diff = await state->InMemoryStore.prepareRollbackDiff(
     ~floors,
-    ~rollbackDiffCheckpointId=state->IndexerState.committedCheckpointId->BigInt.add(1n),
     ~progressedChains=rolledBackChains,
     ~rolledBackAddresses,
   )