envio 3.11.0 → 3.12.1

package/src/PgStorage.res.mjs

@@ -12,16 +12,19 @@ import * as Schema from "./db/Schema.res.mjs";
 import * as ChainId from "./ChainId.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as EntityId from "./EntityId.res.mjs";
+import * as Frontier from "./db/Frontier.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import Postgres from "postgres";
 import * as ChainState from "./ChainState.res.mjs";
 import * as AddressRows from "./AddressRows.res.mjs";
 import * as Performance from "./bindings/Performance.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
+import * as EntityFilter from "./db/EntityFilter.res.mjs";
 import * as IndexCatalog from "./db/IndexCatalog.res.mjs";
 import * as IndexManager from "./db/IndexManager.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
+import * as HistoryPolicy from "./HistoryPolicy.res.mjs";
 import * as InternalTable from "./db/InternalTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Child_process from "child_process";
@@ -29,10 +32,10 @@ import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
 import * as ContractMapping from "./ContractMapping.res.mjs";
 import * as IndexDefinition from "./db/IndexDefinition.res.mjs";
-import * as CheckpointBounds from "./db/CheckpointBounds.res.mjs";
 import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+import * as CheckpointSequence from "./db/CheckpointSequence.res.mjs";
 import * as Primitive_exceptions from "@rescript/runtime/lib/es6/Primitive_exceptions.js";
 
 function makeClient() {
@@ -57,7 +60,25 @@ function formatSeconds(timeRef) {
 
 let slowOnLargeDatabaseNotice = "This can take a long time on a large database.";
 
-function getSchemaIndexes(entities) {
+function partitionTableName(entityConfig, chainId) {
+  let chainIdStr = ChainId.toString(chainId);
+  return Table.fitPgTableName(entityConfig.table.tableName + `$` + chainIdStr, `$` + entityConfig.index.toString() + `$` + chainIdStr);
+}
+
+function indexTableNames(entityConfig, partitionChainIds) {
+  let match = Table.getChainIdField(entityConfig.table);
+  if (match !== undefined) {
+    if (partitionChainIds !== undefined) {
+      return partitionChainIds.map(chainId => partitionTableName(entityConfig, chainId));
+    } else {
+      return [entityConfig.table.tableName];
+    }
+  } else {
+    return [entityConfig.table.tableName];
+  }
+}
+
+function getSchemaIndexes(entities, partitionChainIds) {
   let derivedSchema = Schema.make(entities.map(e => e.table));
   let all = [];
   entities.forEach(param => {
@@ -76,6 +97,10 @@ function getSchemaIndexes(entities) {
     });
   });
   let seen = new Set();
+  let entityByTableName = {};
+  entities.forEach(entityConfig => {
+    entityByTableName[entityConfig.table.tableName] = entityConfig;
+  });
   return all.filter(definition => {
     let key = IndexDefinition.key(definition);
     if (seen.has(key)) {
@@ -84,7 +109,11 @@ function getSchemaIndexes(entities) {
       seen.add(key);
       return true;
     }
-  });
+  }).flatMap(definition => indexTableNames(Stdlib_Option.getOrThrow(entityByTableName[definition.tableName], undefined), partitionChainIds).map(tableName => ({
+    tableName: tableName,
+    columns: definition.columns,
+    method: definition.method
+  })));
 }
 
 function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText, chainIdModeOpt, partitionByColumn) {
@@ -108,11 +137,6 @@ function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText, chainIdMode
   ) + `;`;
 }
 
-function partitionTableName(entityConfig, chainId) {
-  let chainIdStr = ChainId.toString(chainId);
-  return Table.fitPgTableName(entityConfig.table.tableName + `$` + chainIdStr, `$` + entityConfig.index.toString() + `$` + chainIdStr);
-}
-
 let rowSchemaCache = new WeakMap();
 
 function getRowSchema(entityConfig) {
@@ -228,7 +252,7 @@ function makeCreateEntityTableQueries(entityConfig, pgSchema, isNumericArrayAsTe
   ).concat([createTable(getEntityHistory(entityConfig).table, undefined)]);
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt, deferSchemaIndexesOpt, chainIdModeOpt) {
+function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, checkpointSequence, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt, deferSchemaIndexesOpt, chainIdModeOpt) {
   let chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
   let entities = entitiesOpt !== undefined ? entitiesOpt : [];
   let enums = enumsOpt !== undefined ? enumsOpt : [];
@@ -240,12 +264,12 @@ function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfi
     InternalTable.EnvioInfo.table,
     InternalTable.EnvioContracts.table,
     InternalTable.EnvioAddresses.table,
-    InternalTable.Checkpoints.table,
+    InternalTable.Checkpoints.tableFor(checkpointSequence),
     InternalTable.RawEvents.table
   ];
   let chainIds = chainConfigs.map(chainConfig => chainConfig.id);
   let tableQueries = generalTables.map(table => makeCreateTableQuery(table, pgSchema, isHasuraEnabled, chainIdMode, undefined)).concat(entities.flatMap(entityConfig => makeCreateEntityTableQueries(entityConfig, pgSchema, isHasuraEnabled, chainIdMode, chainIds)));
-  let schemaIndexes = getSchemaIndexes(entities);
+  let schemaIndexes = getSchemaIndexes(entities, undefined);
   let query = {
     contents: (
       isEmptyPgSchema && pgSchema === "public" ? `CREATE SCHEMA IF NOT EXISTS "` + pgSchema + `";\n` : `DROP SCHEMA IF EXISTS "` + pgSchema + `" CASCADE;
@@ -278,7 +302,7 @@ function makeLoadQuery(pgSchema, tableName, condition) {
   return `SELECT * FROM "` + pgSchema + `"."` + tableName + `" WHERE ` + condition + `;`;
 }
 
-function makeFilterCondition(filter, table, params) {
+function makeFilterCondition(filter, table, pgSchema, params) {
   let getQueryFieldOrThrow = fieldName => {
     let queryField = Table.queryFields(table)[fieldName];
     if (queryField !== undefined) {
@@ -310,39 +334,76 @@ function makeFilterCondition(filter, table, params) {
     params.push(param);
     return `$` + params.length.toString();
   };
-  let scalarCondition = (fieldName, fieldValue, op) => {
-    let queryField = getQueryFieldOrThrow(fieldName);
-    return `"` + queryField.pgDbFieldName + `" ` + op + ` ` + serializeParamOrThrow(queryField, fieldName, fieldValue, false);
+  let condition = {
+    contents: ""
   };
-  switch (filter.operator) {
-    case "=" :
-      let fieldValue = filter.fieldValue;
-      let fieldName = filter.fieldName;
-      if (getQueryFieldOrThrow(fieldName).isChainId) {
-        return `"` + getQueryFieldOrThrow(fieldName).pgDbFieldName + `" = ` + ChainId.toString(ChainId.normalizeOrThrow(fieldValue));
-      } else {
-        return scalarCondition(fieldName, fieldValue, "=");
+  Utils.Dict.forEachWithKey(EntityFilter.entries(filter), (operators, fieldName) => {
+    let queryField = getQueryFieldOrThrow(fieldName);
+    Utils.Dict.forEachWithKey(operators, (fieldValue, operator) => {
+      let column = `"` + queryField.pgDbFieldName + `"`;
+      let part;
+      let exit = 0;
+      switch (operator) {
+        case "_eq" :
+          if (queryField.isChainId) {
+            part = column + ` = ` + ChainId.toString(ChainId.normalizeOrThrow(fieldValue));
+          } else {
+            exit = 1;
+          }
+          break;
+        case "_in" :
+          if (queryField.isArray || queryField.fieldType === "Boolean") {
+            let candidates = EntityFilter.asArray(fieldValue);
+            part = candidates.length !== 0 ? `(` + candidates.map(candidate => column + ` = ` + serializeParamOrThrow(queryField, fieldName, candidate, false)).join(" OR ") + `)` : "FALSE";
+          } else {
+            let param = serializeParamOrThrow(queryField, fieldName, fieldValue, true);
+            let match = queryField.fieldType;
+            let exit$1 = 0;
+            if (typeof match !== "object" || match.type !== "Enum") {
+              exit$1 = 2;
+            } else {
+              part = column + ` = ANY(` + param + `::TEXT[]::"` + pgSchema + `".` + match.config.name + `[])`;
+            }
+            if (exit$1 === 2) {
+              part = column + ` = ANY(` + param + `)`;
+            }
+          }
+          break;
+        default:
+          exit = 1;
       }
-    case ">" :
-      return scalarCondition(filter.fieldName, filter.fieldValue, ">");
-    case "<" :
-      return scalarCondition(filter.fieldName, filter.fieldValue, "<");
-    case "in" :
-      let fieldName$1 = filter.fieldName;
-      let queryField = getQueryFieldOrThrow(fieldName$1);
-      return `"` + queryField.pgDbFieldName + `" = ANY(` + serializeParamOrThrow(queryField, fieldName$1, filter.fieldValue, true) + `)`;
-    case "and" :
-      let filters = filter.filters;
-      if (filters.length !== 0) {
-        return `(` + filters.map(filter => makeFilterCondition(filter, table, params)).join(" AND ") + `)`;
+      if (exit === 1) {
+        let sqlOperator;
+        switch (operator) {
+          case "_eq" :
+            sqlOperator = "=";
+            break;
+          case "_gt" :
+            sqlOperator = ">";
+            break;
+          case "_gte" :
+            sqlOperator = ">=";
+            break;
+          case "_lt" :
+            sqlOperator = "<";
+            break;
+          case "_lte" :
+            sqlOperator = "<=";
+            break;
+          default:
+            throw {
+              RE_EXN_ID: Persistence.StorageError,
+              message: `Failed loading "` + table.tableName + `" from storage. Unknown filter operator "` + operator + `".`,
+              reason: new Error(`Unknown filter operator "` + operator + `"`),
+              Error: new Error()
+            };
+        }
+        part = column + ` ` + sqlOperator + ` ` + serializeParamOrThrow(queryField, fieldName, fieldValue, false);
       }
-      throw {
-        RE_EXN_ID: Persistence.StorageError,
-        message: `Failed loading "` + table.tableName + `" from storage. The "and" filter must contain at least one nested filter.`,
-        reason: new Error(`Empty "and" filter`),
-        Error: new Error()
-      };
-  }
+      condition.contents = condition.contents === "" ? part : condition.contents + " AND " + part;
+    });
+  });
+  return condition.contents;
 }
 
 function makeChainIdCondition(table, chainId) {
@@ -687,10 +748,29 @@ function executeSet(sql, items, dbFunction) {
   }
 }
 
-async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, updatedEntities, registeredAddresses, sinkPromise, chainMetaData, escapeTables) {
+function pickCheckpoints(column, picked) {
+  if (typeof picked !== "object") {
+    return column;
+  } else {
+    return picked._0.map(index => column[index]);
+  }
+}
+
+async function writeBatch(sql, batch, pgSchema, rollback, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, updatedEntities, registeredAddresses, sinkPromise, chainMetaData, escapeTables) {
   try {
     let chainIdMode = config.chainIdMode;
-    let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold, undefined);
+    let indexes = Stdlib_Array.filterMapWithIndex(batch.checkpointChainIds, (chainId, index) => {
+      if (HistoryPolicy.forChain(batch.history, chainId)) {
+        return index;
+      }
+    });
+    let pickedCheckpoints = Utils.$$Array.isEmpty(indexes) ? undefined : (
+        indexes.length === batch.checkpointIds.length ? "AllCheckpoints" : ({
+            TAG: "CheckpointIndexes",
+            _0: indexes
+          })
+      );
+    let writtenFrontier = Persistence.writtenFrontier(batch, rollback);
     let specificError = {
       contents: undefined
     };
@@ -725,6 +805,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
       }
     };
     let setEntities = updatedEntities.map(param => {
+      let shouldSaveHistory = param.shouldSaveHistory;
       let changes = param.changes;
       let scope = param.scope;
       let entityConfig = param.entityConfig;
@@ -732,7 +813,6 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
       let idsToDelete = [];
       let scopeChainId;
       scopeChainId = scope === "crossChain" ? undefined : Primitive_option.some(scope);
-      let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold, scopeChainId);
       let match = Table.getChainIdField(entityConfig.table);
       let changes$1;
       if (match !== undefined && scopeChainId !== undefined) {
@@ -756,7 +836,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
       let chainIdParams = match$1 !== undefined ? (
           scopeChainId !== undefined ? [Primitive_option.valFromOption(scopeChainId)] : []
         ) : [];
-      let diffCheckpointId = Stdlib_Option.map(rollback, r => r.diffCheckpointId);
+      let diffCheckpointId = Stdlib_Option.flatMap(rollback, r => CheckpointSequence.findForScope(config.checkpointSequence, r.diffFrontier, scope));
       let batchSetUpdates = [];
       let batchDeleteEntityIds = [];
       let batchDeleteCheckpointIds = [];
@@ -872,6 +952,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
             sql => InternalTable.Chains.setProgressedChains(sql, pgSchema, Utils.Dict.mapValuesToArray(batch.progressedChainsById, chainAfterBatch => ({
               chainId: chainAfterBatch.fetchState.chainId,
               progressBlockNumber: chainAfterBatch.progressBlockNumber,
+              progressBlockTime: chainAfterBatch.progressBlockTime,
               sourceBlockNumber: chainAfterBatch.sourceBlockNumber,
               totalEventsProcessed: chainAfterBatch.totalEventsProcessed
             }))),
@@ -883,8 +964,11 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
           if (Utils.$$Array.notEmpty(registeredAddresses)) {
             setOperations.push(sql => InternalTable.EnvioAddresses.insert(sql, pgSchema, registeredAddresses.map(staged => staged.row), chainIdMode));
           }
-          if (shouldSaveHistory) {
-            setOperations.push(sql => InternalTable.Checkpoints.insert(sql, pgSchema, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointBlockHashes, batch.checkpointEventsProcessed, chainIdMode));
+          if (!Utils.Dict.isEmpty(writtenFrontier)) {
+            setOperations.push(sql => InternalTable.Chains.setCheckpointFrontier(sql, pgSchema, writtenFrontier, chainIdMode));
+          }
+          if (pickedCheckpoints !== undefined) {
+            setOperations.push(sql => InternalTable.Checkpoints.insert(sql, pgSchema, pickCheckpoints(batch.checkpointIds, pickedCheckpoints), pickCheckpoints(batch.checkpointChainIds, pickedCheckpoints), pickCheckpoints(batch.checkpointBlockNumbers, pickedCheckpoints), pickCheckpoints(batch.checkpointBlockHashes, pickedCheckpoints), pickCheckpoints(batch.checkpointEventsProcessed, pickedCheckpoints), chainIdMode));
           }
           await Promise.all(setOperations.map(dbFunc => dbFunc(sql)));
           if (sinkPromise === undefined) {
@@ -913,7 +997,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
     if (exn$1.RE_EXN_ID === PgEncodingError) {
       let escapeTables$1 = escapeTables !== undefined ? Primitive_option.valFromOption(escapeTables) : new Set();
       escapeTables$1.add(exn$1.table);
-      return await writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, updatedEntities, registeredAddresses, sinkPromise, chainMetaData, Primitive_option.some(escapeTables$1));
+      return await writeBatch(sql, batch, pgSchema, rollback, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, updatedEntities, registeredAddresses, sinkPromise, chainMetaData, Primitive_option.some(escapeTables$1));
     }
     throw exn$1;
   }
@@ -943,7 +1027,7 @@ function makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema, floors) {
   let keyColumns = rollbackKeyColumns(entityConfig);
   let keyColumnsCommaSeparated = keyColumns.map(c => tableRef + `."` + c + `"`).join(", ");
   let keyMatch = keyColumns.map(c => `h."` + c + `" = ` + tableRef + `."` + c + `"`).join(" AND ");
-  let bounds = CheckpointBounds.sql(floors.floors, Table.getPgChainIdColumn(entityConfig.table), tableRef);
+  let bounds = CheckpointSequence.sql(floors.checkpointBounds, Table.getPgChainIdColumn(entityConfig.table), tableRef);
   return `SELECT DISTINCT ON (` + keyColumnsCommaSeparated + `) ` + dataFieldsCommaSeparated + `, ` + tableRef + `."` + EntityHistory.changeFieldName + `"
   FROM "` + pgSchema + `"."` + historyTableName + `"` + bounds.join + `
   WHERE ` + tableRef + `."` + EntityHistory.checkpointIdFieldName + `" <= ` + bounds.checkpointId + `
@@ -961,7 +1045,7 @@ function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema, floors) {
   let tableRef = `"` + historyTableName + `"`;
   let keyColumns = rollbackKeyColumns(entityConfig);
   let keyMatch = keyColumns.map(c => `h."` + c + `" = ` + tableRef + `."` + c + `"`).join(" AND ");
-  let bounds = CheckpointBounds.sql(floors.floors, Table.getPgChainIdColumn(entityConfig.table), tableRef);
+  let bounds = CheckpointSequence.sql(floors.checkpointBounds, Table.getPgChainIdColumn(entityConfig.table), tableRef);
   return `SELECT DISTINCT ` + keyColumns.map(c => tableRef + `."` + c + `"`).join(", ") + `
   FROM "` + pgSchema + `"."` + historyTableName + `"` + bounds.join + `
   WHERE ` + tableRef + `."` + EntityHistory.checkpointIdFieldName + `" > ` + bounds.checkpointId + `
@@ -982,19 +1066,14 @@ let rollbackChainIdSchema = Utils.$$WeakMap.memoize(table => Stdlib_Option.map(T
 
 let rollbackRemovedIdSchema = Utils.$$WeakMap.memoize(table => S$RescriptSchema.object(s => s.f(Table.idFieldName, Table.getIdSchema(table))));
 
-function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, chainIdModeOpt, ecosystem, sink, onInitialize, onNewTables) {
+function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, chainIdModeOpt, ecosystem, sink, isolatedOpt, onInitialize) {
   let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
+  let isolated = isolatedOpt !== undefined ? isolatedOpt : false;
   let containerName = "envio-postgres";
-  let psqlExecOptions_env = Object.fromEntries([
-    [
-      "PGPASSWORD",
-      pgPassword
-    ],
-    [
-      "PATH",
-      process.env.PATH
-    ]
-  ]);
+  let psqlExecOptions_env = {
+    PGPASSWORD: pgPassword,
+    PATH: process.env.PATH
+  };
   let psqlExecOptions = {
     env: psqlExecOptions_env
   };
@@ -1107,9 +1186,6 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       }
     }
     let cacheTableInfo = await queryCacheTableInfo();
-    if (withUpload && Utils.$$Array.notEmpty(cacheTableInfo) && onNewTables !== undefined) {
-      await onNewTables(cacheTableInfo.map(info => info.table_name));
-    }
     let cache = {};
     cacheTableInfo.forEach(param => {
       let tableName = param.table_name;
@@ -1138,7 +1214,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     if (sink !== undefined) {
       await sink.initialize(entities);
     }
-    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames), true, chainIdMode);
+    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, CheckpointSequence.fromEntities(entities), chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames), true, chainIdMode);
     let rowsByChain = chainConfigs.map(chainConfig => ChainState.configStorageRows(chainConfig, ecosystem, contractMapping));
     let configAddressRows = rowsByChain.flat();
     await sql.begin(async sql => {
@@ -1165,19 +1241,20 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
         endBlock: chainConfig.endBlock,
         maxReorgDepth: chainConfig.maxReorgDepth,
         progressBlockNumber: -1,
+        progressBlockTime: undefined,
         numEventsProcessed: 0,
         firstEventBlockNumber: undefined,
         timestampCaughtUpToHeadOrEndblock: undefined,
         addressRows: AddressRows.seedRowsOf(rowsByChain[idx]),
         sourceBlockNumber: 0
       })),
-      checkpointId: InternalTable.Checkpoints.initialCheckpointId,
+      checkpointFrontier: Frontier.empty(),
       reorgCheckpoints: []
     };
   };
   let loadOrThrow = async (filter, table) => {
     let params = [];
-    let condition = makeFilterCondition(filter, table, params);
+    let condition = makeFilterCondition(filter, table, pgSchema, params);
     let rows;
     try {
       rows = await sql.unsafe(makeLoadQuery(pgSchema, table.tableName, condition), params, {prepare: true});
@@ -1206,12 +1283,8 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     let queryFields = Table.queryFields(table);
     let columns = [];
     let seen = new Set();
-    let collect = filter => {
-      if (filter.operator === "and") {
-        filter.filters.forEach(collect);
-        return;
-      }
-      let match = queryFields[filter.fieldName];
+    filters.forEach(filter => Utils.Dict.forEachWithKey(EntityFilter.entries(filter), (param, fieldName) => {
+      let match = queryFields[fieldName];
       if (match === undefined) {
         return;
       }
@@ -1221,8 +1294,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
         columns.push(pgDbFieldName);
         return;
       }
-    };
-    filters.forEach(collect);
+    }));
     return columns;
   };
   let runAndVerify = async (sql, prepared) => {
@@ -1246,10 +1318,23 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     }
     return IndexManager.resync(indexManager, name, rows);
   };
-  let ensureQueryIndexes = async (table, filters) => {
-    let columns = filterColumns(table, filters);
+  let partitionChainIds = chainIds => {
+    if (isolated) {
+      return Primitive_option.some(chainIds);
+    }
+  };
+  let queryTableName = (entityConfig, scope) => {
+    if (scope === "crossChain") {
+      return entityConfig.table.tableName;
+    } else {
+      return indexTableNames(entityConfig, partitionChainIds([scope]))[0];
+    }
+  };
+  let ensureQueryIndexes = async (entityConfig, scope, filters) => {
+    let tableName = queryTableName(entityConfig, scope);
+    let columns = filterColumns(entityConfig.table, filters);
     await Promise.all(columns.map(column => {
-      let definition = IndexDefinition.single(table.tableName, column);
+      let definition = IndexDefinition.single(tableName, column);
       return Stdlib_Promise.$$catch(IndexManager.ensure(indexManager, definition, "LeadingColumns", async () => {
         let prepared = IndexManager.prepare(indexManager, definition, "LeadingColumns", pgSchema);
         if (prepared === undefined) {
@@ -1258,7 +1343,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
         let verb = prepared.isRebuild ? "Rebuilding unusable index" : "Creating index";
         Logging.info({
           storage: storageName,
-          msg: verb + ` "` + prepared.name + `" to serve a getWhere query on "` + table.tableName + `". Writes to the table are paused until it completes. ` + slowOnLargeDatabaseNotice
+          msg: verb + ` "` + prepared.name + `" to serve a getWhere query on "` + tableName + `". Writes to the table are paused until it completes. ` + slowOnLargeDatabaseNotice
         });
         let timeRef = Performance.now();
         let entry = await runAndVerify(sql, prepared);
@@ -1270,15 +1355,15 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       }), async exn => {
         Logging.warn({
           storage: storageName,
-          msg: `Failed to create an index on "` + table.tableName + `"("` + column + `") for a getWhere query. The query runs without it.`,
+          msg: `Failed to create an index on "` + tableName + `"("` + column + `") for a getWhere query. The query runs without it.`,
           err: Utils.prettifyExn(exn)
         });
         return await resyncIndex(IndexDefinition.name(definition));
       });
     }));
   };
-  let ensureSchemaIndexes = async entities => {
-    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres));
+  let ensureSchemaIndexes = async (entities, chainIds) => {
+    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres), partitionChainIds(chainIds));
     await Promise.all(schemaIndexes.map(definition => Stdlib_Promise.$$catch(IndexManager.ensure(indexManager, definition, "Exact", async () => {
       let prepared = IndexManager.prepare(indexManager, definition, "Exact", pgSchema);
       if (prepared === undefined) {
@@ -1306,7 +1391,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     })));
   };
   let finalizeBackfill = async (entities, chainIds, readyAt) => {
-    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres));
+    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres), partitionChainIds(chainIds));
     let missing = Stdlib_Array.filterMap(schemaIndexes, definition => IndexManager.prepare(indexManager, definition, "Exact", pgSchema));
     let rebuilt = missing.filter(prepared => prepared.isRebuild);
     if (rebuilt.length !== 0) {
@@ -1362,9 +1447,6 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
   let setEffectCacheOrThrow = async (table, itemSchema, items, initialize) => {
     if (initialize) {
       await sql.unsafe(makeCreateTableQuery(table, pgSchema, false, undefined, undefined));
-      if (onNewTables !== undefined) {
-        await onNewTables([table.tableName]);
-      }
     }
     return await setOrThrow$1(items, table, itemSchema);
   };
@@ -1415,22 +1497,31 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       return Logging.errorWithExn(Utils.prettifyExn(exn$1), `Failed to dump cache.`);
     }
   };
-  let resumeInitialState = async (entities, throwIfIncompatible) => {
+  let resumeInitialState = async (entities, chainIds, throwIfIncompatible) => {
     let match = await Promise.all([
       restoreEffectCache(false),
-      InternalTable.Chains.getInitialState(sql, pgSchema).then(rawInitialStates => rawInitialStates.map(rawInitialState => ({
-        id: rawInitialState.id,
-        startBlock: rawInitialState.startBlock,
-        endBlock: Primitive_option.fromNull(rawInitialState.endBlock),
-        maxReorgDepth: rawInitialState.maxReorgDepth,
-        progressBlockNumber: rawInitialState.progressBlockNumber,
-        numEventsProcessed: rawInitialState.numEventsProcessed,
-        firstEventBlockNumber: Primitive_option.fromNull(rawInitialState.firstEventBlockNumber),
-        timestampCaughtUpToHeadOrEndblock: Primitive_option.fromNull(rawInitialState.timestampCaughtUpToHeadOrEndblock),
-        addressRows: rawInitialState.addressRows,
-        sourceBlockNumber: rawInitialState.sourceBlockNumber
-      }))),
-      sql.unsafe(InternalTable.Checkpoints.makeCommitedCheckpointIdQuery(pgSchema)),
+      InternalTable.Chains.getInitialState(sql, pgSchema).then(rawInitialStates => {
+        let rawInitialStates$1 = rawInitialStates.filter(rawInitialState => chainIds.includes(rawInitialState.id));
+        return [
+          rawInitialStates$1.map(rawInitialState => ({
+            id: rawInitialState.id,
+            startBlock: rawInitialState.startBlock,
+            endBlock: Primitive_option.fromNull(rawInitialState.endBlock),
+            maxReorgDepth: rawInitialState.maxReorgDepth,
+            progressBlockNumber: rawInitialState.progressBlockNumber,
+            progressBlockTime: InternalTable.Chains.blockTimeFromDb(rawInitialState.progressBlockTime),
+            numEventsProcessed: rawInitialState.numEventsProcessed,
+            firstEventBlockNumber: Primitive_option.fromNull(rawInitialState.firstEventBlockNumber),
+            timestampCaughtUpToHeadOrEndblock: Primitive_option.fromNull(rawInitialState.timestampCaughtUpToHeadOrEndblock),
+            addressRows: rawInitialState.addressRows,
+            sourceBlockNumber: rawInitialState.sourceBlockNumber
+          })),
+          Frontier.fromEntries(rawInitialStates$1.map(rawInitialState => [
+            rawInitialState.id,
+            BigInt(rawInitialState.checkpointId)
+          ]))
+        ];
+      }),
       sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema)),
       Promise.all([
         InternalTable.EnvioInfo.read(sql, pgSchema),
@@ -1451,21 +1542,22 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
         }
       })
     ]);
-    let match$1 = match[4];
+    let match$1 = match[3];
     let storedContractMapping = match$1[1];
     let storedEnvioInfo = match$1[0];
-    let chains = match[1];
+    let match$2 = match[1];
+    let checkpointFrontier = match$2[1];
+    let chains = match$2[0];
     throwIfIncompatible(storedEnvioInfo, storedContractMapping);
     await reloadIndexCatalog();
-    let checkpointId = BigInt(match[2][0].id);
-    let reorgCheckpoints = match[3].map(raw => ({
+    let reorgCheckpoints = match[2].map(raw => ({
       id: BigInt(raw.id),
       chain_id: ChainId.normalizeOrThrow(raw.chain_id),
       block_number: raw.block_number,
       block_hash: raw.block_hash
     }));
     if (sink !== undefined) {
-      await sink.resume(checkpointId, chains, entities);
+      await sink.resume(checkpointFrontier, chains, entities);
     }
     return {
       cleanRun: false,
@@ -1473,7 +1565,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       envioInfo: storedEnvioInfo,
       cache: match[0],
       chains: chains,
-      checkpointId: checkpointId,
+      checkpointFrontier: checkpointFrontier,
       reorgCheckpoints: reorgCheckpoints
     };
   };
@@ -1487,7 +1579,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
   let getRollbackTargetCheckpoint = (reorgChainId, lastKnownValidBlockNumber) => InternalTable.Checkpoints.getRollbackTargetCheckpoint(sql, pgSchema, reorgChainId, lastKnownValidBlockNumber);
   let getRollbackProgressDiff = floors => InternalTable.Checkpoints.getRollbackProgressDiff(sql, pgSchema, floors);
   let getRollbackData = async (entityConfig, floors) => {
-    let params = CheckpointBounds.params(floors.floors);
+    let params = CheckpointSequence.params(floors.checkpointBounds);
     let match = await Promise.all([
       sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema, floors), params, {prepare: true}),
       sql.unsafe(makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema, floors), params, {prepare: true})
@@ -1521,7 +1613,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       S$RescriptSchema.parseOrThrow(restoredEntitiesResult, Table.pgRowsSchema(entityConfig.table))
     ];
   };
-  let writeBatchMethod = async (batch, rollback, isInReorgThreshold, config, allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, onWrite) => {
+  let writeBatchMethod = async (batch, rollback, config, allEntities, updatedEffectsCache, updatedEntities, registeredAddresses, chainMetaData, onWrite) => {
     let pgUpdates = [];
     let chUpdates = [];
     for (let i = 0, i_finish = updatedEntities.length; i < i_finish; ++i) {
@@ -1537,14 +1629,14 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     let sinkPromise;
     if (sink !== undefined) {
       let timerRef = Performance.now();
-      sinkPromise = sink.writeBatch(batch, chUpdates).then(() => {
+      sinkPromise = sink.writeBatch(batch, rollback !== undefined ? rollback.diffCheckpoints : [], chUpdates).then(() => {
         onWrite(sink.name, Performance.secondsSince(timerRef));
       }).catch(exn => exn);
     } else {
       sinkPromise = undefined;
     }
     let primaryTimerRef = Performance.now();
-    await writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, pgUpdates, registeredAddresses, sinkPromise, chainMetaData, undefined);
+    await writeBatch(sql, batch, pgSchema, rollback, config, allEntities, setEffectCacheOrThrow, setQueryCache, updatedEffectsCache, pgUpdates, registeredAddresses, sinkPromise, chainMetaData, undefined);
     return onWrite(storageName, Performance.secondsSince(primaryTimerRef));
   };
   let close = () => sql.end();
@@ -1596,21 +1688,14 @@ function makeStorageFromEnv(config, sqlOpt, pgSchemaOpt, isHasuraEnabledOpt) {
     if (missing.length !== 0) {
       Stdlib_JsError.throwWithMessage(`ClickHouse storage is enabled but required env vars are not set: ` + missing.join(", ") + `. Please set them, disable clickhouse in the \`storage\` config, or run \`envio dev\` for a pre-configured local ClickHouse.`);
     }
-    tmp = Sink.makeClickHouse(host, database, username, password, config.chainIdMode);
+    tmp = Sink.makeClickHouse(host, database, username, password, config.checkpointSequence, config.chainIdMode);
   } else {
     tmp = undefined;
   }
-  return make(sql, Env.Db.host, pgSchema, Env.Db.port, Env.Db.user, Env.Db.database, Env.Db.password, isHasuraEnabled, config.chainIdMode, config.ecosystem.name, tmp, isHasuraEnabled ? () => Stdlib_Promise.$$catch(Hasura.trackDatabase(Env.Hasura.graphqlEndpoint, {
+  return make(sql, Env.Db.host, pgSchema, Env.Db.port, Env.Db.user, Env.Db.database, Env.Db.password, isHasuraEnabled, config.chainIdMode, config.ecosystem.name, tmp, config.isolated, isHasuraEnabled ? () => Stdlib_Promise.$$catch(Hasura.trackDatabase(Env.Hasura.graphqlEndpoint, {
       role: Env.Hasura.role,
       secret: Env.Hasura.secret
-    }, pgSchema, Config.getPgUserEntities(config), Env.Hasura.aggregateEntities, Env.Hasura.responseLimit, Schema.make(config.userEntities.map(e => e.table))), err => Promise.resolve(Logging.errorWithExn(Utils.prettifyExn(err), `Error tracking tables`))) : undefined, isHasuraEnabled ? tableNames => Stdlib_Promise.$$catch(Hasura.trackTables(Env.Hasura.graphqlEndpoint, {
-      role: Env.Hasura.role,
-      secret: Env.Hasura.secret
-    }, pgSchema, tableNames.map(tableName => ({
-      tableName: tableName,
-      description: undefined,
-      columnConfigs: {}
-    }))), err => Promise.resolve(Logging.errorWithExn(Utils.prettifyExn(err), `Error tracking new tables`))) : undefined);
+    }, pgSchema, Config.getPgUserEntities(config), Env.Hasura.aggregateEntities, Env.Hasura.responseLimit, Schema.make(config.userEntities.map(e => e.table))), err => Promise.resolve(Logging.errorWithExn(Utils.prettifyExn(err), `Error tracking tables`))) : undefined);
 }
 
 function makePersistenceFromConfig(config, storageOpt) {
@@ -1624,9 +1709,10 @@ export {
   makeClient,
   formatSeconds,
   slowOnLargeDatabaseNotice,
+  partitionTableName,
+  indexTableNames,
   getSchemaIndexes,
   makeCreateTableQuery,
-  partitionTableName,
   rowSchemaCache,
   getRowSchema,
   entityHistoryCache,
@@ -1658,6 +1744,7 @@ export {
   deleteByIdsOrThrow,
   makeInsertDeleteUpdatesQuery,
   executeSet,
+  pickCheckpoints,
   writeBatch,
   rollbackKeyColumns,
   makeGetRollbackPreTargetRowsQuery,