envio 3.5.0-alpha.3 → 3.5.0-rc.0

package/src/PgStorage.res.mjs

@@ -9,6 +9,7 @@ import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
 import * as Hasura from "./Hasura.res.mjs";
 import * as Schema from "./db/Schema.res.mjs";
+import * as ChainId from "./ChainId.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as EntityId from "./EntityId.res.mjs";
 import * as Internal from "./Internal.res.mjs";
@@ -16,14 +17,16 @@ import Postgres from "postgres";
 import * as ChainState from "./ChainState.res.mjs";
 import * as Performance from "./bindings/Performance.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
+import * as IndexCatalog from "./db/IndexCatalog.res.mjs";
+import * as IndexManager from "./db/IndexManager.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
-import * as IndexRegistry from "./db/IndexRegistry.res.mjs";
 import * as InternalTable from "./db/InternalTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Child_process from "child_process";
 import * as Stdlib_JsError from "@rescript/runtime/lib/es6/Stdlib_JsError.js";
 import * as Stdlib_Promise from "@rescript/runtime/lib/es6/Stdlib_Promise.js";
+import * as IndexDefinition from "./db/IndexDefinition.res.mjs";
 import * as Primitive_object from "@rescript/runtime/lib/es6/Primitive_object.js";
 import * as Primitive_option from "@rescript/runtime/lib/es6/Primitive_option.js";
 import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
@@ -45,108 +48,33 @@ function makeClient() {
   });
 }
 
-function makeCreateIndexQuery(tableName, indexFields, pgSchema) {
-  let indexName = tableName + "_" + indexFields.join("_");
-  let index = indexFields.map(idx => `"` + idx + `"`).join(", ");
-  return `CREATE INDEX IF NOT EXISTS "` + indexName + `" ON "` + pgSchema + `"."` + tableName + `"(` + index + `);`;
-}
-
-function directionToSql(direction) {
-  if (direction === "Asc") {
-    return "";
-  } else {
-    return " DESC";
-  }
-}
-
-function directionToIndexName(direction) {
-  if (direction === "Asc") {
-    return "";
-  } else {
-    return "_desc";
-  }
-}
-
-function makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema) {
-  let indexName = tableName + "_" + indexFields.map(f => f.fieldName + directionToIndexName(f.direction)).join("_");
-  let index = indexFields.map(f => `"` + f.fieldName + `"` + directionToSql(f.direction)).join(", ");
-  return `CREATE INDEX IF NOT EXISTS "` + indexName + `" ON "` + pgSchema + `"."` + tableName + `"(` + index + `);`;
-}
-
-function makeCreateTableIndexesQuery(table, pgSchema) {
-  let tableName = table.tableName;
-  let createIndex = indexField => makeCreateIndexQuery(tableName, [indexField], pgSchema);
-  let createCompositeIndex = indexFields => makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema);
-  let singleIndexes = Table.getSingleIndexes(table);
-  let compositeIndexes = Table.getCompositeIndexes(table);
-  return singleIndexes.map(createIndex).join("\n") + compositeIndexes.map(createCompositeIndex).join("\n");
-}
-
-function schemaIndexDescription(param) {
-  return param.tableName + "_" + param.indexFields.map(f => f.fieldName + directionToIndexName(f.direction)).join("_");
-}
-
-function schemaIndexName(schemaIndex) {
-  return IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex));
-}
-
-function schemaIndexKey(param) {
-  return IndexRegistry.makeKey(param.tableName, IndexRegistry.fromIndexFields(param.indexFields), IndexRegistry.btree);
-}
-
-function makeCreateSchemaIndexQuery(schemaIndex, pgSchema) {
-  let columns = schemaIndex.indexFields.map(f => `"` + f.fieldName + `"` + directionToSql(f.direction)).join(", ");
-  return `CREATE INDEX IF NOT EXISTS "` + IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex)) + `" ON "` + pgSchema + `"."` + schemaIndex.tableName + `"(` + columns + `);`;
-}
-
 function formatSeconds(timeRef) {
   return (Math.round(Performance.secondsSince(timeRef) * 100) / 100).toString();
 }
 
 let slowOnLargeDatabaseNotice = "This can take a long time on a large database.";
 
-function makeReindexQuery(pgSchema, indexName) {
-  return `REINDEX INDEX "` + pgSchema + `"."` + indexName + `";`;
-}
-
-function nameCollisionError(indexName, pgSchema) {
-  return new Error(`Cannot create the index "` + indexName + `" in schema "` + pgSchema + `". An index of that name already exists and PostgreSQL reports it as invalid, but it isn't one the indexer created — it covers different columns, or it is a unique or partial index — so it is left untouched. Drop or repair it by hand, then restart the indexer.`);
-}
-
-function makeSingleColumnSchemaIndex(tableName, column) {
-  return {
-    tableName: tableName,
-    indexFields: [{
-        fieldName: column,
-        direction: "Asc"
-      }]
-  };
-}
-
 function getSchemaIndexes(entities) {
   let derivedSchema = Schema.make(entities.map(e => e.table));
   let all = [];
   entities.forEach(param => {
     let table = param.table;
     Table.getSingleIndexes(table).forEach(column => {
-      all.push(makeSingleColumnSchemaIndex(table.tableName, column));
+      all.push(IndexDefinition.single(table.tableName, column));
     });
     Table.getCompositeIndexes(table).forEach(indexFields => {
-      all.push({
-        tableName: table.tableName,
-        indexFields: indexFields
-      });
+      all.push(IndexDefinition.fromIndexFields(table.tableName, indexFields));
     });
   });
   entities.forEach(param => {
     Table.getDerivedFromFields(param.table).forEach(derivedFromField => {
       let column = Utils.unwrapResultExn(Schema.getDerivedFromPgFieldName(derivedSchema, derivedFromField));
-      all.push(makeSingleColumnSchemaIndex(derivedFromField.derivedFromEntity, column));
+      all.push(IndexDefinition.single(derivedFromField.derivedFromEntity, column));
     });
   });
   let seen = new Set();
-  return all.filter(schemaIndex => {
-    let key = schemaIndexKey(schemaIndex);
+  return all.filter(definition => {
+    let key = IndexDefinition.key(definition);
     if (seen.has(key)) {
       return false;
     } else {
@@ -156,12 +84,13 @@ function getSchemaIndexes(entities) {
   });
 }
 
-function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText) {
+function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText, chainIdModeOpt) {
+  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
   let fieldsMapped = Table.getFields(table).map(field => {
     let defaultValue = field.defaultValue;
     let isNullable = field.isNullable;
     let fieldName = Table.getPgDbFieldName(field);
-    return `"` + fieldName + `" ` + Table.getPgFieldType(field.fieldType, pgSchema, field.isArray, isNumericArrayAsText, isNullable) + (
+    return `"` + fieldName + `" ` + Table.getPgFieldType(field.fieldType, pgSchema, field.isArray, isNumericArrayAsText, isNullable, chainIdMode) + (
       defaultValue !== undefined ? ` DEFAULT ` + defaultValue : (
           isNullable ? `` : ` NOT NULL`
         )
@@ -244,12 +173,13 @@ function getEntityHistory(entityConfig) {
   return cache$1;
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt, deferSchemaIndexesOpt) {
+function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt, deferSchemaIndexesOpt, chainIdModeOpt) {
   let chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
   let entities = entitiesOpt !== undefined ? entitiesOpt : [];
   let enums = enumsOpt !== undefined ? enumsOpt : [];
   let isEmptyPgSchema = isEmptyPgSchemaOpt !== undefined ? isEmptyPgSchemaOpt : false;
   let deferSchemaIndexes = deferSchemaIndexesOpt !== undefined ? deferSchemaIndexesOpt : false;
+  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
   let generalTables = [
     InternalTable.Chains.table,
     InternalTable.EnvioInfo.table,
@@ -262,7 +192,6 @@ function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfi
     allTables.push(getEntityHistory(entityConfig).table);
   });
   let schemaIndexes = getSchemaIndexes(entities);
-  IndexRegistry.validateIndexNamesOrThrow(schemaIndexes.map(schemaIndexDescription));
   let query = {
     contents: (
       isEmptyPgSchema && pgSchema === "public" ? `CREATE SCHEMA IF NOT EXISTS "` + pgSchema + `";\n` : `DROP SCHEMA IF EXISTS "` + pgSchema + `" CASCADE;
@@ -275,11 +204,11 @@ GRANT ALL ON SCHEMA "` + pgSchema + `" TO public;`)
     query.contents = query.contents + "\n" + enumCreateQuery;
   });
   allTables.forEach(table => {
-    query.contents = query.contents + "\n" + makeCreateTableQuery(table, pgSchema, isHasuraEnabled);
+    query.contents = query.contents + "\n" + makeCreateTableQuery(table, pgSchema, isHasuraEnabled, chainIdMode);
   });
   if (!deferSchemaIndexes) {
-    schemaIndexes.forEach(schemaIndex => {
-      query.contents = query.contents + "\n" + makeCreateSchemaIndexQuery(schemaIndex, pgSchema);
+    schemaIndexes.forEach(definition => {
+      query.contents = query.contents + "\n" + IndexDefinition.makeCreateQuery(definition, pgSchema);
     });
   }
   query.contents = query.contents + "\n" + InternalTable.Views.makeMetaViewQuery(pgSchema);
@@ -368,8 +297,9 @@ function makeLoadAllQuery(pgSchema, tableName) {
   return `SELECT * FROM "` + pgSchema + `"."` + tableName + `";`;
 }
 
-function makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents) {
-  let match = Table.toSqlParams(table, itemSchema, pgSchema);
+function makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents, chainIdModeOpt) {
+  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
+  let match = Table.toSqlParams(table, itemSchema, pgSchema, chainIdMode);
   let quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
   let primaryKeyFieldNames = Table.getPgPrimaryKeyFieldNames(table);
   return `INSERT INTO "` + pgSchema + `"."` + table.tableName + `" (` + match.quotedFieldNames.join(", ") + `)
@@ -380,8 +310,9 @@ SELECT * FROM unnest(` + match.arrayFieldTypes.map((arrayFieldType, idx) => `$`
   ) + ";";
 }
 
-function makeInsertValuesSetQuery(pgSchema, table, itemSchema, itemsCount) {
-  let match = Table.toSqlParams(table, itemSchema, pgSchema);
+function makeInsertValuesSetQuery(pgSchema, table, itemSchema, itemsCount, chainIdModeOpt) {
+  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
+  let match = Table.toSqlParams(table, itemSchema, pgSchema, chainIdMode);
   let quotedNonPrimaryFieldNames = match.quotedNonPrimaryFieldNames;
   let quotedFieldNames = match.quotedFieldNames;
   let primaryKeyFieldNames = Table.getPgPrimaryKeyFieldNames(table);
@@ -408,19 +339,20 @@ VALUES` + placeholders + (
   ) + ";";
 }
 
-function makeTableBatchSetQuery(pgSchema, table, itemSchema) {
-  let match = Table.toSqlParams(table, itemSchema, pgSchema);
+function makeTableBatchSetQuery(pgSchema, table, itemSchema, chainIdModeOpt) {
+  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
+  let match = Table.toSqlParams(table, itemSchema, pgSchema, chainIdMode);
   let isRawEvents = table.tableName === InternalTable.RawEvents.table.tableName;
   let isHistoryUpdate = table.tableName.startsWith(EntityHistory.historyTablePrefix);
   if ((isRawEvents || !match.hasArrayField) && !isHistoryUpdate) {
     return {
-      query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents),
+      query: makeInsertUnnestSetQuery(pgSchema, table, itemSchema, isRawEvents, chainIdMode),
       convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.unnest(match.dbSchema), "Output", "Input", "Sync", false),
       isInsertValues: false
     };
   } else {
     return {
-      query: makeInsertValuesSetQuery(pgSchema, table, itemSchema, 500),
+      query: makeInsertValuesSetQuery(pgSchema, table, itemSchema, 500, chainIdMode),
       convertOrThrow: S$RescriptSchema.compile(S$RescriptSchema.preprocess(S$RescriptSchema.unnest(itemSchema), param => ({
         s: prim => prim.flat(1)
       })), "Output", "Input", "Sync", false),
@@ -496,7 +428,8 @@ function classifyWriteError(specificError, table, exn) {
 
 let setQueryCache = new WeakMap();
 
-async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
+async function setOrThrow(sql, items, table, itemSchema, pgSchema, chainIdModeOpt) {
+  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
   if (items.length === 0) {
     return;
   }
@@ -505,7 +438,7 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
   if (cached !== undefined) {
     data = Primitive_option.valFromOption(cached);
   } else {
-    let newQuery = makeTableBatchSetQuery(pgSchema, table, itemSchema);
+    let newQuery = makeTableBatchSetQuery(pgSchema, table, itemSchema, chainIdMode);
     setQueryCache.set(table, newQuery);
     data = newQuery;
   }
@@ -519,7 +452,7 @@ async function setOrThrow(sql, items, table, itemSchema, pgSchema) {
       let chunkSize = chunk.length;
       let isFullChunk = chunkSize === 500;
       let params = data.convertOrThrow(chunk);
-      let response = isFullChunk ? sql.unsafe(data.query, params, {prepare: true}) : sql.unsafe(makeInsertValuesSetQuery(pgSchema, table, itemSchema, chunkSize), params);
+      let response = isFullChunk ? sql.unsafe(data.query, params, {prepare: true}) : sql.unsafe(makeInsertValuesSetQuery(pgSchema, table, itemSchema, chunkSize, chainIdMode), params);
       responses.push(response);
     });
     await Promise.all(responses);
@@ -654,7 +587,7 @@ function makeInsertDeleteUpdatesQuery(entityConfig, pgSchema) {
     }
   });
   let selectPartsStr = selectParts.join(", ");
-  let checkpointIdPgType = Table.getPgFieldType(EntityHistory.checkpointIdFieldType, pgSchema, false, false, false);
+  let checkpointIdPgType = Table.getPgFieldType(EntityHistory.checkpointIdFieldType, pgSchema, false, false, false, undefined);
   let idPgType = Table.getIdPgFieldType(entityConfig.table, pgSchema);
   return `INSERT INTO "` + pgSchema + `"."` + historyTableName + `" (` + allHistoryFieldNamesStr + `)
 SELECT ` + selectPartsStr + `
@@ -671,6 +604,7 @@ function executeSet(sql, items, dbFunction) {
 
 async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, updatedEntities, sinkPromise, chainMetaData, escapeTables) {
   try {
+    let chainIdMode = config.chainIdMode;
     let shouldSaveHistory = Config.shouldSaveHistory(config, isInReorgThreshold);
     let specificError = {
       contents: undefined
@@ -682,7 +616,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
         if (item.kind !== 0) {
           return;
         }
-        let coordinate = Internal.getItemChainId(item).toString() + `-` + item.blockNumber.toString() + `-` + item.logIndex.toString();
+        let coordinate = ChainId.toString(Internal.getItemChainId(item)) + `-` + item.blockNumber.toString() + `-` + item.logIndex.toString();
         if (seenLogCoordinates.has(coordinate)) {
           return;
         } else {
@@ -699,7 +633,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
     }
     let setRawEvents = async sql => {
       try {
-        return await executeSet(sql, rawEvents, (sql, items) => setOrThrow(sql, items, InternalTable.RawEvents.table, InternalTable.RawEvents.schema, pgSchema));
+        return await executeSet(sql, rawEvents, (sql, items) => setOrThrow(sql, items, InternalTable.RawEvents.table, InternalTable.RawEvents.schema, pgSchema, chainIdMode));
       } catch (raw_exn) {
         let exn = Primitive_exceptions.internalToException(raw_exn);
         return classifyWriteError(specificError, InternalTable.RawEvents.table, exn);
@@ -776,14 +710,14 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
                 }));
               }
               let entityHistory = getEntityHistory(entityConfig);
-              promises.push(setOrThrow(sql, batchSetUpdates, entityHistory.table, entityHistory.setChangeSchema, pgSchema));
+              promises.push(setOrThrow(sql, batchSetUpdates, entityHistory.table, entityHistory.setChangeSchema, pgSchema, chainIdMode));
             }
           }
           if (Utils.$$Array.notEmpty(entitiesToSet)) {
             if (shouldRemoveInvalidUtf8) {
               removeInvalidUtf8InPlace(entitiesToSet);
             }
-            promises.push(setOrThrow(sql, entitiesToSet, entityConfig.table, entityConfig.schema, pgSchema));
+            promises.push(setOrThrow(sql, entitiesToSet, entityConfig.table, entityConfig.schema, pgSchema, chainIdMode));
           }
           if (Utils.$$Array.notEmpty(idsToDelete)) {
             promises.push(deleteByIdsOrThrow(sql, pgSchema, idsToDelete, entityConfig.table));
@@ -800,7 +734,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
     if (rollback !== undefined) {
       let rollbackTargetCheckpointId = rollback.targetCheckpointId;
       rollbackTables = sql => {
-        let promises = allEntities.map(entityConfig => EntityHistory.rollback(sql, pgSchema, entityConfig.name, entityConfig.index, rollbackTargetCheckpointId));
+        let promises = allEntities.filter(entityConfig => entityConfig.storage.postgres).map(entityConfig => EntityHistory.rollback(sql, pgSchema, entityConfig.name, entityConfig.index, rollbackTargetCheckpointId));
         promises.push(InternalTable.Checkpoints.rollback(sql, pgSchema, rollbackTargetCheckpointId));
         return Promise.all(promises);
       };
@@ -826,7 +760,7 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
             setOperations.push(sql => InternalTable.Chains.setMeta(sql, pgSchema, chainMetaData));
           }
           if (shouldSaveHistory) {
-            setOperations.push(sql => InternalTable.Checkpoints.insert(sql, pgSchema, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointBlockHashes, batch.checkpointEventsProcessed));
+            setOperations.push(sql => InternalTable.Checkpoints.insert(sql, pgSchema, batch.checkpointIds, batch.checkpointChainIds, batch.checkpointBlockNumbers, batch.checkpointBlockHashes, batch.checkpointEventsProcessed, chainIdMode));
           }
           await Promise.all(setOperations.map(dbFunc => dbFunc(sql)));
           if (sinkPromise === undefined) {
@@ -901,7 +835,8 @@ let rollbackRowStateSchema = Utils.$$WeakMap.memoize(table => S$RescriptSchema.o
 
 let rollbackRemovedIdsSchema = Utils.$$WeakMap.memoize(table => S$RescriptSchema.array(S$RescriptSchema.object(s => s.f(Table.idFieldName, Table.getIdSchema(table)))));
 
-function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, sink, onInitialize, onNewTables) {
+function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, chainIdModeOpt, sink, onInitialize, onNewTables) {
+  let chainIdMode = chainIdModeOpt !== undefined ? chainIdModeOpt : "int32";
   let containerName = "envio-postgres";
   let psqlExecOptions_env = Object.fromEntries([
     [
@@ -918,20 +853,25 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
   };
   let cacheDirPath = Path.resolve(".envio", "cache");
   let storageName = "postgres";
-  let indexRegistry = IndexRegistry.make();
-  let reloadIndexRegistry = async () => {
-    let rows = S$RescriptSchema.parseOrThrow(await sql.unsafe(IndexRegistry.makeCatalogQuery(pgSchema)), IndexRegistry.catalogRowsSchema);
-    let invalidIndexNames = IndexRegistry.reload(indexRegistry, rows);
+  let indexManager = IndexManager.make();
+  let loadCatalogRows = (sql, indexName) => sql.unsafe(IndexCatalog.makeQuery(pgSchema, indexName)).then(rows => S$RescriptSchema.parseOrThrow(rows, IndexCatalog.rowsSchema));
+  let refreshIndexCatalog = async () => {
+    let rows = await loadCatalogRows(sql, undefined);
+    return IndexManager.reload(indexManager, rows);
+  };
+  let reloadIndexCatalog = async () => {
+    let catalog = await refreshIndexCatalog();
+    let invalidIndexNames = IndexCatalog.invalidNames(catalog);
     if (invalidIndexNames.length !== 0) {
       Logging.warn({
         storage: storageName,
-        msg: `Ignoring invalid PostgreSQL indexes in schema "` + pgSchema + `". They are left untouched — drop and recreate them manually if you need them.`,
+        msg: `Ignoring invalid PostgreSQL indexes in schema "` + pgSchema + `". They can't serve queries, so the indexer builds its own alongside them.`,
         indexes: invalidIndexNames
       });
     }
     return Logging.info({
       storage: storageName,
-      msg: `Found ` + IndexRegistry.size(indexRegistry).toString() + ` existing indexes in the indexer schema.`
+      msg: `Found ` + IndexCatalog.size(catalog).toString() + ` existing indexes in the indexer schema.`
     });
   };
   let isInitialized = async () => Utils.$$Array.notEmpty(await sql.unsafe(`SELECT table_schema FROM information_schema.tables WHERE table_schema = '` + pgSchema + `' AND (table_name = '` + "event_sync_state" + `' OR table_name = '` + InternalTable.Chains.table.tableName + `');`));
@@ -950,21 +890,23 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
         let subEntries = await Fs.promises.readdir(entryPath);
         let tsvs = subEntries.filter(sub => sub.endsWith(".tsv"));
         let chainId = Internal.EffectCache.parseChainId(entry);
-        if (chainId !== undefined) {
-          tsvs.forEach(sub => {
-            let effectName = sub.slice(0, -4);
-            let table = Internal.makeCacheTable(effectName, chainId);
-            result.push([
-              table,
-              Path.join(entryPath, sub)
-            ]);
-          });
-          return;
-        } else if (Utils.$$Array.notEmpty(tsvs)) {
-          return Stdlib_JsError.throwWithMessage(`Invalid effect cache directory ".envio/cache/` + entry + `". Chain cache directories must be named by a numeric chain id (e.g. "1"). Found cache files: ` + tsvs.join(", ") + `.`);
-        } else {
-          return;
+        if (chainId === undefined) {
+          if (Utils.$$Array.notEmpty(tsvs)) {
+            return Stdlib_JsError.throwWithMessage(`Invalid effect cache directory ".envio/cache/` + entry + `". Chain cache directories must be named by a numeric chain id (e.g. "1"). Found cache files: ` + tsvs.join(", ") + `.`);
+          } else {
+            return;
+          }
         }
+        let chainId$1 = Primitive_option.valFromOption(chainId);
+        tsvs.forEach(sub => {
+          let effectName = sub.slice(0, -4);
+          let table = Internal.makeCacheTable(effectName, chainId$1);
+          result.push([
+            table,
+            Path.join(entryPath, sub)
+          ]);
+        });
+        return;
       }
       if (!entry.endsWith(".tsv")) {
         return;
@@ -999,7 +941,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
           await Promise.all(entries.map(param => {
             let inputFile = param[1];
             let table = param[0];
-            return sql.unsafe(makeCreateTableQuery(table, pgSchema, false)).then(() => {
+            return sql.unsafe(makeCreateTableQuery(table, pgSchema, false, undefined)).then(() => {
               let command = psqlExec$1 + ` -c 'COPY "` + pgSchema + `"."` + table.tableName + `" FROM STDIN WITH (FORMAT text, HEADER);' < ` + inputFile;
               return new Promise((resolve, reject) => {
                 Child_process.exec(command, psqlExecOptions, (error, stdout, param) => {
@@ -1053,7 +995,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     if (sink !== undefined) {
       await sink.initialize(chainConfigs, chEntities, enums);
     }
-    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames), true);
+    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames), true, chainIdMode);
     await sql.begin(async sql => {
       await Promise.all(queries.map(query => sql.unsafe(query)));
       return await InternalTable.EnvioInfo.write(sql, pgSchema, envioInfo);
@@ -1071,15 +1013,16 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
       });
     });
     if (ids.length !== 0) {
+      let addrChainIdArrayType = Table.getPgFieldType("ChainId", pgSchema, true, false, false, chainIdMode);
       await sql.unsafe(`INSERT INTO "` + pgSchema + `"."` + Config.EnvioAddresses.table.tableName + `" ("id", "chain_id", "registration_block", "registration_log_index", "contract_name")
-SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::text[]) AS t(id, chain_id, contract_name);`, [
+SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::` + addrChainIdArrayType + `,$3::text[]) AS t(id, chain_id, contract_name);`, [
         ids,
         addrChainIds,
         addrContractNames
       ]);
     }
     let cache = await restoreEffectCache(true);
-    await reloadIndexRegistry();
+    await reloadIndexCatalog();
     if (onInitialize !== undefined) {
       await onInitialize();
     }
@@ -1153,73 +1096,102 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
     filters.forEach(collect);
     return columns;
   };
-  let makeEnsureIndexQuery = schemaIndex => {
-    let indexName = IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex));
-    let match = IndexRegistry.getInvalid(indexRegistry, indexName);
-    if (match === undefined) {
-      return makeCreateSchemaIndexQuery(schemaIndex, pgSchema);
+  let runAndVerify = async (sql, prepared) => {
+    for (let idx = 0, idx_finish = prepared.queries.length; idx < idx_finish; ++idx) {
+      await sql.unsafe(prepared.queries[idx]);
     }
-    if (match.isRepairable) {
-      if (match.key === schemaIndexKey(schemaIndex)) {
-        return makeReindexQuery(pgSchema, indexName);
-      }
-      throw nameCollisionError(indexName, pgSchema);
+    let rows = await loadCatalogRows(sql, prepared.name);
+    return IndexManager.verifyOrThrow(prepared, rows, pgSchema);
+  };
+  let resyncIndex = async name => {
+    let rows;
+    try {
+      rows = await loadCatalogRows(sql, name);
+    } catch (raw_exn) {
+      let exn = Primitive_exceptions.internalToException(raw_exn);
+      return Logging.debug({
+        storage: storageName,
+        msg: `Could not re-read the index "` + name + `" after a failed build. The next attempt reads it again.`,
+        err: Utils.prettifyExn(exn)
+      });
     }
-    throw nameCollisionError(indexName, pgSchema);
+    return IndexManager.resync(indexManager, name, rows);
   };
-  let isRepair = schemaIndex => Stdlib_Option.isSome(IndexRegistry.getInvalid(indexRegistry, IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex))));
   let ensureQueryIndexes = async (table, filters) => {
-    let missing = Stdlib_Array.filterMap(filterColumns(table, filters), column => {
-      let schemaIndex = makeSingleColumnSchemaIndex(table.tableName, column);
-      if (IndexRegistry.has(indexRegistry, schemaIndexKey(schemaIndex))) {
-        return;
-      } else {
-        return schemaIndex;
-      }
-    });
-    if (Utils.$$Array.notEmpty(missing)) {
-      await Promise.all(missing.map(schemaIndex => {
-        let indexName = IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex));
-        return IndexRegistry.ensure(indexRegistry, schemaIndexKey(schemaIndex), table.tableName, async () => {
-          let query = makeEnsureIndexQuery(schemaIndex);
-          let verb = isRepair(schemaIndex) ? "Repairing invalid index" : "Creating index";
-          Logging.info({
-            storage: storageName,
-            msg: verb + ` "` + indexName + `" to serve a getWhere query on "` + table.tableName + `". Writes to the table are paused until it completes. ` + slowOnLargeDatabaseNotice
-          });
-          let timeRef = Performance.now();
-          await sql.unsafe(query);
-          IndexRegistry.clearInvalidName(indexRegistry, indexName);
-          return Logging.info({
-            storage: storageName,
-            msg: `Index "` + indexName + `" is ready after ` + formatSeconds(timeRef) + `s. Resuming indexing.`
-          });
-        }).catch(exn => Logging.warn({
+    let columns = filterColumns(table, filters);
+    await Promise.all(columns.map(column => {
+      let definition = IndexDefinition.single(table.tableName, column);
+      return Stdlib_Promise.$$catch(IndexManager.ensure(indexManager, definition, "LeadingColumns", async () => {
+        let prepared = IndexManager.prepare(indexManager, definition, "LeadingColumns", pgSchema);
+        if (prepared === undefined) {
+          return;
+        }
+        let verb = prepared.isRebuild ? "Rebuilding unusable index" : "Creating index";
+        Logging.info({
+          storage: storageName,
+          msg: verb + ` "` + prepared.name + `" to serve a getWhere query on "` + table.tableName + `". Writes to the table are paused until it completes. ` + slowOnLargeDatabaseNotice
+        });
+        let timeRef = Performance.now();
+        let entry = await runAndVerify(sql, prepared);
+        IndexManager.record(indexManager, entry);
+        return Logging.info({
+          storage: storageName,
+          msg: `Index "` + prepared.name + `" is ready after ` + formatSeconds(timeRef) + `s. Resuming indexing.`
+        });
+      }), async exn => {
+        Logging.warn({
           storage: storageName,
-          msg: `Failed to create the index "` + indexName + `" for a getWhere query. The query runs without it.`,
+          msg: `Failed to create an index on "` + table.tableName + `"("` + column + `") for a getWhere query. The query runs without it.`,
           err: Utils.prettifyExn(exn)
-        }));
-      }));
-      return;
-    }
+        });
+        return await resyncIndex(IndexDefinition.name(definition));
+      });
+    }));
+  };
+  let ensureSchemaIndexes = async entities => {
+    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres));
+    await Promise.all(schemaIndexes.map(definition => Stdlib_Promise.$$catch(IndexManager.ensure(indexManager, definition, "Exact", async () => {
+      let prepared = IndexManager.prepare(indexManager, definition, "Exact", pgSchema);
+      if (prepared === undefined) {
+        return;
+      }
+      let verb = prepared.isRebuild ? "Rebuilding unusable index" : "Creating missing index";
+      Logging.info({
+        storage: storageName,
+        msg: verb + ` "` + prepared.name + `" the schema promises but the database no longer has. Writes to the table are paused until it completes. ` + slowOnLargeDatabaseNotice
+      });
+      let timeRef = Performance.now();
+      let entry = await runAndVerify(sql, prepared);
+      IndexManager.record(indexManager, entry);
+      return Logging.info({
+        storage: storageName,
+        msg: `Index "` + prepared.name + `" is ready after ` + formatSeconds(timeRef) + `s.`
+      });
+    }), async exn => {
+      Logging.warn({
+        storage: storageName,
+        msg: `Failed to restore the schema index "` + IndexDefinition.name(definition) + `". Queries relying on it run unindexed until the next restart.`,
+        err: Utils.prettifyExn(exn)
+      });
+      return await resyncIndex(IndexDefinition.name(definition));
+    })));
   };
   let finalizeBackfill = async (entities, chainIds, readyAt) => {
     let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres));
-    IndexRegistry.validateIndexNamesOrThrow(schemaIndexes.map(schemaIndexDescription));
-    let missing = schemaIndexes.filter(schemaIndex => !IndexRegistry.has(indexRegistry, schemaIndexKey(schemaIndex)));
-    let repaired = missing.filter(isRepair);
-    if (Utils.$$Array.notEmpty(repaired)) {
+    let missing = Stdlib_Array.filterMap(schemaIndexes, definition => IndexManager.prepare(indexManager, definition, "Exact", pgSchema));
+    let rebuilt = missing.filter(prepared => prepared.isRebuild);
+    if (rebuilt.length !== 0) {
       Logging.warn({
         storage: storageName,
-        msg: `PostgreSQL reports ` + repaired.length.toString() + ` existing indexes as invalid, so they can't serve queries. Rebuilding them.`,
-        indexes: repaired.map(schemaIndexName)
+        msg: `PostgreSQL reports ` + rebuilt.length.toString() + ` of the indexer's own indexes as invalid, so they can't serve queries. Rebuilding them.`,
+        indexes: rebuilt.map(prepared => prepared.name)
       });
     }
     if (missing.length !== 0) {
       Logging.info({
         storage: storageName,
         msg: `Creating the ` + missing.length.toString() + ` remaining schema indexes before the indexer reports ready. Writes are paused until they are committed. ` + slowOnLargeDatabaseNotice,
-        indexes: missing.map(schemaIndexName)
+        indexes: missing.map(prepared => prepared.name)
       });
     } else {
       Logging.info({
@@ -1227,28 +1199,40 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
         msg: `All ` + schemaIndexes.length.toString() + ` schema indexes are already in place. Marking the indexer ready.`
       });
     }
-    let indexQueries = missing.map(makeEnsureIndexQuery);
     let timeRef = Performance.now();
+    for (let idx = 0, idx_finish = missing.length; idx < idx_finish; ++idx) {
+      let prepared = missing[idx];
+      let exit = 0;
+      let entry;
+      try {
+        entry = await runAndVerify(sql, prepared);
+        exit = 1;
+      } catch (exn) {
+        await resyncIndex(prepared.name);
+        throw exn;
+      }
+      if (exit === 1) {
+        IndexManager.record(indexManager, entry);
+      }
+    }
+    let setReadyAtQuery = InternalTable.Chains.makeSetReadyAtQuery(pgSchema);
     await sql.begin(async sql => {
-      for (let idx = 0, idx_finish = indexQueries.length; idx < idx_finish; ++idx) {
-        await sql.unsafe(indexQueries[idx]);
+      for (let idx = 0, idx_finish = chainIds.length; idx < idx_finish; ++idx) {
+        await sql.unsafe(setReadyAtQuery, [
+          readyAt,
+          chainIds[idx]
+        ], {prepare: true});
       }
-      return await sql.unsafe(InternalTable.Chains.makeSetReadyAtQuery(pgSchema), [
-        readyAt,
-        chainIds
-      ], {prepare: true});
     });
-    missing.forEach(schemaIndex => IndexRegistry.add(indexRegistry, schemaIndexKey(schemaIndex)));
-    repaired.forEach(schemaIndex => IndexRegistry.clearInvalidName(indexRegistry, IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex))));
     return Logging.info({
       storage: storageName,
       msg: `Committed ` + missing.length.toString() + ` schema indexes and the ready timestamp in ` + formatSeconds(timeRef) + `s.`
     });
   };
-  let setOrThrow$1 = (items, table, itemSchema) => setOrThrow(sql, items, table, itemSchema, pgSchema);
+  let setOrThrow$1 = (items, table, itemSchema) => setOrThrow(sql, items, table, itemSchema, pgSchema, undefined);
   let setEffectCacheOrThrow = async (table, itemSchema, items, initialize) => {
     if (initialize) {
-      await sql.unsafe(makeCreateTableQuery(table, pgSchema, false));
+      await sql.unsafe(makeCreateTableQuery(table, pgSchema, false, undefined));
       if (onNewTables !== undefined) {
         await onNewTables([table.tableName]);
       }
@@ -1321,11 +1305,11 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema)),
       InternalTable.EnvioInfo.read(sql, pgSchema)
     ]);
-    await reloadIndexRegistry();
+    await reloadIndexCatalog();
     let checkpointId = BigInt(match[2][0].id);
     let reorgCheckpoints = match[3].map(raw => ({
       id: BigInt(raw.id),
-      chain_id: raw.chain_id,
+      chain_id: ChainId.normalizeOrThrow(raw.chain_id),
       block_number: raw.block_number,
       block_hash: raw.block_hash
     }));
@@ -1404,6 +1388,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
     resumeInitialState: resumeInitialState,
     loadOrThrow: loadOrThrow,
     ensureQueryIndexes: ensureQueryIndexes,
+    ensureSchemaIndexes: ensureSchemaIndexes,
     finalizeBackfill: finalizeBackfill,
     dumpEffectCache: dumpEffectCache,
     reset: reset,
@@ -1444,11 +1429,11 @@ function makeStorageFromEnv(config, sqlOpt, pgSchemaOpt, isHasuraEnabledOpt) {
     if (missing.length !== 0) {
       Stdlib_JsError.throwWithMessage(`ClickHouse storage is enabled but required env vars are not set: ` + missing.join(", ") + `. Please set them, disable clickhouse in the \`storage\` config, or run \`envio dev\` for a pre-configured local ClickHouse.`);
     }
-    tmp = Sink.makeClickHouse(host, database, username, password);
+    tmp = Sink.makeClickHouse(host, database, username, password, config.chainIdMode);
   } else {
     tmp = undefined;
   }
-  return make(sql, Env.Db.host, pgSchema, Env.Db.port, Env.Db.user, Env.Db.database, Env.Db.password, isHasuraEnabled, tmp, isHasuraEnabled ? () => Stdlib_Promise.$$catch(Hasura.trackDatabase(Env.Hasura.graphqlEndpoint, {
+  return make(sql, Env.Db.host, pgSchema, Env.Db.port, Env.Db.user, Env.Db.database, Env.Db.password, isHasuraEnabled, config.chainIdMode, tmp, isHasuraEnabled ? () => Stdlib_Promise.$$catch(Hasura.trackDatabase(Env.Hasura.graphqlEndpoint, {
       role: Env.Hasura.role,
       secret: Env.Hasura.secret
     }, pgSchema, Config.getPgUserEntities(config), Env.Hasura.aggregateEntities, Env.Hasura.responseLimit, Schema.make(config.allEntities.map(e => e.table))), err => Promise.resolve(Logging.errorWithExn(Utils.prettifyExn(err), `Error tracking tables`))) : undefined, isHasuraEnabled ? tableNames => Stdlib_Promise.$$catch(Hasura.trackTables(Env.Hasura.graphqlEndpoint, {
@@ -1470,20 +1455,8 @@ let maxItemsPerQuery = 500;
 
 export {
   makeClient,
-  makeCreateIndexQuery,
-  directionToSql,
-  directionToIndexName,
-  makeCreateCompositeIndexQuery,
-  makeCreateTableIndexesQuery,
-  schemaIndexDescription,
-  schemaIndexName,
-  schemaIndexKey,
-  makeCreateSchemaIndexQuery,
   formatSeconds,
   slowOnLargeDatabaseNotice,
-  makeReindexQuery,
-  nameCollisionError,
-  makeSingleColumnSchemaIndex,
   getSchemaIndexes,
   makeCreateTableQuery,
   entityHistoryCache,