envio 3.5.0-alpha.1 → 3.5.0-alpha.3

package/src/PgStorage.res.mjs

@@ -10,6 +10,7 @@ import * as Config from "./Config.res.mjs";
 import * as Hasura from "./Hasura.res.mjs";
 import * as Schema from "./db/Schema.res.mjs";
 import * as Logging from "./Logging.res.mjs";
+import * as EntityId from "./EntityId.res.mjs";
 import * as Internal from "./Internal.res.mjs";
 import Postgres from "postgres";
 import * as ChainState from "./ChainState.res.mjs";
@@ -17,6 +18,7 @@ import * as Performance from "./bindings/Performance.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
+import * as IndexRegistry from "./db/IndexRegistry.res.mjs";
 import * as InternalTable from "./db/InternalTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Child_process from "child_process";
@@ -71,13 +73,87 @@ function makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema) {
   return `CREATE INDEX IF NOT EXISTS "` + indexName + `" ON "` + pgSchema + `"."` + tableName + `"(` + index + `);`;
 }
 
-function makeCreateTableIndicesQuery(table, pgSchema) {
+function makeCreateTableIndexesQuery(table, pgSchema) {
   let tableName = table.tableName;
   let createIndex = indexField => makeCreateIndexQuery(tableName, [indexField], pgSchema);
   let createCompositeIndex = indexFields => makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema);
-  let singleIndices = Table.getSingleIndices(table);
-  let compositeIndices = Table.getCompositeIndices(table);
-  return singleIndices.map(createIndex).join("\n") + compositeIndices.map(createCompositeIndex).join("\n");
+  let singleIndexes = Table.getSingleIndexes(table);
+  let compositeIndexes = Table.getCompositeIndexes(table);
+  return singleIndexes.map(createIndex).join("\n") + compositeIndexes.map(createCompositeIndex).join("\n");
+}
+
+function schemaIndexDescription(param) {
+  return param.tableName + "_" + param.indexFields.map(f => f.fieldName + directionToIndexName(f.direction)).join("_");
+}
+
+function schemaIndexName(schemaIndex) {
+  return IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex));
+}
+
+function schemaIndexKey(param) {
+  return IndexRegistry.makeKey(param.tableName, IndexRegistry.fromIndexFields(param.indexFields), IndexRegistry.btree);
+}
+
+function makeCreateSchemaIndexQuery(schemaIndex, pgSchema) {
+  let columns = schemaIndex.indexFields.map(f => `"` + f.fieldName + `"` + directionToSql(f.direction)).join(", ");
+  return `CREATE INDEX IF NOT EXISTS "` + IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex)) + `" ON "` + pgSchema + `"."` + schemaIndex.tableName + `"(` + columns + `);`;
+}
+
+function formatSeconds(timeRef) {
+  return (Math.round(Performance.secondsSince(timeRef) * 100) / 100).toString();
+}
+
+let slowOnLargeDatabaseNotice = "This can take a long time on a large database.";
+
+function makeReindexQuery(pgSchema, indexName) {
+  return `REINDEX INDEX "` + pgSchema + `"."` + indexName + `";`;
+}
+
+function nameCollisionError(indexName, pgSchema) {
+  return new Error(`Cannot create the index "` + indexName + `" in schema "` + pgSchema + `". An index of that name already exists and PostgreSQL reports it as invalid, but it isn't one the indexer created — it covers different columns, or it is a unique or partial index — so it is left untouched. Drop or repair it by hand, then restart the indexer.`);
+}
+
+function makeSingleColumnSchemaIndex(tableName, column) {
+  return {
+    tableName: tableName,
+    indexFields: [{
+        fieldName: column,
+        direction: "Asc"
+      }]
+  };
+}
+
+function getSchemaIndexes(entities) {
+  let derivedSchema = Schema.make(entities.map(e => e.table));
+  let all = [];
+  entities.forEach(param => {
+    let table = param.table;
+    Table.getSingleIndexes(table).forEach(column => {
+      all.push(makeSingleColumnSchemaIndex(table.tableName, column));
+    });
+    Table.getCompositeIndexes(table).forEach(indexFields => {
+      all.push({
+        tableName: table.tableName,
+        indexFields: indexFields
+      });
+    });
+  });
+  entities.forEach(param => {
+    Table.getDerivedFromFields(param.table).forEach(derivedFromField => {
+      let column = Utils.unwrapResultExn(Schema.getDerivedFromPgFieldName(derivedSchema, derivedFromField));
+      all.push(makeSingleColumnSchemaIndex(derivedFromField.derivedFromEntity, column));
+    });
+  });
+  let seen = new Set();
+  return all.filter(schemaIndex => {
+    let key = schemaIndexKey(schemaIndex);
+    if (seen.has(key)) {
+      return false;
+    } else {
+      seen.add(key);
+      return true;
+    }
+  });
 }
 
 function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText) {
@@ -157,7 +233,7 @@ function getEntityHistory(entityConfig) {
     checkpointIdField,
     actionField
   ]), undefined);
-  let setChangeSchema = EntityHistory.makeSetUpdateSchema(entityConfig.schema);
+  let setChangeSchema = EntityHistory.makeSetUpdateSchema(Table.getIdSchema(entityConfig.table), entityConfig.schema);
   let cache_setChangeSchemaRows = S$RescriptSchema.array(setChangeSchema);
   let cache$1 = {
     table: table,
@@ -168,11 +244,12 @@ function getEntityHistory(entityConfig) {
   return cache$1;
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt) {
+function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt, deferSchemaIndexesOpt) {
   let chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
   let entities = entitiesOpt !== undefined ? entitiesOpt : [];
   let enums = enumsOpt !== undefined ? enumsOpt : [];
   let isEmptyPgSchema = isEmptyPgSchemaOpt !== undefined ? isEmptyPgSchemaOpt : false;
+  let deferSchemaIndexes = deferSchemaIndexesOpt !== undefined ? deferSchemaIndexesOpt : false;
   let generalTables = [
     InternalTable.Chains.table,
     InternalTable.EnvioInfo.table,
@@ -180,13 +257,12 @@ function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfi
     InternalTable.RawEvents.table
   ];
   let allTables = generalTables.slice();
-  let allEntityTables = [];
   entities.forEach(entityConfig => {
-    allEntityTables.push(entityConfig.table);
     allTables.push(entityConfig.table);
     allTables.push(getEntityHistory(entityConfig).table);
   });
-  let derivedSchema = Schema.make(allEntityTables);
+  let schemaIndexes = getSchemaIndexes(entities);
+  IndexRegistry.validateIndexNamesOrThrow(schemaIndexes.map(schemaIndexDescription));
   let query = {
     contents: (
       isEmptyPgSchema && pgSchema === "public" ? `CREATE SCHEMA IF NOT EXISTS "` + pgSchema + `";\n` : `DROP SCHEMA IF EXISTS "` + pgSchema + `" CASCADE;
@@ -201,19 +277,11 @@ GRANT ALL ON SCHEMA "` + pgSchema + `" TO public;`)
   allTables.forEach(table => {
     query.contents = query.contents + "\n" + makeCreateTableQuery(table, pgSchema, isHasuraEnabled);
   });
-  allTables.forEach(table => {
-    let indices = makeCreateTableIndicesQuery(table, pgSchema);
-    if (indices !== "") {
-      query.contents = query.contents + "\n" + indices;
-      return;
-    }
-  });
-  entities.forEach(entity => {
-    Table.getDerivedFromFields(entity.table).forEach(derivedFromField => {
-      let indexField = Utils.unwrapResultExn(Schema.getDerivedFromPgFieldName(derivedSchema, derivedFromField));
-      query.contents = query.contents + "\n" + makeCreateIndexQuery(derivedFromField.derivedFromEntity, [indexField], pgSchema);
+  if (!deferSchemaIndexes) {
+    schemaIndexes.forEach(schemaIndex => {
+      query.contents = query.contents + "\n" + makeCreateSchemaIndexQuery(schemaIndex, pgSchema);
     });
-  });
+  }
   query.contents = query.contents + "\n" + InternalTable.Views.makeMetaViewQuery(pgSchema);
   query.contents = query.contents + "\n" + InternalTable.Views.makeChainMetadataViewQuery(pgSchema);
   let initialChainsValuesQuery = InternalTable.Chains.makeInitialValuesQuery(pgSchema, chainConfigs);
@@ -292,8 +360,8 @@ function makeDeleteByIdQuery(pgSchema, tableName) {
   return `DELETE FROM "` + pgSchema + `"."` + tableName + `" WHERE id = $1;`;
 }
 
-function makeDeleteByIdsQuery(pgSchema, tableName) {
-  return `DELETE FROM "` + pgSchema + `"."` + tableName + `" WHERE id = ANY($1::text[]);`;
+function makeDeleteByIdsQuery(pgSchema, tableName, idPgType) {
+  return `DELETE FROM "` + pgSchema + `"."` + tableName + `" WHERE id = ANY($1::` + idPgType + `[]);`;
 }
 
 function makeLoadAllQuery(pgSchema, tableName) {
@@ -547,9 +615,10 @@ async function getConnectedPsqlExec(pgUser, pgHost, pgDatabase, pgPort, containe
 }
 
 async function deleteByIdsOrThrow(sql, pgSchema, ids, table) {
+  let idsJson = Table.encodeIdsToJson(table, ids);
   try {
     await (
-      ids.length !== 1 ? sql.unsafe(makeDeleteByIdsQuery(pgSchema, table.tableName), [ids], {prepare: true}) : sql.unsafe(makeDeleteByIdQuery(pgSchema, table.tableName), ids, {prepare: true})
+      ids.length !== 1 ? sql.unsafe(makeDeleteByIdsQuery(pgSchema, table.tableName, Table.getIdPgFieldType(table, pgSchema)), [idsJson], {prepare: true}) : sql.unsafe(makeDeleteByIdQuery(pgSchema, table.tableName), idsJson, {prepare: true})
     );
     return;
   } catch (raw_exn) {
@@ -586,9 +655,10 @@ function makeInsertDeleteUpdatesQuery(entityConfig, pgSchema) {
   });
   let selectPartsStr = selectParts.join(", ");
   let checkpointIdPgType = Table.getPgFieldType(EntityHistory.checkpointIdFieldType, pgSchema, false, false, false);
+  let idPgType = Table.getIdPgFieldType(entityConfig.table, pgSchema);
   return `INSERT INTO "` + pgSchema + `"."` + historyTableName + `" (` + allHistoryFieldNamesStr + `)
 SELECT ` + selectPartsStr + `
-FROM UNNEST($1::text[], $2::` + checkpointIdPgType + `[]) AS u(` + Table.idFieldName + `, ` + EntityHistory.checkpointIdFieldName + `)`;
+FROM UNNEST($1::` + idPgType + `[], $2::` + checkpointIdPgType + `[]) AS u(` + Table.idFieldName + `, ` + EntityHistory.checkpointIdFieldName + `)`;
 }
 
 function executeSet(sql, items, dbFunction) {
@@ -648,15 +718,16 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
       let orderedIds = [];
       param.changes.forEach(change => {
         let entityId = change.entityId;
-        if (Stdlib_Option.isNone(latestChangeById[entityId])) {
+        let entityKey = EntityId.toKey(entityId);
+        if (Stdlib_Option.isNone(latestChangeById[entityKey])) {
           orderedIds.push(entityId);
         }
-        latestChangeById[entityId] = change;
+        latestChangeById[entityKey] = change;
         if (!shouldSaveHistory) {
           return;
         }
         if (change.checkpointId === diffCheckpointId) {
-          idsWithDiff.add(entityId);
+          idsWithDiff.add(entityKey);
           return;
         }
         if (change.type === "SET") {
@@ -668,13 +739,14 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
       });
       let backfillHistoryIds = new Set();
       orderedIds.forEach(entityId => {
-        let match = latestChangeById[entityId];
+        let entityKey = EntityId.toKey(entityId);
+        let match = latestChangeById[entityKey];
         if (match.type === "SET") {
           entitiesToSet.push(match.entity);
         } else {
           idsToDelete.push(match.entityId);
         }
-        if (shouldSaveHistory && !idsWithDiff.has(entityId)) {
+        if (shouldSaveHistory && !idsWithDiff.has(entityKey)) {
           backfillHistoryIds.add(entityId);
           return;
         }
@@ -685,11 +757,11 @@ async function writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, co
           let promises = [];
           if (shouldSaveHistory) {
             if (backfillHistoryIds.size !== 0) {
-              await EntityHistory.backfillHistory(sql, pgSchema, entityConfig.name, entityConfig.index, Array.from(backfillHistoryIds));
+              await EntityHistory.backfillHistory(sql, pgSchema, entityConfig.table, entityConfig.index, Array.from(backfillHistoryIds));
             }
             if (Utils.$$Array.notEmpty(batchDeleteCheckpointIds)) {
               promises.push(sql.unsafe(makeInsertDeleteUpdatesQuery(entityConfig, pgSchema), [
-                batchDeleteEntityIds,
+                Table.encodeIdsToJson(entityConfig.table, batchDeleteEntityIds),
                 Utils.$$BigInt.arrayToStringArray(batchDeleteCheckpointIds)
               ], {prepare: true}));
             }
@@ -822,10 +894,12 @@ function makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema) {
     )`;
 }
 
-let rollbackRowStateSchema = S$RescriptSchema.object(s => [
-  s.f(Table.idFieldName, S$RescriptSchema.string),
+let rollbackRowStateSchema = Utils.$$WeakMap.memoize(table => S$RescriptSchema.object(s => [
+  s.f(Table.idFieldName, Table.getIdSchema(table)),
   s.f(EntityHistory.changeFieldName, EntityHistory.RowAction.schema)
-]);
+]));
+
+let rollbackRemovedIdsSchema = Utils.$$WeakMap.memoize(table => S$RescriptSchema.array(S$RescriptSchema.object(s => s.f(Table.idFieldName, Table.getIdSchema(table)))));
 
 function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isHasuraEnabled, sink, onInitialize, onNewTables) {
   let containerName = "envio-postgres";
@@ -843,6 +917,23 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     env: psqlExecOptions_env
   };
   let cacheDirPath = Path.resolve(".envio", "cache");
+  let storageName = "postgres";
+  let indexRegistry = IndexRegistry.make();
+  let reloadIndexRegistry = async () => {
+    let rows = S$RescriptSchema.parseOrThrow(await sql.unsafe(IndexRegistry.makeCatalogQuery(pgSchema)), IndexRegistry.catalogRowsSchema);
+    let invalidIndexNames = IndexRegistry.reload(indexRegistry, rows);
+    if (invalidIndexNames.length !== 0) {
+      Logging.warn({
+        storage: storageName,
+        msg: `Ignoring invalid PostgreSQL indexes in schema "` + pgSchema + `". They are left untouched — drop and recreate them manually if you need them.`,
+        indexes: invalidIndexNames
+      });
+    }
+    return Logging.info({
+      storage: storageName,
+      msg: `Found ` + IndexRegistry.size(indexRegistry).toString() + ` existing indexes in the indexer schema.`
+    });
+  };
   let isInitialized = async () => Utils.$$Array.notEmpty(await sql.unsafe(`SELECT table_schema FROM information_schema.tables WHERE table_schema = '` + pgSchema + `' AND (table_name = '` + "event_sync_state" + `' OR table_name = '` + InternalTable.Chains.table.tableName + `');`));
   let scanCacheDir = async () => {
     let topEntries;
@@ -962,7 +1053,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     if (sink !== undefined) {
       await sink.initialize(chainConfigs, chEntities, enums);
     }
-    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames));
+    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames), true);
     await sql.begin(async sql => {
       await Promise.all(queries.map(query => sql.unsafe(query)));
       return await InternalTable.EnvioInfo.write(sql, pgSchema, envioInfo);
@@ -988,6 +1079,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       ]);
     }
     let cache = await restoreEffectCache(true);
+    await reloadIndexRegistry();
     if (onInitialize !== undefined) {
       await onInitialize();
     }
@@ -1038,6 +1130,121 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       };
     }
   };
+  let filterColumns = (table, filters) => {
+    let queryFields = Table.queryFields(table);
+    let columns = [];
+    let seen = new Set();
+    let collect = filter => {
+      if (filter.operator === "and") {
+        filter.filters.forEach(collect);
+        return;
+      }
+      let match = queryFields[filter.fieldName];
+      if (match === undefined) {
+        return;
+      }
+      let pgDbFieldName = match.pgDbFieldName;
+      if (!seen.has(pgDbFieldName)) {
+        seen.add(pgDbFieldName);
+        columns.push(pgDbFieldName);
+        return;
+      }
+    };
+    filters.forEach(collect);
+    return columns;
+  };
+  let makeEnsureIndexQuery = schemaIndex => {
+    let indexName = IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex));
+    let match = IndexRegistry.getInvalid(indexRegistry, indexName);
+    if (match === undefined) {
+      return makeCreateSchemaIndexQuery(schemaIndex, pgSchema);
+    }
+    if (match.isRepairable) {
+      if (match.key === schemaIndexKey(schemaIndex)) {
+        return makeReindexQuery(pgSchema, indexName);
+      }
+      throw nameCollisionError(indexName, pgSchema);
+    }
+    throw nameCollisionError(indexName, pgSchema);
+  };
+  let isRepair = schemaIndex => Stdlib_Option.isSome(IndexRegistry.getInvalid(indexRegistry, IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex))));
+  let ensureQueryIndexes = async (table, filters) => {
+    let missing = Stdlib_Array.filterMap(filterColumns(table, filters), column => {
+      let schemaIndex = makeSingleColumnSchemaIndex(table.tableName, column);
+      if (IndexRegistry.has(indexRegistry, schemaIndexKey(schemaIndex))) {
+        return;
+      } else {
+        return schemaIndex;
+      }
+    });
+    if (Utils.$$Array.notEmpty(missing)) {
+      await Promise.all(missing.map(schemaIndex => {
+        let indexName = IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex));
+        return IndexRegistry.ensure(indexRegistry, schemaIndexKey(schemaIndex), table.tableName, async () => {
+          let query = makeEnsureIndexQuery(schemaIndex);
+          let verb = isRepair(schemaIndex) ? "Repairing invalid index" : "Creating index";
+          Logging.info({
+            storage: storageName,
+            msg: verb + ` "` + indexName + `" to serve a getWhere query on "` + table.tableName + `". Writes to the table are paused until it completes. ` + slowOnLargeDatabaseNotice
+          });
+          let timeRef = Performance.now();
+          await sql.unsafe(query);
+          IndexRegistry.clearInvalidName(indexRegistry, indexName);
+          return Logging.info({
+            storage: storageName,
+            msg: `Index "` + indexName + `" is ready after ` + formatSeconds(timeRef) + `s. Resuming indexing.`
+          });
+        }).catch(exn => Logging.warn({
+          storage: storageName,
+          msg: `Failed to create the index "` + indexName + `" for a getWhere query. The query runs without it.`,
+          err: Utils.prettifyExn(exn)
+        }));
+      }));
+      return;
+    }
+  };
+  let finalizeBackfill = async (entities, chainIds, readyAt) => {
+    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres));
+    IndexRegistry.validateIndexNamesOrThrow(schemaIndexes.map(schemaIndexDescription));
+    let missing = schemaIndexes.filter(schemaIndex => !IndexRegistry.has(indexRegistry, schemaIndexKey(schemaIndex)));
+    let repaired = missing.filter(isRepair);
+    if (Utils.$$Array.notEmpty(repaired)) {
+      Logging.warn({
+        storage: storageName,
+        msg: `PostgreSQL reports ` + repaired.length.toString() + ` existing indexes as invalid, so they can't serve queries. Rebuilding them.`,
+        indexes: repaired.map(schemaIndexName)
+      });
+    }
+    if (missing.length !== 0) {
+      Logging.info({
+        storage: storageName,
+        msg: `Creating the ` + missing.length.toString() + ` remaining schema indexes before the indexer reports ready. Writes are paused until they are committed. ` + slowOnLargeDatabaseNotice,
+        indexes: missing.map(schemaIndexName)
+      });
+    } else {
+      Logging.info({
+        storage: storageName,
+        msg: `All ` + schemaIndexes.length.toString() + ` schema indexes are already in place. Marking the indexer ready.`
+      });
+    }
+    let indexQueries = missing.map(makeEnsureIndexQuery);
+    let timeRef = Performance.now();
+    await sql.begin(async sql => {
+      for (let idx = 0, idx_finish = indexQueries.length; idx < idx_finish; ++idx) {
+        await sql.unsafe(indexQueries[idx]);
+      }
+      return await sql.unsafe(InternalTable.Chains.makeSetReadyAtQuery(pgSchema), [
+        readyAt,
+        chainIds
+      ], {prepare: true});
+    });
+    missing.forEach(schemaIndex => IndexRegistry.add(indexRegistry, schemaIndexKey(schemaIndex)));
+    repaired.forEach(schemaIndex => IndexRegistry.clearInvalidName(indexRegistry, IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex))));
+    return Logging.info({
+      storage: storageName,
+      msg: `Committed ` + missing.length.toString() + ` schema indexes and the ready timestamp in ` + formatSeconds(timeRef) + `s.`
+    });
+  };
   let setOrThrow$1 = (items, table, itemSchema) => setOrThrow(sql, items, table, itemSchema, pgSchema);
   let setEffectCacheOrThrow = async (table, itemSchema, items, initialize) => {
     if (initialize) {
@@ -1114,6 +1321,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema)),
       InternalTable.EnvioInfo.read(sql, pgSchema)
     ]);
+    await reloadIndexRegistry();
     let checkpointId = BigInt(match[2][0].id);
     let reorgCheckpoints = match[3].map(raw => ({
       id: BigInt(raw.id),
@@ -1147,10 +1355,10 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       sql.unsafe(makeGetRollbackRemovedIdsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true}),
       sql.unsafe(makeGetRollbackPreTargetRowsQuery(entityConfig, pgSchema), [rollbackTargetCheckpointId.toString()], {prepare: true})
     ]);
-    let removedIds = match[0].map(row => row.id);
+    let removedIds = S$RescriptSchema.parseOrThrow(match[0], rollbackRemovedIdsSchema(entityConfig.table));
     let restoredEntitiesResult = [];
     match[1].forEach(row => {
-      let match = S$RescriptSchema.parseOrThrow(row, rollbackRowStateSchema);
+      let match = S$RescriptSchema.parseOrThrow(row, rollbackRowStateSchema(entityConfig.table));
       if (match[1] === "SET") {
         restoredEntitiesResult.push(row);
         return;
@@ -1186,15 +1394,17 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
     }
     let primaryTimerRef = Performance.now();
     await writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, pgUpdates, sinkPromise, chainMetaData, undefined);
-    return onWrite("postgres", Performance.secondsSince(primaryTimerRef));
+    return onWrite(storageName, Performance.secondsSince(primaryTimerRef));
   };
   let close = () => sql.end();
   return {
-    name: "postgres",
+    name: storageName,
     isInitialized: isInitialized,
     initialize: initialize,
     resumeInitialState: resumeInitialState,
     loadOrThrow: loadOrThrow,
+    ensureQueryIndexes: ensureQueryIndexes,
+    finalizeBackfill: finalizeBackfill,
     dumpEffectCache: dumpEffectCache,
     reset: reset,
     setChainMeta: setChainMeta,
@@ -1264,7 +1474,17 @@ export {
   directionToSql,
   directionToIndexName,
   makeCreateCompositeIndexQuery,
-  makeCreateTableIndicesQuery,
+  makeCreateTableIndexesQuery,
+  schemaIndexDescription,
+  schemaIndexName,
+  schemaIndexKey,
+  makeCreateSchemaIndexQuery,
+  formatSeconds,
+  slowOnLargeDatabaseNotice,
+  makeReindexQuery,
+  nameCollisionError,
+  makeSingleColumnSchemaIndex,
+  getSchemaIndexes,
   makeCreateTableQuery,
   entityHistoryCache,
   getEntityHistory,
@@ -1297,6 +1517,7 @@ export {
   makeGetRollbackPreTargetRowsQuery,
   makeGetRollbackRemovedIdsQuery,
   rollbackRowStateSchema,
+  rollbackRemovedIdsSchema,
   make,
   makeStorageFromEnv,
   makePersistenceFromConfig,

package/src/SimulateItems.res

@@ -343,8 +343,8 @@ let parse = (
       // run here the way real routing does (one item per registration).
       // Registrations are built the same way `HandlerRegister.finishRegistration`
       // does at startup (not stubs), so the address filter and `where` behave
-      // identically to real indexing — the dead-input tracker relies on
-      // `clientAddressFilter` actually gating unrouted items. Drop registrations
+      // identically to real indexing — the dead-input tracker relies on the
+      // source's address gate actually dropping unrouted items. Drop registrations
       // whose `where` excludes this chain (they wouldn't be fetched live), and
       // ignore the bare handler-less fallback so a missing handler surfaces below
       // instead of silently running nothing.

package/src/TestIndexer.res

@@ -22,7 +22,7 @@ type t<'processConfig> = {process: 'processConfig => promise<processResult>}
 
 type entityChange = {
   sets: array<unknown>,
-  deleted: array<string>,
+  deleted: array<EntityId.t>,
 }
 
 type testIndexerState = {
@@ -136,11 +136,12 @@ let handleWriteBatch = (
       switch change {
       | Set({entityId, entity, checkpointId}) =>
         // The store keeps decoded entities so load comparisons (bigint /
-        // BigDecimal) work on real values.
-        entityDict->Dict.set(entityId, entity)
+        // BigDecimal) work on real values. Ids are keyed by their string form
+        // since they may be string/int/bigint.
+        entityDict->Dict.set(entityId->EntityId.toKey, entity)
         entityChangeFor(checkpointId).sets->Array.push(entity->Utils.magic)->ignore
       | Delete({entityId, checkpointId}) =>
-        Dict.delete(entityDict->Obj.magic, entityId)
+        Dict.delete(entityDict->Obj.magic, entityId->EntityId.toKey)
         entityChangeFor(checkpointId).deleted->Array.push(entityId)->ignore
       }
     }
@@ -197,7 +198,7 @@ let handleWriteBatch = (
             entityObj->Dict.set("sets", sets->(Utils.magic: array<unknown> => unknown))
           }
           if deleted->Array.length > 0 {
-            entityObj->Dict.set("deleted", deleted->(Utils.magic: array<string> => unknown))
+            entityObj->Dict.set("deleted", deleted->(Utils.magic: array<EntityId.t> => unknown))
           }
           // Match the capitalized entity accessor the generated change types expose.
           change->Dict.set(
@@ -484,6 +485,9 @@ let makeInMemoryStorage = (~state: testIndexerState): Persistence.storage => {
     state
     ->handleLoad(~tableName=table.tableName, ~filter)
     ->(Utils.magic: array<Internal.entity> => array<unknown>),
+  // The in-memory storage has no indexes to build, and it's always ready.
+  ensureQueryIndexes: async (~table as _, ~filters as _) => (),
+  finalizeBackfill: async (~entities as _, ~chainIds as _, ~readyAt as _) => (),
   writeBatch: async (
     ~batch,
     ~rollback as _,

package/src/TestIndexer.res.mjs

@@ -5,6 +5,7 @@ import * as Utils from "./Utils.res.mjs";
 import * as Config from "./Config.res.mjs";
 import * as Logging from "./Logging.res.mjs";
 import * as ChainMap from "./ChainMap.res.mjs";
+import * as EntityId from "./EntityId.res.mjs";
 import * as Stdlib_Int from "@rescript/runtime/lib/es6/Stdlib_Int.js";
 import * as IndexerLoop from "./IndexerLoop.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
@@ -102,12 +103,12 @@ function handleWriteBatch(state, updatedEntities, checkpointIds, checkpointChain
     let processChange = change => {
       if (change.type === "SET") {
         let entity = change.entity;
-        entityDict[change.entityId] = entity;
+        entityDict[EntityId.toKey(change.entityId)] = entity;
         entityChangeFor(change.checkpointId).sets.push(entity);
         return;
       }
       let entityId = change.entityId;
-      Stdlib_Dict.$$delete(entityDict, entityId);
+      Stdlib_Dict.$$delete(entityDict, EntityId.toKey(entityId));
       entityChangeFor(change.checkpointId).deleted.push(entityId);
     };
     param.changes.forEach(processChange);
@@ -314,6 +315,8 @@ function makeInMemoryStorage(state) {
     initialize: async (param, param$1, param$2, param$3) => Stdlib_JsError.throwWithMessage("TestIndexer: initialize should not be called; the initial state is derived from config."),
     resumeInitialState: async () => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
     loadOrThrow: async (filter, table) => handleLoad(state, table.tableName, filter),
+    ensureQueryIndexes: async (param, param$1) => {},
+    finalizeBackfill: async (param, param$1, param$2) => {},
     dumpEffectCache: async () => {},
     reset: async () => {},
     setChainMeta: async param => {},

package/src/UserContext.res

@@ -144,7 +144,7 @@ let getWhereHandler = (params: entityContextParams, filter: dict<dict<unknown>>)
 }
 
 let noopSet = (_entity: Internal.entity) => ()
-let noopDeleteUnsafe = (_entityId: string) => ()
+let noopDeleteUnsafe = (_entityId: EntityId.t) => ()
 
 // Reads against ClickHouse-only entities have no Postgres table to hit;
 // surface a friendly error instead of letting the SQL layer fail with
@@ -168,7 +168,7 @@ let entityTraps: Utils.Proxy.traps<entityContextParams> = {
           ->InMemoryTable.Entity.set(
             ~committedCheckpointId=params.indexerState->IndexerState.committedCheckpointId,
             Set({
-              entityId: entity.id,
+              entityId: entity.id->EntityId.unsafeOfString,
               checkpointId: params.checkpointId,
               entity,
             }),
@@ -282,7 +282,7 @@ let entityTraps: Utils.Proxy.traps<entityContextParams> = {
             }),
           )
         }
-      }->(Utils.magic: (string => unit) => unknown)
+      }->(Utils.magic: (EntityId.t => unit) => unknown)
     | _ =>
       JsError.throwWithMessage(`Invalid context.${params.entityConfig.name}.${prop} operation.`)
     }

package/src/Writing.res

@@ -237,7 +237,7 @@ let dropCommitted = (state: IndexerState.t, ~keepLoadedFromDb) => {
 
 // Blocks until the store holds fewer than keepLatestChangesLimit changes,
 // freeing committed changes first and awaiting commits as a last resort.
-let rec awaitCapacity = async (state: IndexerState.t) => {
+let rec awaitCapacityLoop = async (state: IndexerState.t) => {
   // After a failed write nothing will free capacity, so bail instead of waiting
   // on a commit that won't come (the error already went to onError).
   if !(state->IndexerState.hasFailedWrite) && state->getChangesCount >= keepLatestChangesLimit {
@@ -259,11 +259,21 @@ let rec awaitCapacity = async (state: IndexerState.t) => {
     ) {
       state->schedule
       await state->waitForCommit
-      await state->awaitCapacity
+      await state->awaitCapacityLoop
     }
   }
 }
 
+// Only the over-limit path is a real stall. Timing every call would charge each
+// batch the microtask hop of awaiting an already-resolved promise, which reads
+// as storage backpressure that isn't there.
+let awaitCapacity = async (state: IndexerState.t) =>
+  if state->getChangesCount >= keepLatestChangesLimit {
+    let timeRef = Performance.now()
+    await state->awaitCapacityLoop
+    state->IndexerState.recordStalledOnStorageWrite(~seconds=timeRef->Performance.secondsSince)
+  }
+
 // Awaits until everything processed is persisted. On a failed write we stop
 // draining (onError already surfaced it) rather than throw.
 let rec flush = async (state: IndexerState.t) => {