envio 3.5.0-alpha.2 → 3.5.0-alpha.3

package/src/PgStorage.res.mjs

@@ -18,6 +18,7 @@ import * as Performance from "./bindings/Performance.res.mjs";
 import * as Persistence from "./Persistence.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as EntityHistory from "./db/EntityHistory.res.mjs";
+import * as IndexRegistry from "./db/IndexRegistry.res.mjs";
 import * as InternalTable from "./db/InternalTable.res.mjs";
 import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
 import * as Child_process from "child_process";
@@ -72,13 +73,87 @@ function makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema) {
   return `CREATE INDEX IF NOT EXISTS "` + indexName + `" ON "` + pgSchema + `"."` + tableName + `"(` + index + `);`;
 }
 
-function makeCreateTableIndicesQuery(table, pgSchema) {
+function makeCreateTableIndexesQuery(table, pgSchema) {
   let tableName = table.tableName;
   let createIndex = indexField => makeCreateIndexQuery(tableName, [indexField], pgSchema);
   let createCompositeIndex = indexFields => makeCreateCompositeIndexQuery(tableName, indexFields, pgSchema);
-  let singleIndices = Table.getSingleIndices(table);
-  let compositeIndices = Table.getCompositeIndices(table);
-  return singleIndices.map(createIndex).join("\n") + compositeIndices.map(createCompositeIndex).join("\n");
+  let singleIndexes = Table.getSingleIndexes(table);
+  let compositeIndexes = Table.getCompositeIndexes(table);
+  return singleIndexes.map(createIndex).join("\n") + compositeIndexes.map(createCompositeIndex).join("\n");
+}
+
+function schemaIndexDescription(param) {
+  return param.tableName + "_" + param.indexFields.map(f => f.fieldName + directionToIndexName(f.direction)).join("_");
+}
+
+function schemaIndexName(schemaIndex) {
+  return IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex));
+}
+
+function schemaIndexKey(param) {
+  return IndexRegistry.makeKey(param.tableName, IndexRegistry.fromIndexFields(param.indexFields), IndexRegistry.btree);
+}
+
+function makeCreateSchemaIndexQuery(schemaIndex, pgSchema) {
+  let columns = schemaIndex.indexFields.map(f => `"` + f.fieldName + `"` + directionToSql(f.direction)).join(", ");
+  return `CREATE INDEX IF NOT EXISTS "` + IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex)) + `" ON "` + pgSchema + `"."` + schemaIndex.tableName + `"(` + columns + `);`;
+}
+
+function formatSeconds(timeRef) {
+  return (Math.round(Performance.secondsSince(timeRef) * 100) / 100).toString();
+}
+
+let slowOnLargeDatabaseNotice = "This can take a long time on a large database.";
+
+function makeReindexQuery(pgSchema, indexName) {
+  return `REINDEX INDEX "` + pgSchema + `"."` + indexName + `";`;
+}
+
+function nameCollisionError(indexName, pgSchema) {
+  return new Error(`Cannot create the index "` + indexName + `" in schema "` + pgSchema + `". An index of that name already exists and PostgreSQL reports it as invalid, but it isn't one the indexer created — it covers different columns, or it is a unique or partial index — so it is left untouched. Drop or repair it by hand, then restart the indexer.`);
+}
+
+function makeSingleColumnSchemaIndex(tableName, column) {
+  return {
+    tableName: tableName,
+    indexFields: [{
+        fieldName: column,
+        direction: "Asc"
+      }]
+  };
+}
+
+function getSchemaIndexes(entities) {
+  let derivedSchema = Schema.make(entities.map(e => e.table));
+  let all = [];
+  entities.forEach(param => {
+    let table = param.table;
+    Table.getSingleIndexes(table).forEach(column => {
+      all.push(makeSingleColumnSchemaIndex(table.tableName, column));
+    });
+    Table.getCompositeIndexes(table).forEach(indexFields => {
+      all.push({
+        tableName: table.tableName,
+        indexFields: indexFields
+      });
+    });
+  });
+  entities.forEach(param => {
+    Table.getDerivedFromFields(param.table).forEach(derivedFromField => {
+      let column = Utils.unwrapResultExn(Schema.getDerivedFromPgFieldName(derivedSchema, derivedFromField));
+      all.push(makeSingleColumnSchemaIndex(derivedFromField.derivedFromEntity, column));
+    });
+  });
+  let seen = new Set();
+  return all.filter(schemaIndex => {
+    let key = schemaIndexKey(schemaIndex);
+    if (seen.has(key)) {
+      return false;
+    } else {
+      seen.add(key);
+      return true;
+    }
+  });
 }
 
 function makeCreateTableQuery(table, pgSchema, isNumericArrayAsText) {
@@ -169,11 +244,12 @@ function getEntityHistory(entityConfig) {
   return cache$1;
 }
 
-function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt) {
+function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigsOpt, entitiesOpt, enumsOpt, isEmptyPgSchemaOpt, deferSchemaIndexesOpt) {
   let chainConfigs = chainConfigsOpt !== undefined ? chainConfigsOpt : [];
   let entities = entitiesOpt !== undefined ? entitiesOpt : [];
   let enums = enumsOpt !== undefined ? enumsOpt : [];
   let isEmptyPgSchema = isEmptyPgSchemaOpt !== undefined ? isEmptyPgSchemaOpt : false;
+  let deferSchemaIndexes = deferSchemaIndexesOpt !== undefined ? deferSchemaIndexesOpt : false;
   let generalTables = [
     InternalTable.Chains.table,
     InternalTable.EnvioInfo.table,
@@ -181,13 +257,12 @@ function makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfi
     InternalTable.RawEvents.table
   ];
   let allTables = generalTables.slice();
-  let allEntityTables = [];
   entities.forEach(entityConfig => {
-    allEntityTables.push(entityConfig.table);
     allTables.push(entityConfig.table);
     allTables.push(getEntityHistory(entityConfig).table);
   });
-  let derivedSchema = Schema.make(allEntityTables);
+  let schemaIndexes = getSchemaIndexes(entities);
+  IndexRegistry.validateIndexNamesOrThrow(schemaIndexes.map(schemaIndexDescription));
   let query = {
     contents: (
       isEmptyPgSchema && pgSchema === "public" ? `CREATE SCHEMA IF NOT EXISTS "` + pgSchema + `";\n` : `DROP SCHEMA IF EXISTS "` + pgSchema + `" CASCADE;
@@ -202,19 +277,11 @@ GRANT ALL ON SCHEMA "` + pgSchema + `" TO public;`)
   allTables.forEach(table => {
     query.contents = query.contents + "\n" + makeCreateTableQuery(table, pgSchema, isHasuraEnabled);
   });
-  allTables.forEach(table => {
-    let indices = makeCreateTableIndicesQuery(table, pgSchema);
-    if (indices !== "") {
-      query.contents = query.contents + "\n" + indices;
-      return;
-    }
-  });
-  entities.forEach(entity => {
-    Table.getDerivedFromFields(entity.table).forEach(derivedFromField => {
-      let indexField = Utils.unwrapResultExn(Schema.getDerivedFromPgFieldName(derivedSchema, derivedFromField));
-      query.contents = query.contents + "\n" + makeCreateIndexQuery(derivedFromField.derivedFromEntity, [indexField], pgSchema);
+  if (!deferSchemaIndexes) {
+    schemaIndexes.forEach(schemaIndex => {
+      query.contents = query.contents + "\n" + makeCreateSchemaIndexQuery(schemaIndex, pgSchema);
     });
-  });
+  }
   query.contents = query.contents + "\n" + InternalTable.Views.makeMetaViewQuery(pgSchema);
   query.contents = query.contents + "\n" + InternalTable.Views.makeChainMetadataViewQuery(pgSchema);
   let initialChainsValuesQuery = InternalTable.Chains.makeInitialValuesQuery(pgSchema, chainConfigs);
@@ -850,6 +917,23 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     env: psqlExecOptions_env
   };
   let cacheDirPath = Path.resolve(".envio", "cache");
+  let storageName = "postgres";
+  let indexRegistry = IndexRegistry.make();
+  let reloadIndexRegistry = async () => {
+    let rows = S$RescriptSchema.parseOrThrow(await sql.unsafe(IndexRegistry.makeCatalogQuery(pgSchema)), IndexRegistry.catalogRowsSchema);
+    let invalidIndexNames = IndexRegistry.reload(indexRegistry, rows);
+    if (invalidIndexNames.length !== 0) {
+      Logging.warn({
+        storage: storageName,
+        msg: `Ignoring invalid PostgreSQL indexes in schema "` + pgSchema + `". They are left untouched — drop and recreate them manually if you need them.`,
+        indexes: invalidIndexNames
+      });
+    }
+    return Logging.info({
+      storage: storageName,
+      msg: `Found ` + IndexRegistry.size(indexRegistry).toString() + ` existing indexes in the indexer schema.`
+    });
+  };
   let isInitialized = async () => Utils.$$Array.notEmpty(await sql.unsafe(`SELECT table_schema FROM information_schema.tables WHERE table_schema = '` + pgSchema + `' AND (table_name = '` + "event_sync_state" + `' OR table_name = '` + InternalTable.Chains.table.tableName + `');`));
   let scanCacheDir = async () => {
     let topEntries;
@@ -969,7 +1053,7 @@ function make(sql, pgHost, pgSchema, pgPort, pgUser, pgDatabase, pgPassword, isH
     if (sink !== undefined) {
       await sink.initialize(chainConfigs, chEntities, enums);
     }
-    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames));
+    let queries = makeInitializeTransaction(pgSchema, pgUser, isHasuraEnabled, chainConfigs, pgEntities, enums, Utils.$$Array.isEmpty(schemaTableNames), true);
     await sql.begin(async sql => {
       await Promise.all(queries.map(query => sql.unsafe(query)));
       return await InternalTable.EnvioInfo.write(sql, pgSchema, envioInfo);
@@ -995,6 +1079,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       ]);
     }
     let cache = await restoreEffectCache(true);
+    await reloadIndexRegistry();
     if (onInitialize !== undefined) {
       await onInitialize();
     }
@@ -1045,6 +1130,121 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       };
     }
   };
+  let filterColumns = (table, filters) => {
+    let queryFields = Table.queryFields(table);
+    let columns = [];
+    let seen = new Set();
+    let collect = filter => {
+      if (filter.operator === "and") {
+        filter.filters.forEach(collect);
+        return;
+      }
+      let match = queryFields[filter.fieldName];
+      if (match === undefined) {
+        return;
+      }
+      let pgDbFieldName = match.pgDbFieldName;
+      if (!seen.has(pgDbFieldName)) {
+        seen.add(pgDbFieldName);
+        columns.push(pgDbFieldName);
+        return;
+      }
+    };
+    filters.forEach(collect);
+    return columns;
+  };
+  let makeEnsureIndexQuery = schemaIndex => {
+    let indexName = IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex));
+    let match = IndexRegistry.getInvalid(indexRegistry, indexName);
+    if (match === undefined) {
+      return makeCreateSchemaIndexQuery(schemaIndex, pgSchema);
+    }
+    if (match.isRepairable) {
+      if (match.key === schemaIndexKey(schemaIndex)) {
+        return makeReindexQuery(pgSchema, indexName);
+      }
+      throw nameCollisionError(indexName, pgSchema);
+    }
+    throw nameCollisionError(indexName, pgSchema);
+  };
+  let isRepair = schemaIndex => Stdlib_Option.isSome(IndexRegistry.getInvalid(indexRegistry, IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex))));
+  let ensureQueryIndexes = async (table, filters) => {
+    let missing = Stdlib_Array.filterMap(filterColumns(table, filters), column => {
+      let schemaIndex = makeSingleColumnSchemaIndex(table.tableName, column);
+      if (IndexRegistry.has(indexRegistry, schemaIndexKey(schemaIndex))) {
+        return;
+      } else {
+        return schemaIndex;
+      }
+    });
+    if (Utils.$$Array.notEmpty(missing)) {
+      await Promise.all(missing.map(schemaIndex => {
+        let indexName = IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex));
+        return IndexRegistry.ensure(indexRegistry, schemaIndexKey(schemaIndex), table.tableName, async () => {
+          let query = makeEnsureIndexQuery(schemaIndex);
+          let verb = isRepair(schemaIndex) ? "Repairing invalid index" : "Creating index";
+          Logging.info({
+            storage: storageName,
+            msg: verb + ` "` + indexName + `" to serve a getWhere query on "` + table.tableName + `". Writes to the table are paused until it completes. ` + slowOnLargeDatabaseNotice
+          });
+          let timeRef = Performance.now();
+          await sql.unsafe(query);
+          IndexRegistry.clearInvalidName(indexRegistry, indexName);
+          return Logging.info({
+            storage: storageName,
+            msg: `Index "` + indexName + `" is ready after ` + formatSeconds(timeRef) + `s. Resuming indexing.`
+          });
+        }).catch(exn => Logging.warn({
+          storage: storageName,
+          msg: `Failed to create the index "` + indexName + `" for a getWhere query. The query runs without it.`,
+          err: Utils.prettifyExn(exn)
+        }));
+      }));
+      return;
+    }
+  };
+  let finalizeBackfill = async (entities, chainIds, readyAt) => {
+    let schemaIndexes = getSchemaIndexes(entities.filter(e => e.storage.postgres));
+    IndexRegistry.validateIndexNamesOrThrow(schemaIndexes.map(schemaIndexDescription));
+    let missing = schemaIndexes.filter(schemaIndex => !IndexRegistry.has(indexRegistry, schemaIndexKey(schemaIndex)));
+    let repaired = missing.filter(isRepair);
+    if (Utils.$$Array.notEmpty(repaired)) {
+      Logging.warn({
+        storage: storageName,
+        msg: `PostgreSQL reports ` + repaired.length.toString() + ` existing indexes as invalid, so they can't serve queries. Rebuilding them.`,
+        indexes: repaired.map(schemaIndexName)
+      });
+    }
+    if (missing.length !== 0) {
+      Logging.info({
+        storage: storageName,
+        msg: `Creating the ` + missing.length.toString() + ` remaining schema indexes before the indexer reports ready. Writes are paused until they are committed. ` + slowOnLargeDatabaseNotice,
+        indexes: missing.map(schemaIndexName)
+      });
+    } else {
+      Logging.info({
+        storage: storageName,
+        msg: `All ` + schemaIndexes.length.toString() + ` schema indexes are already in place. Marking the indexer ready.`
+      });
+    }
+    let indexQueries = missing.map(makeEnsureIndexQuery);
+    let timeRef = Performance.now();
+    await sql.begin(async sql => {
+      for (let idx = 0, idx_finish = indexQueries.length; idx < idx_finish; ++idx) {
+        await sql.unsafe(indexQueries[idx]);
+      }
+      return await sql.unsafe(InternalTable.Chains.makeSetReadyAtQuery(pgSchema), [
+        readyAt,
+        chainIds
+      ], {prepare: true});
+    });
+    missing.forEach(schemaIndex => IndexRegistry.add(indexRegistry, schemaIndexKey(schemaIndex)));
+    repaired.forEach(schemaIndex => IndexRegistry.clearInvalidName(indexRegistry, IndexRegistry.truncateIndexName(schemaIndexDescription(schemaIndex))));
+    return Logging.info({
+      storage: storageName,
+      msg: `Committed ` + missing.length.toString() + ` schema indexes and the ready timestamp in ` + formatSeconds(timeRef) + `s.`
+    });
+  };
   let setOrThrow$1 = (items, table, itemSchema) => setOrThrow(sql, items, table, itemSchema, pgSchema);
   let setEffectCacheOrThrow = async (table, itemSchema, items, initialize) => {
     if (initialize) {
@@ -1121,6 +1321,7 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       sql.unsafe(InternalTable.Checkpoints.makeGetReorgCheckpointsQuery(pgSchema)),
       InternalTable.EnvioInfo.read(sql, pgSchema)
     ]);
+    await reloadIndexRegistry();
     let checkpointId = BigInt(match[2][0].id);
     let reorgCheckpoints = match[3].map(raw => ({
       id: BigInt(raw.id),
@@ -1193,15 +1394,17 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
     }
     let primaryTimerRef = Performance.now();
     await writeBatch(sql, batch, pgSchema, rollback, isInReorgThreshold, config, allEntities, setEffectCacheOrThrow, updatedEffectsCache, pgUpdates, sinkPromise, chainMetaData, undefined);
-    return onWrite("postgres", Performance.secondsSince(primaryTimerRef));
+    return onWrite(storageName, Performance.secondsSince(primaryTimerRef));
   };
   let close = () => sql.end();
   return {
-    name: "postgres",
+    name: storageName,
     isInitialized: isInitialized,
     initialize: initialize,
     resumeInitialState: resumeInitialState,
     loadOrThrow: loadOrThrow,
+    ensureQueryIndexes: ensureQueryIndexes,
+    finalizeBackfill: finalizeBackfill,
     dumpEffectCache: dumpEffectCache,
     reset: reset,
     setChainMeta: setChainMeta,
@@ -1271,7 +1474,17 @@ export {
   directionToSql,
   directionToIndexName,
   makeCreateCompositeIndexQuery,
-  makeCreateTableIndicesQuery,
+  makeCreateTableIndexesQuery,
+  schemaIndexDescription,
+  schemaIndexName,
+  schemaIndexKey,
+  makeCreateSchemaIndexQuery,
+  formatSeconds,
+  slowOnLargeDatabaseNotice,
+  makeReindexQuery,
+  nameCollisionError,
+  makeSingleColumnSchemaIndex,
+  getSchemaIndexes,
   makeCreateTableQuery,
   entityHistoryCache,
   getEntityHistory,

package/src/TestIndexer.res

@@ -485,6 +485,9 @@ let makeInMemoryStorage = (~state: testIndexerState): Persistence.storage => {
     state
     ->handleLoad(~tableName=table.tableName, ~filter)
     ->(Utils.magic: array<Internal.entity> => array<unknown>),
+  // The in-memory storage has no indexes to build, and it's always ready.
+  ensureQueryIndexes: async (~table as _, ~filters as _) => (),
+  finalizeBackfill: async (~entities as _, ~chainIds as _, ~readyAt as _) => (),
   writeBatch: async (
     ~batch,
     ~rollback as _,

package/src/TestIndexer.res.mjs

@@ -315,6 +315,8 @@ function makeInMemoryStorage(state) {
     initialize: async (param, param$1, param$2, param$3) => Stdlib_JsError.throwWithMessage("TestIndexer: initialize should not be called; the initial state is derived from config."),
     resumeInitialState: async () => Stdlib_JsError.throwWithMessage("TestIndexer: resumeInitialState should not be called; the initial state is derived from config."),
     loadOrThrow: async (filter, table) => handleLoad(state, table.tableName, filter),
+    ensureQueryIndexes: async (param, param$1) => {},
+    finalizeBackfill: async (param, param$1, param$2) => {},
     dumpEffectCache: async () => {},
     reset: async () => {},
     setChainMeta: async param => {},

package/src/db/EntityFilter.res

@@ -176,10 +176,6 @@ let parseGetWhereOrThrow = (filter: dict<dict<unknown>>, ~entityName, ~table: Ta
       JsError.throwWithMessage(
         `The field "${apiFieldName}" on entity "${entityName}" is a derived field and cannot be used in getWhere(). Use the source entity's indexed field instead.`,
       )
-    | Some(Field({isPrimaryKey: false, isIndex: false, linkedEntity: None})) =>
-      JsError.throwWithMessage(
-        `The field "${apiFieldName}" on entity "${entityName}" does not have an index. To use it in getWhere(), add the @index directive in your schema.graphql:\n\n  ${apiFieldName}: ... @index\n\nThen run 'pnpm envio codegen' to regenerate.`,
-      )
     | Some(Field(_)) => ()
     }
 

package/src/db/EntityFilter.res.mjs

@@ -142,14 +142,7 @@ function parseGetWhereOrThrow(filter, entityName, table) {
     });
     let match = Table.getFieldByApiName(table, apiFieldName);
     if (match !== undefined) {
-      if (match.TAG === "Field") {
-        let match$1 = match._0;
-        if (match$1.isPrimaryKey || match$1.isIndex || match$1.linkedEntity !== undefined) {
-          
-        } else {
-          Stdlib_JsError.throwWithMessage(`The field "` + apiFieldName + `" on entity "` + entityName + `" does not have an index. To use it in getWhere(), add the @index directive in your schema.graphql:\n\n  ` + apiFieldName + `: ... @index\n\nThen run 'pnpm envio codegen' to regenerate.`);
-        }
-      } else {
+      if (match.TAG !== "Field") {
         Stdlib_JsError.throwWithMessage(`The field "` + apiFieldName + `" on entity "` + entityName + `" is a derived field and cannot be used in getWhere(). Use the source entity's indexed field instead.`);
       }
     } else {

package/src/db/IndexRegistry.res

@@ -0,0 +1,219 @@
+// An in-memory mirror of the indexes PostgreSQL holds for the indexer schema.
+// Postgres stays authoritative: the registry is loaded from pg_catalog at
+// startup/resume and afterwards only grows — optimistically, right after a
+// CREATE INDEX we issued succeeds. Only one Indexer instance writes to the
+// schema, so no catalog refresh is needed between those two points.
+
+let btree = "btree"
+
+type column = {
+  name: string,
+  direction: Table.indexFieldDirection,
+}
+
+let fromIndexFields = (indexFields: array<Table.compositeIndexField>) =>
+  indexFields->Array.map(({fieldName, direction}) => {name: fieldName, direction})
+
+// Identity of an index is table + ordered columns + direction + method. Names
+// are deliberately not part of it: the catalog is matched on what the index
+// actually covers, not on how it was spelled.
+let makeKey = (~tableName, ~columns: array<column>, ~method) =>
+  `${tableName}|${method}|${columns
+    ->Array.map(({name, direction}) =>
+      switch direction {
+      | Table.Asc => name
+      | Desc => name ++ " DESC"
+      }
+    )
+    ->Array.joinUnsafe(",")}`
+
+type catalogRow = {
+  tableName: string,
+  indexName: string,
+  method: string,
+  // 1/0 rather than a bool: postgres.js hands booleans back inconsistently
+  // depending on the driver's type resolution, so the query casts it.
+  isValid: int,
+  // 1 only for an index the indexer could have created: plain btree over plain
+  // columns. A unique or partial index carries a definition our key doesn't
+  // capture, so it can never be one of ours to rebuild.
+  isPlain: int,
+  columns: array<string>,
+  // "ASC"/"DESC" per column, in the same order as `columns`.
+  directions: array<string>,
+}
+
+let catalogRowsSchema = S.array(
+  S.object((s): catalogRow => {
+    tableName: s.field("tableName", S.string),
+    indexName: s.field("indexName", S.string),
+    method: s.field("method", S.string),
+    isValid: s.field("isValid", S.int),
+    isPlain: s.field("isPlain", S.int),
+    columns: s.field("columns", S.array(S.string)),
+    directions: s.field("directions", S.array(S.string)),
+  }),
+)
+
+// One row per index, with its key columns aggregated in ordinal order.
+// INCLUDE columns are excluded (they don't affect what the index can serve),
+// and expression columns come back as their printed definition so an
+// expression index can never be mistaken for a plain-column one.
+let makeCatalogQuery = (~pgSchema) =>
+  `SELECT
+  t.relname AS "tableName",
+  i.relname AS "indexName",
+  am.amname AS "method",
+  CASE WHEN ix.indisvalid AND ix.indisready THEN 1 ELSE 0 END AS "isValid",
+  CASE
+    WHEN ix.indisunique OR ix.indpred IS NOT NULL OR ix.indexprs IS NOT NULL THEN 0
+    ELSE 1
+  END AS "isPlain",
+  array_agg(
+    CASE WHEN k.attnum = 0
+      THEN pg_get_indexdef(ix.indexrelid, k.ord::int, true)
+      ELSE a.attname
+    END ORDER BY k.ord
+  ) AS "columns",
+  array_agg(
+    CASE WHEN (ix.indoption[k.ord - 1] & 1) = 1 THEN 'DESC' ELSE 'ASC' END ORDER BY k.ord
+  ) AS "directions"
+FROM pg_index ix
+JOIN pg_class i ON i.oid = ix.indexrelid
+JOIN pg_class t ON t.oid = ix.indrelid
+JOIN pg_namespace n ON n.oid = t.relnamespace
+JOIN pg_am am ON am.oid = i.relam
+JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) ON k.ord <= ix.indnkeyatts
+LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum
+WHERE n.nspname = '${pgSchema}'
+GROUP BY t.relname, i.relname, am.amname, ix.indisvalid, ix.indisready,
+         ix.indisunique, ix.indpred, ix.indexprs;`
+
+// What we know about an index that came back invalid, enough to decide whether
+// rebuilding it would produce the index we actually wanted.
+type invalidIndex = {
+  key: string,
+  isRepairable: bool,
+}
+
+type t = {
+  keys: Utils.Set.t<string>,
+  // Every index Postgres reports as invalid, by name. Kept because
+  // `CREATE INDEX IF NOT EXISTS` matches on name: with an invalid index already
+  // under the name we want, the DDL is a silent no-op and we'd register a key
+  // for something the planner refuses to use.
+  invalidByName: dict<invalidIndex>,
+  // Keyed by index key so identical concurrent requests await one build.
+  inflight: dict<promise<unit>>,
+  // Tail of the build chain per table, so two different indexes on the same
+  // table are built one after the other rather than at once.
+  tableQueue: dict<promise<unit>>,
+}
+
+let make = () => {
+  keys: Utils.Set.make(),
+  invalidByName: Dict.make(),
+  inflight: Dict.make(),
+  tableQueue: Dict.make(),
+}
+
+let has = (registry, key) => registry.keys->Utils.Set.has(key)
+
+let add = (registry, key) => registry.keys->Utils.Set.add(key)->ignore
+
+let size = registry => registry.keys->Utils.Set.size
+
+let toArray = registry => registry.keys->Utils.Set.toArray->Array.toSorted(String.compare)
+
+// A plain CREATE INDEX either commits or leaves nothing behind, so an index can
+// only be invalid because something before us left it that way — the startup
+// snapshot stays accurate until the next restart.
+let getInvalid = (registry, name) => registry.invalidByName->Utils.Dict.dangerouslyGetNonOption(name)
+
+let clearInvalidName = (registry, name) => registry.invalidByName->Utils.Dict.deleteInPlace(name)
+
+// Replaces the whole registry with what the catalog reports. Returns the names
+// of indexes Postgres flagged invalid (eg a CREATE INDEX CONCURRENTLY that died
+// midway). They stay out of the registry, so the next request that needs one
+// repairs it.
+let reload = (registry, ~rows: array<catalogRow>) => {
+  registry.keys->Utils.Set.clear
+  registry.invalidByName
+  ->Dict.keysToArray
+  ->Array.forEach(name => registry.invalidByName->Utils.Dict.deleteInPlace(name))
+  let invalidIndexNames = []
+  rows->Array.forEach(row => {
+    let key = makeKey(
+      ~tableName=row.tableName,
+      ~columns=row.columns->Array.mapWithIndex((name, idx) => {
+        name,
+        direction: switch row.directions->Array.get(idx) {
+        | Some("DESC") => Table.Desc
+        | _ => Asc
+        },
+      }),
+      ~method=row.method,
+    )
+    if row.isValid === 0 {
+      invalidIndexNames->Array.push(row.indexName)->ignore
+      registry.invalidByName->Dict.set(row.indexName, {key, isRepairable: row.isPlain === 1})
+    } else {
+      registry->add(key)
+    }
+  })
+  invalidIndexNames
+}
+
+// Runs `build` unless the index is already registered. The key is added only
+// after `build` resolves, so a failed build leaves the registry untouched and
+// the next request retries.
+let ensure = (registry, ~key, ~tableName, ~build) =>
+  if registry->has(key) {
+    Promise.resolve()
+  } else {
+    switch registry.inflight->Utils.Dict.dangerouslyGetNonOption(key) {
+    | Some(promise) => promise
+    | None =>
+      let tail = switch registry.tableQueue->Utils.Dict.dangerouslyGetNonOption(tableName) {
+      | Some(tail) => tail
+      | None => Promise.resolve()
+      }
+      let promise =
+        tail
+        // The queued build may have been satisfied while waiting behind another
+        // one on the same table (eg a finalize pass created it).
+        ->Promise.then(() => registry->has(key) ? Promise.resolve() : build())
+        ->Promise.thenResolve(() => registry->add(key))
+      registry.inflight->Dict.set(key, promise)
+      registry.tableQueue->Dict.set(tableName, promise->Utils.Promise.silentCatch)
+      promise->Promise.finally(() => registry.inflight->Utils.Dict.deleteInPlace(key))
+    }
+  }
+
+let pgMaxIdentifierLength = 63
+
+// Postgres truncates identifiers past 63 characters on its own. Doing it here
+// instead keeps the name we emit, log and read back from the catalog identical
+// to the one it stores. Descriptions are ASCII (GraphQL names are), so
+// characters and Postgres' byte limit line up.
+let truncateIndexName = description =>
+  description->String.length > pgMaxIdentifierLength
+    ? description->String.slice(~start=0, ~end=pgMaxIdentifierLength)
+    : description
+
+// Two distinct descriptions can collapse onto one truncated name — and the
+// second `CREATE INDEX IF NOT EXISTS` would then be skipped without a trace,
+// leaving the registry claiming an index that doesn't exist.
+let validateIndexNamesOrThrow = (descriptions: array<string>) => {
+  let byName = Dict.make()
+  descriptions->Array.forEach(description => {
+    let name = description->truncateIndexName
+    switch byName->Utils.Dict.dangerouslyGetNonOption(name) {
+    | Some(existing) if existing !== description =>
+      JsError.throwWithMessage(
+        `Index names "${existing}" and "${description}" both truncate to "${name}" at PostgreSQL's ${pgMaxIdentifierLength->Int.toString}-character identifier limit. Rename a field or an entity so the generated index names stay distinct.`,
+      )
+    | _ => byName->Dict.set(name, description)
+    }
+  })
+}