envio 3.5.0-alpha.2 → 3.5.0-alpha.3

package/src/PgStorage.res

@@ -58,7 +58,7 @@ let makeCreateCompositeIndexQuery = (
   `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${pgSchema}"."${tableName}"(${index});`
 }
 
-let makeCreateTableIndicesQuery = (table: Table.table, ~pgSchema) => {
+let makeCreateTableIndexesQuery = (table: Table.table, ~pgSchema) => {
   let tableName = table.tableName
   let createIndex = indexField =>
     makeCreateIndexQuery(~tableName, ~indexFields=[indexField], ~pgSchema)
@@ -66,11 +66,138 @@ let makeCreateTableIndicesQuery = (table: Table.table, ~pgSchema) => {
     makeCreateCompositeIndexQuery(~tableName, ~indexFields, ~pgSchema)
   }
 
-  let singleIndices = table->Table.getSingleIndices
-  let compositeIndices = table->Table.getCompositeIndices
+  let singleIndexes = table->Table.getSingleIndexes
+  let compositeIndexes = table->Table.getCompositeIndexes
 
-  singleIndices->Array.map(createIndex)->Array.joinUnsafe("\n") ++
-    compositeIndices->Array.map(createCompositeIndex)->Array.joinUnsafe("\n")
+  singleIndexes->Array.map(createIndex)->Array.joinUnsafe("\n") ++
+    compositeIndexes->Array.map(createCompositeIndex)->Array.joinUnsafe("\n")
+}
+
+// One index the schema promises: an `@index` field, a composite index, or the
+// index backing a derived relationship. Carries enough to derive its name, its
+// DDL and its registry key, so the deferred creation at the end of backfill and
+// the registry stay described by the same value.
+type schemaIndex = {
+  tableName: string,
+  indexFields: array<Table.compositeIndexField>,
+}
+
+// `<Entity>_<column>` for a single column, with each further column appended in
+// order (and `_desc` where it applies). The full description, before Postgres'
+// identifier limit is applied — `validateIndexNamesOrThrow` compares these, so
+// it can tell two descriptions apart after they truncate to the same name.
+let schemaIndexDescription = ({tableName, indexFields}: schemaIndex) =>
+  tableName ++
+  "_" ++
+  indexFields
+  ->Array.map(f => f.fieldName ++ directionToIndexName(f.direction))
+  ->Array.joinUnsafe("_")
+
+// The identifier Postgres actually stores. Automatic and schema-defined indexes
+// share it, so both are named the same way and both stay reachable through
+// `CREATE INDEX IF NOT EXISTS`.
+let schemaIndexName = schemaIndex =>
+  schemaIndex->schemaIndexDescription->IndexRegistry.truncateIndexName
+
+let schemaIndexKey = ({tableName, indexFields}: schemaIndex) =>
+  IndexRegistry.makeKey(
+    ~tableName,
+    ~columns=indexFields->IndexRegistry.fromIndexFields,
+    ~method=IndexRegistry.btree,
+  )
+
+// Plain DDL, not CONCURRENTLY: it builds from a single table scan instead of
+// two, and the SHARE lock it takes blocks writes but not reads, so queries keep
+// being served while it runs. The indexer is the only writer, and both callers
+// are happy to wait on it.
+let makeCreateSchemaIndexQuery = (schemaIndex, ~pgSchema) => {
+  let {tableName, indexFields} = schemaIndex
+  let columns =
+    indexFields
+    ->Array.map(f => `"${f.fieldName}"${directionToSql(f.direction)}`)
+    ->Array.joinUnsafe(", ")
+  `CREATE INDEX IF NOT EXISTS "${schemaIndex->schemaIndexName}" ON "${pgSchema}"."${tableName}"(${columns});`
+}
+
+let formatSeconds = (timeRef: Performance.timeRef) =>
+  (Math.round(timeRef->Performance.secondsSince *. 100.) /. 100.)->Float.toString
+
+// Every index build blocks writes to its table for as long as it runs, and on a
+// large database that is not quick. Both build paths say so up front, so a
+// stalled-looking indexer is explainable from the logs alone.
+let slowOnLargeDatabaseNotice = "This can take a long time on a large database."
+
+// An index Postgres reports as INVALID still owns its name, so
+// `CREATE INDEX IF NOT EXISTS` would quietly skip it and leave the query
+// unserved. REINDEX rebuilds it from its own stored definition — a repair, not
+// a replacement — which is right only when that definition is the one we
+// wanted. Runs inside a transaction, unlike REINDEX CONCURRENTLY.
+let makeReindexQuery = (~pgSchema, ~indexName) => `REINDEX INDEX "${pgSchema}"."${indexName}";`
+
+// A name taken by an invalid index the indexer didn't create isn't ours to
+// touch: rebuilding it reuses its own definition, so it would come back valid
+// but still not serve the query, and dropping it could take out a unique
+// constraint someone else relies on.
+let nameCollisionError = (~indexName, ~pgSchema) =>
+  Utils.Error.make(
+    `Cannot create the index "${indexName}" in schema "${pgSchema}". An index of that name already exists and PostgreSQL reports it as invalid, but it isn't one the indexer created — it covers different columns, or it is a unique or partial index — so it is left untouched. Drop or repair it by hand, then restart the indexer.`,
+  )
+
+let makeSingleColumnSchemaIndex = (~tableName, ~column): schemaIndex => {
+  tableName,
+  indexFields: [{fieldName: column, direction: Table.Asc}],
+}
+
+// Every index the entity schema promises. Deferred past the initial DDL and
+// created in one transaction once backfill completes, so a resumed indexer that
+// reports itself ready always has all of them.
+//
+// `entities` is the Postgres-backed set, and every `@derivedFrom` target within
+// it resolves: config parsing rejects a Postgres entity deriving from one that
+// isn't in Postgres (`validate_relationship_storage`).
+let getSchemaIndexes = (~entities: array<Internal.entityConfig>): array<schemaIndex> => {
+  let derivedSchema = Schema.make(entities->Array.map(e => e.table))
+  let all = []
+
+  entities->Array.forEach(({table}) => {
+    table
+    ->Table.getSingleIndexes
+    ->Array.forEach(column =>
+      all->Array.push(makeSingleColumnSchemaIndex(~tableName=table.tableName, ~column))->ignore
+    )
+    table
+    ->Table.getCompositeIndexes
+    ->Array.forEach(indexFields =>
+      all->Array.push({tableName: table.tableName, indexFields})->ignore
+    )
+  })
+
+  entities->Array.forEach(({table}) =>
+    table
+    ->Table.getDerivedFromFields
+    ->Array.forEach(derivedFromField => {
+      let column =
+        derivedSchema->Schema.getDerivedFromPgFieldName(derivedFromField)->Utils.unwrapResultExn
+      all
+      ->Array.push(
+        makeSingleColumnSchemaIndex(~tableName=derivedFromField.derivedFromEntity, ~column),
+      )
+      ->ignore
+    })
+  )
+
+  // An `@index` field and a derived relationship pointing at it describe the
+  // same index, so the list is deduped on identity rather than on name.
+  let seen = Utils.Set.make()
+  all->Array.filter(schemaIndex => {
+    let key = schemaIndex->schemaIndexKey
+    if seen->Utils.Set.has(key) {
+      false
+    } else {
+      seen->Utils.Set.add(key)->ignore
+      true
+    }
+  })
 }
 
 let makeCreateTableQuery = (table: Table.table, ~pgSchema, ~isNumericArrayAsText) => {
@@ -152,7 +279,7 @@ let getEntityHistory = (~entityConfig: Internal.entityConfig): EntityHistory.pgE
         ~entityName=entityTableName,
         ~entityIndex=entityConfig.index,
       )
-      //ignore composite indices
+      //ignore composite indexes
       let table = Table.mkTable(
         historyTableName,
         ~fields=dataFields->Array.concat([checkpointIdField, actionField]),
@@ -183,6 +310,10 @@ let makeInitializeTransaction = (
   ~entities=[],
   ~enums=[],
   ~isEmptyPgSchema=false,
+  // Backfill writes are far cheaper without the schema's read indexes, so the
+  // initial DDL creates only tables, primary keys, views and chain rows; the
+  // rest is created by `finalizeBackfill` before the indexer reports ready.
+  ~deferSchemaIndexes=false,
 ) => {
   let generalTables = [
     InternalTable.Chains.table,
@@ -192,13 +323,13 @@ let makeInitializeTransaction = (
   ]
 
   let allTables = generalTables->Array.copy
-  let allEntityTables = []
   entities->Array.forEach((entityConfig: Internal.entityConfig) => {
-    allEntityTables->Array.push(entityConfig.table)->ignore
     allTables->Array.push(entityConfig.table)->ignore
     allTables->Array.push(getEntityHistory(~entityConfig).table)->ignore
   })
-  let derivedSchema = Schema.make(allEntityTables)
+
+  let schemaIndexes = getSchemaIndexes(~entities)
+  IndexRegistry.validateIndexNamesOrThrow(schemaIndexes->Array.map(schemaIndexDescription))
 
   let query = ref(
     (
@@ -232,31 +363,12 @@ GRANT ALL ON SCHEMA "${pgSchema}" TO public;`,
       makeCreateTableQuery(table, ~pgSchema, ~isNumericArrayAsText=isHasuraEnabled)
   })
 
-  // Then batch all indices (better performance when tables exist)
-  allTables->Array.forEach((table: Table.table) => {
-    let indices = makeCreateTableIndicesQuery(table, ~pgSchema)
-    if indices !== "" {
-      query := query.contents ++ "\n" ++ indices
-    }
-  })
-
-  // Add derived indices
-  entities->Array.forEach((entity: Internal.entityConfig) => {
-    entity.table
-    ->Table.getDerivedFromFields
-    ->Array.forEach(derivedFromField => {
-      let indexField =
-        derivedSchema->Schema.getDerivedFromPgFieldName(derivedFromField)->Utils.unwrapResultExn
-      query :=
-        query.contents ++
-        "\n" ++
-        makeCreateIndexQuery(
-          ~tableName=derivedFromField.derivedFromEntity,
-          ~indexFields=[indexField],
-          ~pgSchema,
-        )
+  // Then batch all indexes (better performance when tables exist)
+  if !deferSchemaIndexes {
+    schemaIndexes->Array.forEach(schemaIndex => {
+      query := query.contents ++ "\n" ++ makeCreateSchemaIndexQuery(schemaIndex, ~pgSchema)
     })
-  })
+  }
 
   // Create views for Hasura integration
   query := query.contents ++ "\n" ++ InternalTable.Views.makeMetaViewQuery(~pgSchema)
@@ -1293,6 +1405,37 @@ let make = (
     "cache",
   ])
 
+  // The metric label, the `storage` field on this backend's logs, and the
+  // storage record's own name are all the same string.
+  let storageName = "postgres"
+
+  let indexRegistry = IndexRegistry.make()
+
+  // The only point where the catalog is consulted. Afterwards this process is
+  // the sole writer of the schema, so the registry is kept in sync by adding
+  // each index right after its DDL succeeds.
+  let reloadIndexRegistry = async () => {
+    let rows =
+      (await sql->Postgres.unsafe(IndexRegistry.makeCatalogQuery(~pgSchema)))->S.parseOrThrow(
+        IndexRegistry.catalogRowsSchema,
+      )
+    switch indexRegistry->IndexRegistry.reload(~rows) {
+    | [] => ()
+    | invalidIndexNames =>
+      Logging.warn({
+        "storage": storageName,
+        "msg": `Ignoring invalid PostgreSQL indexes in schema "${pgSchema}". They are left untouched — drop and recreate them manually if you need them.`,
+        "indexes": invalidIndexNames,
+      })
+    }
+    Logging.info({
+      "storage": storageName,
+      "msg": `Found ${indexRegistry
+        ->IndexRegistry.size
+        ->Int.toString} existing indexes in the indexer schema.`,
+    })
+  }
+
   let isInitialized = async () => {
     let envioTables = await sql->Postgres.unsafe(
       `SELECT table_schema FROM information_schema.tables WHERE table_schema = '${pgSchema}' AND (table_name = '${// This is for indexer before envio@2.28
@@ -1484,6 +1627,7 @@ let make = (
       ~chainConfigs,
       ~isEmptyPgSchema=schemaTableNames->Utils.Array.isEmpty,
       ~isHasuraEnabled,
+      ~deferSchemaIndexes=true,
     )
     // Execute all queries within a single transaction for integrity.
     // The envio_info row is written in the same transaction so a successful
@@ -1520,6 +1664,8 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
 
     let cache = await restoreEffectCache(~withUpload=true)
 
+    await reloadIndexRegistry()
+
     // Integration with other tools like Hasura
     switch onInitialize {
     | Some(onInitialize) => await onInitialize()
@@ -1576,6 +1722,177 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
     }
   }
 
+  // The physical columns a filter reads, deduped and in first-seen order.
+  // Unknown field names are left to `loadOrThrow`, which reports them properly.
+  let filterColumns = (~table: Table.table, ~filters: array<EntityFilter.t>) => {
+    let queryFields = table->Table.queryFields
+    let columns = []
+    let seen = Utils.Set.make()
+    let rec collect = (filter: EntityFilter.t) =>
+      switch filter {
+      | Eq({fieldName}) | Gt({fieldName}) | Lt({fieldName}) | In({fieldName}) =>
+        switch queryFields->Utils.Dict.dangerouslyGetNonOption(fieldName) {
+        | Some({pgDbFieldName}) =>
+          if !(seen->Utils.Set.has(pgDbFieldName)) {
+            seen->Utils.Set.add(pgDbFieldName)->ignore
+            columns->Array.push(pgDbFieldName)->ignore
+          }
+        | None => ()
+        }
+      | And({filters}) => filters->Array.forEach(collect)
+      }
+    filters->Array.forEach(collect)
+    columns
+  }
+
+  // The DDL that leaves `schemaIndex` in place. Normally a create; a repair
+  // when an invalid index already holds the name with the definition we want.
+  let makeEnsureIndexQuery = schemaIndex => {
+    let indexName = schemaIndex->schemaIndexName
+    switch indexRegistry->IndexRegistry.getInvalid(indexName) {
+    | None => makeCreateSchemaIndexQuery(schemaIndex, ~pgSchema)
+    | Some({key, isRepairable: true}) if key === schemaIndex->schemaIndexKey =>
+      makeReindexQuery(~pgSchema, ~indexName)
+    | Some(_) => throw(nameCollisionError(~indexName, ~pgSchema))
+    }
+  }
+
+  let isRepair = schemaIndex =>
+    indexRegistry->IndexRegistry.getInvalid(schemaIndex->schemaIndexName)->Option.isSome
+
+  let ensureQueryIndexes = async (~table: Table.table, ~filters: array<EntityFilter.t>) => {
+    let missing = filterColumns(~table, ~filters)->Array.filterMap(column => {
+      let schemaIndex = makeSingleColumnSchemaIndex(~tableName=table.tableName, ~column)
+      indexRegistry->IndexRegistry.has(schemaIndex->schemaIndexKey) ? None : Some(schemaIndex)
+    })
+
+    if missing->Utils.Array.notEmpty {
+      let _ = await missing
+      ->Array.map(schemaIndex => {
+        let indexName = schemaIndex->schemaIndexName
+        indexRegistry
+        ->IndexRegistry.ensure(
+          ~key=schemaIndex->schemaIndexKey,
+          ~tableName=table.tableName,
+          ~build=async () => {
+            // Resolved before logging so a repair is reported as one, and an
+            // unrelated index holding the name fails before any DDL runs.
+            let query = makeEnsureIndexQuery(schemaIndex)
+            let verb = schemaIndex->isRepair ? "Repairing invalid index" : "Creating index"
+            // Logged from inside the build so it reports the one request that
+            // actually creates the index, not the ones waiting on it.
+            Logging.info({
+              "storage": storageName,
+              "msg": `${verb} "${indexName}" to serve a getWhere query on "${table.tableName}". Writes to the table are paused until it completes. ${slowOnLargeDatabaseNotice}`,
+            })
+            let timeRef = Performance.now()
+            let _ = await sql->Postgres.unsafe(query)
+            indexRegistry->IndexRegistry.clearInvalidName(indexName)
+            Logging.info({
+              "storage": storageName,
+              "msg": `Index "${indexName}" is ready after ${timeRef->formatSeconds}s. Resuming indexing.`,
+            })
+          },
+        )
+        // A failed build leaves the registry untouched, so the next getWhere
+        // retries. Meanwhile the query still runs — just without the index.
+        ->Utils.Promise.catchResolve(exn => {
+          Logging.warn({
+            "storage": storageName,
+            "msg": `Failed to create the index "${indexName}" for a getWhere query. The query runs without it.`,
+            "err": exn->Utils.prettifyExn,
+          })
+        })
+      })
+      ->Promise.all
+    }
+  }
+
+  let finalizeBackfill = async (
+    ~entities: array<Internal.entityConfig>,
+    ~chainIds: array<int>,
+    ~readyAt: Date.t,
+  ) => {
+    let schemaIndexes = getSchemaIndexes(
+      ~entities=entities->Array.filter((e: Internal.entityConfig) => e.storage.postgres),
+    )
+    IndexRegistry.validateIndexNamesOrThrow(schemaIndexes->Array.map(schemaIndexDescription))
+
+    let missing =
+      schemaIndexes->Array.filter(schemaIndex =>
+        !(indexRegistry->IndexRegistry.has(schemaIndex->schemaIndexKey))
+      )
+
+    // `IF NOT EXISTS` would quietly skip an index whose name is taken by an
+    // invalid one, and the indexer would then report ready with an index the
+    // planner can't use. Those are repaired in the same transaction instead.
+    let repaired = missing->Array.filter(isRepair)
+    if repaired->Utils.Array.notEmpty {
+      Logging.warn({
+        "storage": storageName,
+        "msg": `PostgreSQL reports ${repaired
+          ->Array.length
+          ->Int.toString} existing indexes as invalid, so they can't serve queries. Rebuilding them.`,
+        "indexes": repaired->Array.map(schemaIndexName),
+      })
+    }
+
+    switch missing {
+    | [] =>
+      Logging.info({
+        "storage": storageName,
+        "msg": `All ${schemaIndexes
+          ->Array.length
+          ->Int.toString} schema indexes are already in place. Marking the indexer ready.`,
+      })
+    | _ =>
+      Logging.info({
+        "storage": storageName,
+        "msg": `Creating the ${missing
+          ->Array.length
+          ->Int.toString} remaining schema indexes before the indexer reports ready. Writes are paused until they are committed. ${slowOnLargeDatabaseNotice}`,
+        "indexes": missing->Array.map(schemaIndexName),
+      })
+    }
+    // Resolved up front so a name held by an unrelated invalid index fails
+    // before the transaction opens, rather than rolling one back.
+    let indexQueries = missing->Array.map(makeEnsureIndexQuery)
+    let timeRef = Performance.now()
+
+    await sql->Postgres.beginSql(async sql => {
+      // Sequential rather than Promise.all: they share one connection inside
+      // the transaction, and several of them target the same table.
+      for idx in 0 to indexQueries->Array.length - 1 {
+        let _ = await sql->Postgres.unsafe(indexQueries->Array.getUnsafe(idx))
+      }
+      await sql
+      ->Postgres.preparedUnsafe(
+        InternalTable.Chains.makeSetReadyAtQuery(~pgSchema),
+        [
+          readyAt->(Utils.magic: Date.t => unknown),
+          chainIds->(Utils.magic: array<int> => unknown),
+        ]->(Utils.magic: array<unknown> => unknown),
+      )
+      ->Utils.Promise.ignoreValue
+    })
+
+    // Only after the commit: a rollback takes both the indexes and ready_at
+    // with it, and the registry must not claim what the schema doesn't have.
+    missing->Array.forEach(schemaIndex =>
+      indexRegistry->IndexRegistry.add(schemaIndex->schemaIndexKey)
+    )
+    repaired->Array.forEach(schemaIndex =>
+      indexRegistry->IndexRegistry.clearInvalidName(schemaIndex->schemaIndexName)
+    )
+
+    Logging.info({
+      "storage": storageName,
+      "msg": `Committed ${missing
+        ->Array.length
+        ->Int.toString} schema indexes and the ready timestamp in ${timeRef->formatSeconds}s.`,
+    })
+  }
+
   let setOrThrow = (
     type item,
     ~items: array<item>,
@@ -1720,6 +2037,8 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       InternalTable.EnvioInfo.read(sql, ~pgSchema),
     ))
 
+    await reloadIndexRegistry()
+
     let checkpointId = (checkpointIdResult->Array.getUnsafe(0))["id"]->BigInt.fromStringOrThrow
 
     // Convert string checkpoint IDs from DB to bigint
@@ -1869,17 +2188,19 @@ SELECT id, chain_id, -1, -1, contract_name FROM unnest($1::text[],$2::int[],$3::
       ~sinkPromise,
       ~chainMetaData,
     )
-    onWrite(~storage="postgres", ~timeSeconds=primaryTimerRef->Performance.secondsSince)
+    onWrite(~storage=storageName, ~timeSeconds=primaryTimerRef->Performance.secondsSince)
   }
 
   let close = () => sql->Postgres.endSql
 
   {
-    name: "postgres",
+    name: storageName,
     isInitialized,
     initialize,
     resumeInitialState,
     loadOrThrow,
+    ensureQueryIndexes,
+    finalizeBackfill,
     dumpEffectCache,
     reset,
     setChainMeta,