envio 3.5.0-alpha.1 → 3.5.0-alpha.3

package/src/Writing.res.mjs

@@ -4,6 +4,7 @@ import * as Env from "./Env.res.mjs";
 import * as Utils from "./Utils.res.mjs";
 import * as Throttler from "./Throttler.res.mjs";
 import * as EffectState from "./EffectState.res.mjs";
+import * as Performance from "./bindings/Performance.res.mjs";
 import * as IndexerState from "./IndexerState.res.mjs";
 import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
 import * as InMemoryStore from "./InMemoryStore.res.mjs";
@@ -182,7 +183,7 @@ function dropCommitted(state, keepLoadedFromDb) {
   EffectState.forEach(IndexerState.effectState(state), inMemTable => InMemoryStore.dropCommittedEffects(inMemTable, committedCheckpointId, keepLoadedFromDb));
 }
 
-async function awaitCapacity(state) {
+async function awaitCapacityLoop(state) {
   if (!IndexerState.hasFailedWrite(state) && getChangesCount(state) >= Env.inMemoryObjectsTarget) {
     dropCommitted(state, true);
     if (getChangesCount(state) >= Env.inMemoryObjectsTarget) {
@@ -191,13 +192,22 @@ async function awaitCapacity(state) {
     if (getChangesCount(state) >= Env.inMemoryObjectsTarget && Utils.$$Array.notEmpty(IndexerState.processedBatches(state))) {
       schedule(state);
       await new Promise((resolve, param) => IndexerState.addCommitWaiter(state, resolve));
-      return await awaitCapacity(state);
+      return await awaitCapacityLoop(state);
     } else {
       return;
     }
   }
 }
 
+async function awaitCapacity(state) {
+  if (getChangesCount(state) < Env.inMemoryObjectsTarget) {
+    return;
+  }
+  let timeRef = Performance.now();
+  await awaitCapacityLoop(state);
+  return IndexerState.recordStalledOnStorageWrite(state, Performance.secondsSince(timeRef));
+}
+
 async function flush(state) {
   if (IndexerState.hasFailedWrite(state)) {
     return;
@@ -224,6 +234,7 @@ export {
   setChainMeta,
   commitBatch,
   dropCommitted,
+  awaitCapacityLoop,
   awaitCapacity,
   flush,
 }

package/src/bindings/ClickHouse.res

@@ -91,7 +91,6 @@ let getClickHouseFieldType = (
         ->Array.joinUnsafe(", ")
       `${enumType}(${enumValues})`
     }
-  | Entity(_) => "String"
   }
 
   let baseType = if isArray {
@@ -261,11 +260,14 @@ let setUpdatesOrThrow = async (
         convertOrThrow: S.compile(
           S.array(
             S.union([
-              EntityHistory.makeSetUpdateSchema(makeClickHouseEntitySchema(entityConfig.table)),
+              EntityHistory.makeSetUpdateSchema(
+                ~idSchema=entityConfig.table->Table.getIdSchema,
+                makeClickHouseEntitySchema(entityConfig.table),
+              ),
               S.object(s => {
                 s.tag(EntityHistory.changeFieldName, EntityHistory.RowAction.DELETE)
                 Change.Delete({
-                  entityId: s.field(Table.idFieldName, S.string),
+                  entityId: s.field(Table.idFieldName, entityConfig.table->Table.getIdSchema),
                   checkpointId: s.field(
                     EntityHistory.checkpointIdFieldName,
                     EntityHistory.unsafeCheckpointIdSchema,

package/src/bindings/ClickHouse.res.mjs

@@ -68,9 +68,6 @@ function getClickHouseFieldType(fieldType, isNullable, isArray) {
         let enumValues = config$1.variants.map(variant => `'` + variant + `'`).join(", ");
         baseType = enumType + `(` + enumValues + `)`;
         break;
-      case "Entity" :
-        baseType = "String";
-        break;
     }
   }
   let baseType$1 = isArray ? `Array(` + baseType + `)` : baseType;
@@ -203,12 +200,12 @@ async function setUpdatesOrThrow(client, cache, changes, entityConfig, database)
   } else {
     let cached_tableName = database + `.\`` + EntityHistory.historyTableName(entityConfig.name, entityConfig.index) + `\``;
     let cached_convertOrThrow = S$RescriptSchema.compile(S$RescriptSchema.array(S$RescriptSchema.union([
-      EntityHistory.makeSetUpdateSchema(makeClickHouseEntitySchema(entityConfig.table)),
+      EntityHistory.makeSetUpdateSchema(Table.getIdSchema(entityConfig.table), makeClickHouseEntitySchema(entityConfig.table)),
       S$RescriptSchema.object(s => {
         s.tag(EntityHistory.changeFieldName, "DELETE");
         return {
           type: "DELETE",
-          entityId: s.f(Table.idFieldName, S$RescriptSchema.string),
+          entityId: s.f(Table.idFieldName, Table.getIdSchema(entityConfig.table)),
           checkpointId: s.f(EntityHistory.checkpointIdFieldName, EntityHistory.unsafeCheckpointIdSchema)
         };
       })

package/src/bindings/Vitest.res

@@ -34,7 +34,6 @@ type rec expectation<'a> = {
   toHavePropertyValue: 'b. (string, 'b) => unit,
   // Exception matchers
   toThrow: unit => unit,
-  toThrowError: string => unit,
   // Snapshot matchers
   toMatchSnapshot: unit => unit,
   // Negation
@@ -151,3 +150,25 @@ module Async = {
 
 @module("vitest")
 external expect: ('a, ~message: string=?) => expectation<'a> = "expect"
+
+// Runs `fn` and returns the thrown error's message — mirroring JS's
+// `e instanceof Error ? e.message : String(e)` — or `None` when nothing threw.
+let messageOfThrown: (unit => 'a) => option<string> = %raw(`function (fn) {
+  try {
+    fn();
+    return undefined;
+  } catch (e) {
+    return e instanceof Error ? e.message : String(e);
+  }
+}`)
+
+// Strict counterpart to the built-in `toThrow`, which only checks that the
+// thrown message *contains* the expected string. `toThrowErrorEqual` requires
+// the whole message to match, so a test can pin the complete error text.
+// A plain ReScript function (not a custom `expect.extend` matcher), so it needs
+// no per-package vitest setup.
+// Compared as options rather than through a "didn't throw" placeholder string,
+// so a function that throws nothing can never match — not even when `expected`
+// happens to equal the placeholder.
+let toThrowErrorEqual = (t: testContext, fn: unit => 'a, ~message=?, expected: string) =>
+  t.expect(fn->messageOfThrown, ~message?).toEqual(Some(expected))

package/src/bindings/Vitest.res.mjs

@@ -13,7 +13,22 @@ let Async = {
   itSkipInClaudeCloud: itSkipInClaudeCloud
 };
 
+let messageOfThrown = (function (fn) {
+  try {
+    fn();
+    return undefined;
+  } catch (e) {
+    return e instanceof Error ? e.message : String(e);
+  }
+});
+
+function toThrowErrorEqual(t, fn, message, expected) {
+  t.expect(messageOfThrown(fn), message).toEqual(expected);
+}
+
 export {
   Async,
+  messageOfThrown,
+  toThrowErrorEqual,
 }
 /* isClaudeCloud Not a pure module */

package/src/db/EntityFilter.res

@@ -176,10 +176,6 @@ let parseGetWhereOrThrow = (filter: dict<dict<unknown>>, ~entityName, ~table: Ta
       JsError.throwWithMessage(
         `The field "${apiFieldName}" on entity "${entityName}" is a derived field and cannot be used in getWhere(). Use the source entity's indexed field instead.`,
       )
-    | Some(Field({isPrimaryKey: false, isIndex: false, linkedEntity: None})) =>
-      JsError.throwWithMessage(
-        `The field "${apiFieldName}" on entity "${entityName}" does not have an index. To use it in getWhere(), add the @index directive in your schema.graphql:\n\n  ${apiFieldName}: ... @index\n\nThen run 'pnpm envio codegen' to regenerate.`,
-      )
     | Some(Field(_)) => ()
     }
 

package/src/db/EntityFilter.res.mjs

@@ -142,14 +142,7 @@ function parseGetWhereOrThrow(filter, entityName, table) {
     });
     let match = Table.getFieldByApiName(table, apiFieldName);
     if (match !== undefined) {
-      if (match.TAG === "Field") {
-        let match$1 = match._0;
-        if (match$1.isPrimaryKey || match$1.isIndex || match$1.linkedEntity !== undefined) {
-          
-        } else {
-          Stdlib_JsError.throwWithMessage(`The field "` + apiFieldName + `" on entity "` + entityName + `" does not have an index. To use it in getWhere(), add the @index directive in your schema.graphql:\n\n  ` + apiFieldName + `: ... @index\n\nThen run 'pnpm envio codegen' to regenerate.`);
-        }
-      } else {
+      if (match.TAG !== "Field") {
         Stdlib_JsError.throwWithMessage(`The field "` + apiFieldName + `" on entity "` + entityName + `" is a derived field and cannot be used in getWhere(). Use the source entity's indexed field instead.`);
       }
     } else {

package/src/db/EntityHistory.res

@@ -30,12 +30,14 @@ let unsafeCheckpointIdSchema =
     serializer: bigint => bigint->BigInt.toString,
   })
 
-let makeSetUpdateSchema: S.t<'entity> => S.t<Change.t<'entity>> = entitySchema => {
+let makeSetUpdateSchema = (~idSchema: S.t<EntityId.t>, entitySchema: S.t<'entity>): S.t<
+  Change.t<'entity>,
+> => {
   S.object(s => {
     s.tag(changeFieldName, RowAction.SET)
     Change.Set({
       checkpointId: s.field(checkpointIdFieldName, unsafeCheckpointIdSchema),
-      entityId: s.field(Table.idFieldName, S.string),
+      entityId: s.field(Table.idFieldName, idSchema),
       entity: s.flatten(entitySchema),
     })
   })
@@ -118,10 +120,10 @@ let pruneStaleEntityHistory = (
 
 // If an entity doesn't have a history before the update
 // we create it automatically with envio_checkpoint_id 0
-let makeBackfillHistoryQuery = (~pgSchema, ~entityName, ~entityIndex) => {
+let makeBackfillHistoryQuery = (~pgSchema, ~entityName, ~entityIndex, ~idPgType) => {
   let historyTableRef = `"${pgSchema}"."${historyTableName(~entityName, ~entityIndex)}"`
   `WITH target_ids AS (
-  SELECT UNNEST($1::${(Text: Postgres.columnType :> string)}[]) AS id
+  SELECT UNNEST($1::${idPgType}[]) AS id
 ),
 missing_history AS (
   SELECT e.*
@@ -135,11 +137,18 @@ SELECT *, 0 AS ${checkpointIdFieldName}, '${(RowAction.SET :> string)}' as ${cha
 FROM missing_history;`
 }
 
-let backfillHistory = (sql, ~pgSchema, ~entityName, ~entityIndex, ~ids: array<string>) => {
+let backfillHistory = (
+  sql,
+  ~pgSchema,
+  ~table: Table.table,
+  ~entityIndex,
+  ~ids: array<EntityId.t>,
+) => {
+  let idPgType = table->Table.getIdPgFieldType(~pgSchema)
   sql
   ->Postgres.preparedUnsafe(
-    makeBackfillHistoryQuery(~entityName, ~entityIndex, ~pgSchema),
-    [ids]->Obj.magic,
+    makeBackfillHistoryQuery(~entityName=table.tableName, ~entityIndex, ~pgSchema, ~idPgType),
+    [table->Table.encodeIdsToJson(ids)]->(Utils.magic: array<JSON.t> => unknown),
   )
   ->Utils.Promise.ignoreValue
 }

package/src/db/EntityHistory.res.mjs

@@ -48,12 +48,12 @@ let unsafeCheckpointIdSchema = S$RescriptSchema.transform(S$RescriptSchema.setNa
   s: bigint => bigint.toString()
 }));
 
-function makeSetUpdateSchema(entitySchema) {
+function makeSetUpdateSchema(idSchema, entitySchema) {
   return S$RescriptSchema.object(s => {
     s.tag(changeFieldName, "SET");
     return {
       type: "SET",
-      entityId: s.f(Table.idFieldName, S$RescriptSchema.string),
+      entityId: s.f(Table.idFieldName, idSchema),
       entity: s.flatten(entitySchema),
       checkpointId: s.f(checkpointIdFieldName, unsafeCheckpointIdSchema)
     };
@@ -97,10 +97,10 @@ function pruneStaleEntityHistory(sql, entityName, entityIndex, pgSchema, safeChe
   return sql.unsafe(makePruneStaleEntityHistoryQuery(entityName, entityIndex, pgSchema), [safeCheckpointId.toString()], {prepare: true});
 }
 
-function makeBackfillHistoryQuery(pgSchema, entityName, entityIndex) {
+function makeBackfillHistoryQuery(pgSchema, entityName, entityIndex, idPgType) {
   let historyTableRef = `"` + pgSchema + `"."` + historyTableName(entityName, entityIndex) + `"`;
   return `WITH target_ids AS (
-  SELECT UNNEST($1::` + "TEXT" + `[]) AS id
+  SELECT UNNEST($1::` + idPgType + `[]) AS id
 ),
 missing_history AS (
   SELECT e.*
@@ -114,8 +114,9 @@ SELECT *, 0 AS ` + checkpointIdFieldName + `, '` + "SET" + `' as ` + changeField
 FROM missing_history;`;
 }
 
-function backfillHistory(sql, pgSchema, entityName, entityIndex, ids) {
-  return sql.unsafe(makeBackfillHistoryQuery(pgSchema, entityName, entityIndex), [ids], {prepare: true});
+function backfillHistory(sql, pgSchema, table, entityIndex, ids) {
+  let idPgType = Table.getIdPgFieldType(table, pgSchema);
+  return sql.unsafe(makeBackfillHistoryQuery(pgSchema, table.tableName, entityIndex, idPgType), [Table.encodeIdsToJson(table, ids)], {prepare: true});
 }
 
 function rollback(sql, pgSchema, entityName, entityIndex, rollbackTargetCheckpointId) {

package/src/db/IndexRegistry.res

@@ -0,0 +1,219 @@
+// An in-memory mirror of the indexes PostgreSQL holds for the indexer schema.
+// Postgres stays authoritative: the registry is loaded from pg_catalog at
+// startup/resume and afterwards only grows — optimistically, right after a
+// CREATE INDEX we issued succeeds. Only one Indexer instance writes to the
+// schema, so no catalog refresh is needed between those two points.
+
+let btree = "btree"
+
+type column = {
+  name: string,
+  direction: Table.indexFieldDirection,
+}
+
+let fromIndexFields = (indexFields: array<Table.compositeIndexField>) =>
+  indexFields->Array.map(({fieldName, direction}) => {name: fieldName, direction})
+
+// Identity of an index is table + ordered columns + direction + method. Names
+// are deliberately not part of it: the catalog is matched on what the index
+// actually covers, not on how it was spelled.
+let makeKey = (~tableName, ~columns: array<column>, ~method) =>
+  `${tableName}|${method}|${columns
+    ->Array.map(({name, direction}) =>
+      switch direction {
+      | Table.Asc => name
+      | Desc => name ++ " DESC"
+      }
+    )
+    ->Array.joinUnsafe(",")}`
+
+type catalogRow = {
+  tableName: string,
+  indexName: string,
+  method: string,
+  // 1/0 rather than a bool: postgres.js hands booleans back inconsistently
+  // depending on the driver's type resolution, so the query casts it.
+  isValid: int,
+  // 1 only for an index the indexer could have created: plain btree over plain
+  // columns. A unique or partial index carries a definition our key doesn't
+  // capture, so it can never be one of ours to rebuild.
+  isPlain: int,
+  columns: array<string>,
+  // "ASC"/"DESC" per column, in the same order as `columns`.
+  directions: array<string>,
+}
+
+let catalogRowsSchema = S.array(
+  S.object((s): catalogRow => {
+    tableName: s.field("tableName", S.string),
+    indexName: s.field("indexName", S.string),
+    method: s.field("method", S.string),
+    isValid: s.field("isValid", S.int),
+    isPlain: s.field("isPlain", S.int),
+    columns: s.field("columns", S.array(S.string)),
+    directions: s.field("directions", S.array(S.string)),
+  }),
+)
+
+// One row per index, with its key columns aggregated in ordinal order.
+// INCLUDE columns are excluded (they don't affect what the index can serve),
+// and expression columns come back as their printed definition so an
+// expression index can never be mistaken for a plain-column one.
+let makeCatalogQuery = (~pgSchema) =>
+  `SELECT
+  t.relname AS "tableName",
+  i.relname AS "indexName",
+  am.amname AS "method",
+  CASE WHEN ix.indisvalid AND ix.indisready THEN 1 ELSE 0 END AS "isValid",
+  CASE
+    WHEN ix.indisunique OR ix.indpred IS NOT NULL OR ix.indexprs IS NOT NULL THEN 0
+    ELSE 1
+  END AS "isPlain",
+  array_agg(
+    CASE WHEN k.attnum = 0
+      THEN pg_get_indexdef(ix.indexrelid, k.ord::int, true)
+      ELSE a.attname
+    END ORDER BY k.ord
+  ) AS "columns",
+  array_agg(
+    CASE WHEN (ix.indoption[k.ord - 1] & 1) = 1 THEN 'DESC' ELSE 'ASC' END ORDER BY k.ord
+  ) AS "directions"
+FROM pg_index ix
+JOIN pg_class i ON i.oid = ix.indexrelid
+JOIN pg_class t ON t.oid = ix.indrelid
+JOIN pg_namespace n ON n.oid = t.relnamespace
+JOIN pg_am am ON am.oid = i.relam
+JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) ON k.ord <= ix.indnkeyatts
+LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum
+WHERE n.nspname = '${pgSchema}'
+GROUP BY t.relname, i.relname, am.amname, ix.indisvalid, ix.indisready,
+         ix.indisunique, ix.indpred, ix.indexprs;`
+
+// What we know about an index that came back invalid, enough to decide whether
+// rebuilding it would produce the index we actually wanted.
+type invalidIndex = {
+  key: string,
+  isRepairable: bool,
+}
+
+type t = {
+  keys: Utils.Set.t<string>,
+  // Every index Postgres reports as invalid, by name. Kept because
+  // `CREATE INDEX IF NOT EXISTS` matches on name: with an invalid index already
+  // under the name we want, the DDL is a silent no-op and we'd register a key
+  // for something the planner refuses to use.
+  invalidByName: dict<invalidIndex>,
+  // Keyed by index key so identical concurrent requests await one build.
+  inflight: dict<promise<unit>>,
+  // Tail of the build chain per table, so two different indexes on the same
+  // table are built one after the other rather than at once.
+  tableQueue: dict<promise<unit>>,
+}
+
+let make = () => {
+  keys: Utils.Set.make(),
+  invalidByName: Dict.make(),
+  inflight: Dict.make(),
+  tableQueue: Dict.make(),
+}
+
+let has = (registry, key) => registry.keys->Utils.Set.has(key)
+
+let add = (registry, key) => registry.keys->Utils.Set.add(key)->ignore
+
+let size = registry => registry.keys->Utils.Set.size
+
+let toArray = registry => registry.keys->Utils.Set.toArray->Array.toSorted(String.compare)
+
+// A plain CREATE INDEX either commits or leaves nothing behind, so an index can
+// only be invalid because something before us left it that way — the startup
+// snapshot stays accurate until the next restart.
+let getInvalid = (registry, name) => registry.invalidByName->Utils.Dict.dangerouslyGetNonOption(name)
+
+let clearInvalidName = (registry, name) => registry.invalidByName->Utils.Dict.deleteInPlace(name)
+
+// Replaces the whole registry with what the catalog reports. Returns the names
+// of indexes Postgres flagged invalid (eg a CREATE INDEX CONCURRENTLY that died
+// midway). They stay out of the registry, so the next request that needs one
+// repairs it.
+let reload = (registry, ~rows: array<catalogRow>) => {
+  registry.keys->Utils.Set.clear
+  registry.invalidByName
+  ->Dict.keysToArray
+  ->Array.forEach(name => registry.invalidByName->Utils.Dict.deleteInPlace(name))
+  let invalidIndexNames = []
+  rows->Array.forEach(row => {
+    let key = makeKey(
+      ~tableName=row.tableName,
+      ~columns=row.columns->Array.mapWithIndex((name, idx) => {
+        name,
+        direction: switch row.directions->Array.get(idx) {
+        | Some("DESC") => Table.Desc
+        | _ => Asc
+        },
+      }),
+      ~method=row.method,
+    )
+    if row.isValid === 0 {
+      invalidIndexNames->Array.push(row.indexName)->ignore
+      registry.invalidByName->Dict.set(row.indexName, {key, isRepairable: row.isPlain === 1})
+    } else {
+      registry->add(key)
+    }
+  })
+  invalidIndexNames
+}
+
+// Runs `build` unless the index is already registered. The key is added only
+// after `build` resolves, so a failed build leaves the registry untouched and
+// the next request retries.
+let ensure = (registry, ~key, ~tableName, ~build) =>
+  if registry->has(key) {
+    Promise.resolve()
+  } else {
+    switch registry.inflight->Utils.Dict.dangerouslyGetNonOption(key) {
+    | Some(promise) => promise
+    | None =>
+      let tail = switch registry.tableQueue->Utils.Dict.dangerouslyGetNonOption(tableName) {
+      | Some(tail) => tail
+      | None => Promise.resolve()
+      }
+      let promise =
+        tail
+        // The queued build may have been satisfied while waiting behind another
+        // one on the same table (eg a finalize pass created it).
+        ->Promise.then(() => registry->has(key) ? Promise.resolve() : build())
+        ->Promise.thenResolve(() => registry->add(key))
+      registry.inflight->Dict.set(key, promise)
+      registry.tableQueue->Dict.set(tableName, promise->Utils.Promise.silentCatch)
+      promise->Promise.finally(() => registry.inflight->Utils.Dict.deleteInPlace(key))
+    }
+  }
+
+let pgMaxIdentifierLength = 63
+
+// Postgres truncates identifiers past 63 characters on its own. Doing it here
+// instead keeps the name we emit, log and read back from the catalog identical
+// to the one it stores. Descriptions are ASCII (GraphQL names are), so
+// characters and Postgres' byte limit line up.
+let truncateIndexName = description =>
+  description->String.length > pgMaxIdentifierLength
+    ? description->String.slice(~start=0, ~end=pgMaxIdentifierLength)
+    : description
+
+// Two distinct descriptions can collapse onto one truncated name — and the
+// second `CREATE INDEX IF NOT EXISTS` would then be skipped without a trace,
+// leaving the registry claiming an index that doesn't exist.
+let validateIndexNamesOrThrow = (descriptions: array<string>) => {
+  let byName = Dict.make()
+  descriptions->Array.forEach(description => {
+    let name = description->truncateIndexName
+    switch byName->Utils.Dict.dangerouslyGetNonOption(name) {
+    | Some(existing) if existing !== description =>
+      JsError.throwWithMessage(
+        `Index names "${existing}" and "${description}" both truncate to "${name}" at PostgreSQL's ${pgMaxIdentifierLength->Int.toString}-character identifier limit. Rename a field or an entity so the generated index names stay distinct.`,
+      )
+    | _ => byName->Dict.set(name, description)
+    }
+  })
+}