envio 3.5.0-alpha.2 → 3.5.0-rc.0

package/src/db/IndexCatalog.res.mjs

@@ -0,0 +1,222 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+import * as Utils from "../Utils.res.mjs";
+import * as Stdlib_Array from "@rescript/runtime/lib/es6/Stdlib_Array.js";
+import * as Stdlib_Option from "@rescript/runtime/lib/es6/Stdlib_Option.js";
+import * as IndexDefinition from "./IndexDefinition.res.mjs";
+import * as Primitive_string from "@rescript/runtime/lib/es6/Primitive_string.js";
+import * as S$RescriptSchema from "rescript-schema/src/S.res.mjs";
+
+let rowsSchema = S$RescriptSchema.array(S$RescriptSchema.object(s => ({
+  tableName: s.f("tableName", S$RescriptSchema.string),
+  indexName: s.f("indexName", S$RescriptSchema.string),
+  method: s.f("method", S$RescriptSchema.string),
+  isValid: s.f("isValid", S$RescriptSchema.int),
+  isUnique: s.f("isUnique", S$RescriptSchema.int),
+  isPartial: s.f("isPartial", S$RescriptSchema.int),
+  isExpression: s.f("isExpression", S$RescriptSchema.int),
+  predicate: s.f("predicate", S$RescriptSchema.$$null(S$RescriptSchema.string)),
+  columns: s.f("columns", S$RescriptSchema.array(S$RescriptSchema.string)),
+  directions: s.f("directions", S$RescriptSchema.array(S$RescriptSchema.string))
+})));
+
+function makeQuery(pgSchema, indexName) {
+  return `SELECT
+  t.relname AS "tableName",
+  i.relname AS "indexName",
+  am.amname AS "method",
+  CASE WHEN ix.indisvalid AND ix.indisready THEN 1 ELSE 0 END AS "isValid",
+  CASE WHEN ix.indisunique THEN 1 ELSE 0 END AS "isUnique",
+  CASE WHEN ix.indpred IS NOT NULL THEN 1 ELSE 0 END AS "isPartial",
+  CASE WHEN ix.indexprs IS NOT NULL THEN 1 ELSE 0 END AS "isExpression",
+  pg_get_expr(ix.indpred, ix.indrelid, true) AS "predicate",
+  array_agg(
+    CASE WHEN k.attnum = 0
+      THEN pg_get_indexdef(ix.indexrelid, k.ord::int, true)
+      ELSE a.attname
+    END ORDER BY k.ord
+  ) AS "columns",
+  array_agg(
+    CASE WHEN (ix.indoption[k.ord - 1] & 1) = 1 THEN 'DESC' ELSE 'ASC' END ORDER BY k.ord
+  ) AS "directions"
+FROM pg_index ix
+JOIN pg_class i ON i.oid = ix.indexrelid
+JOIN pg_class t ON t.oid = ix.indrelid
+JOIN pg_namespace n ON n.oid = t.relnamespace
+JOIN pg_am am ON am.oid = i.relam
+JOIN LATERAL unnest(ix.indkey) WITH ORDINALITY AS k(attnum, ord) ON k.ord <= ix.indnkeyatts
+LEFT JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum
+WHERE n.nspname = '` + pgSchema + `'` + (
+    indexName !== undefined ? ` AND i.relname = '` + indexName + `'` : ""
+  ) + `
+GROUP BY t.relname, i.relname, am.amname, ix.indisvalid, ix.indisready,
+         ix.indisunique, ix.indpred, ix.indrelid, (ix.indexprs IS NOT NULL);`;
+}
+
+function fromRow(row) {
+  return {
+    name: row.indexName,
+    tableName: row.tableName,
+    method: row.method,
+    columns: row.columns.map((name, idx) => {
+      let match = row.directions[idx];
+      let tmp;
+      tmp = match === "DESC" ? "Desc" : "Asc";
+      return {
+        name: name,
+        direction: tmp
+      };
+    }),
+    isValid: row.isValid === 1,
+    isUnique: row.isUnique === 1,
+    isPartial: row.isPartial === 1,
+    isExpression: row.isExpression === 1,
+    predicate: row.predicate
+  };
+}
+
+function leadsWith(entry, definition) {
+  if (definition.columns.length <= entry.columns.length) {
+    return definition.columns.every((column, idx) => {
+      let actual = entry.columns[idx];
+      if (actual !== undefined && actual.name === column.name) {
+        return actual.direction === column.direction;
+      } else {
+        return false;
+      }
+    });
+  } else {
+    return false;
+  }
+}
+
+function coverageKey(coverage) {
+  if (coverage === "Exact") {
+    return "=";
+  } else {
+    return "^";
+  }
+}
+
+function rejectReason(entry, definition, coverage) {
+  if (entry.tableName !== definition.tableName) {
+    return `it is on table "` + entry.tableName + `"`;
+  } else if (entry.isValid) {
+    if (entry.isPartial) {
+      return `it is partial (WHERE ` + Stdlib_Option.getOr(entry.predicate, "...") + `), so it only covers part of the table`;
+    } else if (entry.isExpression) {
+      return "it indexes an expression rather than plain columns";
+    } else if (entry.method !== definition.method) {
+      return `it uses the ` + entry.method + ` access method, not ` + definition.method;
+    } else if (leadsWith(entry, definition)) {
+      if (coverage === "Exact" && entry.columns.length !== definition.columns.length) {
+        return `it covers (` + entry.columns.map(IndexDefinition.columnKey).join(", ") + `) rather than exactly the declared columns`;
+      } else {
+        return;
+      }
+    } else {
+      return `it covers (` + entry.columns.map(IndexDefinition.columnKey).join(", ") + `) instead of leading with the requested columns`;
+    }
+  } else {
+    return "PostgreSQL reports it as invalid or not ready";
+  }
+}
+
+function satisfies(entry, definition, coverage) {
+  return Stdlib_Option.isNone(rejectReason(entry, definition, coverage));
+}
+
+function isExactly(entry, definition) {
+  if (entry.tableName === definition.tableName && entry.method === definition.method && !entry.isUnique && !entry.isPartial && !entry.isExpression && entry.columns.length === definition.columns.length) {
+    return leadsWith(entry, definition);
+  } else {
+    return false;
+  }
+}
+
+function make() {
+  return {
+    byName: {},
+    covering: {}
+  };
+}
+
+function set(catalog, entry) {
+  catalog.byName[entry.name] = entry;
+  catalog.covering = {};
+}
+
+function remove(catalog, name) {
+  Utils.Dict.deleteInPlace(catalog.byName, name);
+  catalog.covering = {};
+}
+
+function fromRows(rows) {
+  let catalog = {
+    byName: {},
+    covering: {}
+  };
+  rows.forEach(row => set(catalog, fromRow(row)));
+  return catalog;
+}
+
+function getByName(catalog, name) {
+  return catalog.byName[name];
+}
+
+function entries(catalog) {
+  return Object.values(catalog.byName);
+}
+
+function size(catalog) {
+  return Object.keys(catalog.byName).length;
+}
+
+let nothingCovers = "";
+
+function find(catalog, definition, coverage) {
+  let cacheKey = coverageKey(coverage) + IndexDefinition.key(definition);
+  let name = catalog.covering[cacheKey];
+  if (name !== undefined) {
+    if (name === nothingCovers) {
+      return;
+    } else {
+      return catalog.byName[name];
+    }
+  }
+  let found = Object.values(catalog.byName).find(entry => satisfies(entry, definition, coverage));
+  catalog.covering[cacheKey] = found !== undefined ? found.name : nothingCovers;
+  return found;
+}
+
+function invalidNames(catalog) {
+  return Stdlib_Array.filterMap(Object.values(catalog.byName), entry => {
+    if (entry.isValid) {
+      return;
+    } else {
+      return entry.name;
+    }
+  }).toSorted(Primitive_string.compare);
+}
+
+export {
+  rowsSchema,
+  makeQuery,
+  fromRow,
+  leadsWith,
+  coverageKey,
+  rejectReason,
+  satisfies,
+  isExactly,
+  make,
+  set,
+  remove,
+  fromRows,
+  getByName,
+  entries,
+  size,
+  nothingCovers,
+  find,
+  invalidNames,
+}
+/* rowsSchema Not a pure module */

package/src/db/IndexDefinition.res

@@ -0,0 +1,131 @@
+// What the indexer wants an index to be: a table, its ordered key columns with
+// their directions, and an access method. That tuple is the index's identity;
+// the name is derived from it rather than being part of it, so the catalog can
+// always be matched on what an index actually covers.
+
+let btree = "btree"
+
+type column = {
+  name: string,
+  direction: Table.indexFieldDirection,
+}
+
+type t = {
+  tableName: string,
+  columns: array<column>,
+  method: string,
+}
+
+let make = (~tableName, ~columns, ~method=btree) => {tableName, columns, method}
+
+let single = (~tableName, ~column) =>
+  make(~tableName, ~columns=[{name: column, direction: Table.Asc}])
+
+let fromIndexFields = (~tableName, ~indexFields: array<Table.compositeIndexField>) =>
+  make(
+    ~tableName,
+    ~columns=indexFields->Array.map(({fieldName, direction}) => {name: fieldName, direction}),
+  )
+
+let columnKey = ({name, direction}) =>
+  switch direction {
+  | Table.Asc => name
+  | Desc => `${name} DESC`
+  }
+
+let key = ({tableName, columns, method}) =>
+  `${tableName}|${method}|${columns->Array.map(columnKey)->Array.joinUnsafe(",")}`
+
+let describe = definition =>
+  `${definition.tableName}(${definition.columns
+    ->Array.map(columnKey)
+    ->Array.joinUnsafe(", ")}) using ${definition.method}`
+
+@scope("Math") @val external imul: (int, int) => int = "imul"
+
+// 2166136261 as a signed 32-bit int.
+let fnvOffsetBasis = -2128831035
+let fnvPrime = 16777619
+
+let fnv1a = (input, ~seed) => {
+  let hash = ref(seed)
+  for idx in 0 to input->String.length - 1 {
+    hash := imul(hash.contents->Int.bitwiseXor(input->String.charCodeAtUnsafe(idx)), fnvPrime)
+  }
+  hash.contents
+}
+
+let hashLength = 10
+
+let toBase36 = (value, ~length) => value->Int.toString(~radix=36)->String.padStart(length, "0")
+
+// 50 bits of the identity, base36-encoded. Two indexes sharing a hash would
+// share an identifier, so the width matters: 50 bits is far past the number of
+// indexes a schema can hold, while still fitting in 10 characters. The halves
+// are encoded separately because combining them would overflow the 32-bit
+// integer arithmetic the shifts compile to.
+let identityHash = definition => {
+  let key = definition->key
+  let low = fnv1a(key, ~seed=fnvOffsetBasis)->Int.shiftRightUnsigned(2)
+  let high =
+    fnv1a(key, ~seed=fnvOffsetBasis->Int.bitwiseXor(0x27d4eb2f))->Int.shiftRightUnsigned(12)
+  toBase36(low, ~length=6) ++ toBase36(high, ~length=4)
+}
+
+let directionSuffix = (direction: Table.indexFieldDirection) =>
+  switch direction {
+  | Asc => ""
+  | Desc => "_desc"
+  }
+
+// `<Entity>_<column>`, with each further column appended in order. Only there so
+// a human reading `\d` output can tell what the index is for.
+let readablePrefix = ({tableName, columns}) =>
+  tableName ++
+  "_" ++
+  columns
+  ->Array.map(({name, direction}) => name ++ directionSuffix(direction))
+  ->Array.joinUnsafe("_")
+
+let pgMaxIdentifierLength = 63
+let maxPrefixLength = pgMaxIdentifierLength - hashLength - 1
+
+// Postgres truncates identifiers past 63 bytes on its own, and two long field
+// names would then collapse onto one name. Truncating only the readable half
+// and keeping the hash whole makes every generated name distinct by
+// construction. Prefixes are ASCII (GraphQL names are), so characters and
+// Postgres' byte limit line up.
+let name = definition => {
+  let prefix = definition->readablePrefix
+  let prefix =
+    prefix->String.length > maxPrefixLength
+      ? prefix->String.slice(~start=0, ~end=maxPrefixLength)
+      : prefix
+  `${prefix}_${definition->identityHash}`
+}
+
+let directionSql = (direction: Table.indexFieldDirection) =>
+  switch direction {
+  | Asc => ""
+  | Desc => " DESC"
+  }
+
+let columnsSql = ({columns}) =>
+  columns
+  ->Array.map(({name, direction}) => `"${name}"${directionSql(direction)}`)
+  ->Array.joinUnsafe(", ")
+
+// Plain DDL, not CONCURRENTLY: it builds from a single table scan instead of
+// two, and the SHARE lock it takes blocks writes but not reads, so queries keep
+// being served while it runs. The indexer is the only writer, and every caller
+// is happy to wait on it.
+//
+// No `IF NOT EXISTS` either — a skipped create is indistinguishable from a
+// successful one, and the whole point of the generated name is that nothing
+// else can hold it.
+let makeCreateQuery = (definition, ~pgSchema) => {
+  let using = definition.method === btree ? "" : ` USING ${definition.method}`
+  `CREATE INDEX "${definition->name}" ON "${pgSchema}"."${definition.tableName}"${using}(${definition->columnsSql});`
+}
+
+let makeDropQuery = (~pgSchema, ~indexName) => `DROP INDEX IF EXISTS "${pgSchema}"."${indexName}";`

package/src/db/IndexDefinition.res.mjs

@@ -0,0 +1,138 @@
+// Generated by ReScript, PLEASE EDIT WITH CARE
+
+
+let btree = "btree";
+
+function make(tableName, columns, methodOpt) {
+  let method = methodOpt !== undefined ? methodOpt : btree;
+  return {
+    tableName: tableName,
+    columns: columns,
+    method: method
+  };
+}
+
+function single(tableName, column) {
+  return make(tableName, [{
+      name: column,
+      direction: "Asc"
+    }], undefined);
+}
+
+function fromIndexFields(tableName, indexFields) {
+  return make(tableName, indexFields.map(param => ({
+    name: param.fieldName,
+    direction: param.direction
+  })), undefined);
+}
+
+function columnKey(param) {
+  let name = param.name;
+  if (param.direction === "Asc") {
+    return name;
+  } else {
+    return name + ` DESC`;
+  }
+}
+
+function key(param) {
+  return param.tableName + `|` + param.method + `|` + param.columns.map(columnKey).join(",");
+}
+
+function describe(definition) {
+  return definition.tableName + `(` + definition.columns.map(columnKey).join(", ") + `) using ` + definition.method;
+}
+
+function fnv1a(input, seed) {
+  let hash = seed;
+  for (let idx = 0, idx_finish = input.length; idx < idx_finish; ++idx) {
+    hash = Math.imul(hash ^ input.charCodeAt(idx), 16777619);
+  }
+  return hash;
+}
+
+function toBase36(value, length) {
+  return value.toString(36).padStart(length, "0");
+}
+
+function identityHash(definition) {
+  let key$1 = key(definition);
+  let low = (fnv1a(key$1, -2128831035) >>> 2);
+  let high = (fnv1a(key$1, -1496811798) >>> 12);
+  return toBase36(low, 6) + toBase36(high, 4);
+}
+
+function directionSuffix(direction) {
+  if (direction === "Asc") {
+    return "";
+  } else {
+    return "_desc";
+  }
+}
+
+function readablePrefix(param) {
+  return param.tableName + "_" + param.columns.map(param => param.name + directionSuffix(param.direction)).join("_");
+}
+
+let maxPrefixLength = 52;
+
+function name(definition) {
+  let prefix = readablePrefix(definition);
+  let prefix$1 = prefix.length > maxPrefixLength ? prefix.slice(0, maxPrefixLength) : prefix;
+  return prefix$1 + `_` + identityHash(definition);
+}
+
+function directionSql(direction) {
+  if (direction === "Asc") {
+    return "";
+  } else {
+    return " DESC";
+  }
+}
+
+function columnsSql(param) {
+  return param.columns.map(param => `"` + param.name + `"` + directionSql(param.direction)).join(", ");
+}
+
+function makeCreateQuery(definition, pgSchema) {
+  let using = definition.method === btree ? "" : ` USING ` + definition.method;
+  return `CREATE INDEX "` + name(definition) + `" ON "` + pgSchema + `"."` + definition.tableName + `"` + using + `(` + columnsSql(definition) + `);`;
+}
+
+function makeDropQuery(pgSchema, indexName) {
+  return `DROP INDEX IF EXISTS "` + pgSchema + `"."` + indexName + `";`;
+}
+
+let fnvOffsetBasis = -2128831035;
+
+let fnvPrime = 16777619;
+
+let hashLength = 10;
+
+let pgMaxIdentifierLength = 63;
+
+export {
+  btree,
+  make,
+  single,
+  fromIndexFields,
+  columnKey,
+  key,
+  describe,
+  fnvOffsetBasis,
+  fnvPrime,
+  fnv1a,
+  hashLength,
+  toBase36,
+  identityHash,
+  directionSuffix,
+  readablePrefix,
+  pgMaxIdentifierLength,
+  maxPrefixLength,
+  name,
+  directionSql,
+  columnsSql,
+  makeCreateQuery,
+  makeDropQuery,
+}
+/* No side effect */

package/src/db/IndexManager.res

@@ -0,0 +1,145 @@
+// Decides what has to happen for a desired index to exist, serialises the work
+// so two builds never fight over the same table, and refuses to record an index
+// as present until Postgres has been asked again what it actually built.
+
+// A build the catalog doesn't already cover. `isRebuild` means an unusable
+// index holds the generated name with exactly our identity, so it can only be
+// one of ours — dropped and built again rather than left to block the name.
+type prepared = {
+  definition: IndexDefinition.t,
+  name: string,
+  isRebuild: bool,
+  queries: array<string>,
+}
+
+type t = {
+  mutable catalog: IndexCatalog.t,
+  // Keyed by index identity and coverage so concurrent identical requests
+  // await one build.
+  inflight: dict<promise<unit>>,
+  // Tail of the build chain per table, so two different indexes on the same
+  // table are built one after the other rather than at once.
+  tableQueue: dict<promise<unit>>,
+}
+
+let make = () => {
+  catalog: IndexCatalog.make(),
+  inflight: Dict.make(),
+  tableQueue: Dict.make(),
+}
+
+let reload = (manager, ~rows) => {
+  manager.catalog = IndexCatalog.fromRows(~rows)
+  manager.catalog
+}
+
+let catalog = manager => manager.catalog
+
+let isSatisfied = (manager, definition, ~coverage) =>
+  manager.catalog->IndexCatalog.find(definition, ~coverage)->Option.isSome
+
+// The name is derived from a hash of the full identity, so an index holding it
+// was generated for this exact identity. Anything else under that name is
+// someone else's, and guessing which is worse than stopping.
+let conflictError = (~entry: IndexCatalog.entry, ~definition, ~pgSchema) =>
+  Utils.Error.make(
+    `Cannot create the index "${entry.name}" in schema "${pgSchema}" for ${definition->IndexDefinition.describe}. A different index already holds that name and the indexer can't safely replace it: ${entry
+      ->IndexCatalog.rejectReason(definition, ~coverage=IndexCatalog.Exact)
+      ->Option.getOr("it is unique")}. Drop that index by hand, then restart the indexer.`,
+  )
+
+let verificationError = (~name, ~definition, ~pgSchema, ~reason) =>
+  Utils.Error.make(
+    `The index "${name}" in schema "${pgSchema}" is not usable after its DDL ran: ${reason}. It was meant to cover ${definition->IndexDefinition.describe}. Drop it by hand, then restart the indexer.`,
+  )
+
+// `None` when the catalog already covers the definition. Resolving this before
+// any DDL runs means a name held by an unrelated index fails while nothing is
+// half-built.
+let prepare = (manager, ~definition, ~coverage, ~pgSchema): option<prepared> =>
+  switch manager.catalog->IndexCatalog.find(definition, ~coverage) {
+  | Some(_) => None
+  | None =>
+    let name = definition->IndexDefinition.name
+    let create = definition->IndexDefinition.makeCreateQuery(~pgSchema)
+    switch manager.catalog->IndexCatalog.getByName(name) {
+    | None => Some({definition, name, isRebuild: false, queries: [create]})
+    | Some(entry) if entry->IndexCatalog.isExactly(definition) =>
+      Some({
+        definition,
+        name,
+        isRebuild: true,
+        queries: [IndexDefinition.makeDropQuery(~pgSchema, ~indexName=name), create],
+      })
+    | Some(entry) => throw(conflictError(~entry, ~definition, ~pgSchema))
+    }
+  }
+
+// The DDL succeeding is not proof the index exists in a form the planner will
+// use: a build can leave an index INVALID, and a name we assumed free would
+// have made it a no-op. `rows` is a fresh read of that one index from
+// pg_catalog — inside the same transaction when there is one, so a failure
+// rolls the DDL back with it.
+let verifyOrThrow = (prepared, ~rows, ~pgSchema): IndexCatalog.entry => {
+  let {definition, name} = prepared
+  switch rows->Array.map(IndexCatalog.fromRow)->Array.find(entry => entry.name === name) {
+  | None =>
+    throw(verificationError(~name, ~definition, ~pgSchema, ~reason="PostgreSQL has no such index"))
+  | Some(entry) =>
+    // The build is verified against `Exact` whatever the request asked for: we
+    // just created this index from the definition, so anything short of a
+    // byte-for-byte match means Postgres built something else.
+    switch entry->IndexCatalog.rejectReason(definition, ~coverage=IndexCatalog.Exact) {
+    | Some(reason) => throw(verificationError(~name, ~definition, ~pgSchema, ~reason))
+    | None => entry
+    }
+  }
+}
+
+// Applied only once the DDL is durable — inside a transaction that means after
+// the commit, so a rollback can't leave the catalog claiming an index the
+// schema doesn't have.
+let record = (manager, entry) => manager.catalog->IndexCatalog.set(entry)
+
+// Puts one index back in step with the database. A build that commits its DDL
+// and then fails to read it back would otherwise leave the catalog claiming the
+// name is free, and every later attempt would replan a create and hit
+// "relation already exists" for the rest of the run.
+let resync = (manager, ~name, ~rows) =>
+  switch rows->Array.map(IndexCatalog.fromRow)->Array.find(entry => entry.name === name) {
+  | Some(entry) => manager.catalog->IndexCatalog.set(entry)
+  | None => manager.catalog->IndexCatalog.remove(name)
+  }
+
+// Runs `build` unless the catalog already covers the definition. Nothing is
+// recorded here: `build` owns the verification, so a failed or unverified build
+// leaves the manager untouched and the next request retries.
+let ensure = (manager, ~definition: IndexDefinition.t, ~coverage, ~build) =>
+  if manager->isSatisfied(definition, ~coverage) {
+    Promise.resolve()
+  } else {
+    // Keyed by coverage as well as identity: an `Exact` request joining an
+    // in-flight `LeadingColumns` one would resolve as soon as that build
+    // decided a leading composite already served it, and the declared index
+    // would never be created.
+    let key = `${coverage->IndexCatalog.coverageKey}${definition->IndexDefinition.key}`
+    switch manager.inflight->Utils.Dict.dangerouslyGetNonOption(key) {
+    | Some(promise) => promise
+    | None =>
+      let tail = switch manager.tableQueue->Utils.Dict.dangerouslyGetNonOption(
+        definition.tableName,
+      ) {
+      | Some(tail) => tail
+      | None => Promise.resolve()
+      }
+      let promise = tail
+      // The queued build may have been satisfied while waiting behind another
+      // one on the same table (eg a finalize pass created it).
+      ->Promise.then(() =>
+        manager->isSatisfied(definition, ~coverage) ? Promise.resolve() : build()
+      )
+      manager.inflight->Dict.set(key, promise)
+      manager.tableQueue->Dict.set(definition.tableName, promise->Utils.Promise.silentCatch)
+      promise->Promise.finally(() => manager.inflight->Utils.Dict.deleteInPlace(key))
+    }
+  }