@capacms/mcp 0.2.0

package/lib/explore.mjs

@@ -0,0 +1,471 @@
+/**
+ * explore.mjs — what a model's content actually looks like, measured.
+ *
+ * An agent asked to filter by category needs the categories that exist, not
+ * the field's type; one asked to fix a page needs to know that `views` is null
+ * on half the entries. This module reads up to a few hundred entries through
+ * GraphQL and reduces them to per-field statistics small enough for a context
+ * window: counts, ranges, the distinct values of an enum-like field, and how
+ * many related entries each entry points at.
+ *
+ * TWO READS OF EACH PAGE, BECAUSE A FILTER AND A QUERY SEE DIFFERENT THINGS.
+ * A filter (`{ null: true }`, `{ exists: true }`) tests what is stored. A
+ * query returns what resolves: a reference to a deleted or draft-only entry is
+ * null, and so is a value that does not fit its field's type (`"42"` in a
+ * number field). Counted from the query alone, both read as missing and the
+ * agent's next filter disagrees with what it was told. So each page GraphQL
+ * returned is read again over REST, relations unexpanded, for the same ids
+ * (`storedRequest`): every count comes from what is stored, and the two reads
+ * side by side say which references dangle and which values do not fit.
+ *
+ * REST passes a scalar through as stored, but it renders a media value and a
+ * reference, and shows as null what it cannot render: text in an image field,
+ * a media list that is not a list, a reference that is not an id. Only the
+ * API's filters still see such a value. So for those fields, the entries of a
+ * page that REST shows as null are read once more with `{ exists: true }`
+ * (`hiddenValueRequests`), and each one it matches stores a value of the
+ * wrong type.
+ *
+ * A relation list is rendered too, and REST renders one stored as null, as
+ * `[]`, or as anything that is not a list, all as an empty list. Two filters
+ * tell them apart, so the entries of a page whose list REST shows empty are
+ * read twice more: with `{ null: true }` (`nullListRequests`), which matches
+ * the missing ones, and with `not` of `has` an id no entry has
+ * (`listRequests`), which matches the ones stored as a list: a filter is
+ * three-valued, and `not` keeps a value that is not a list unknown
+ * (docs/api/entries.md). Those are empty, and the rest store something that
+ * is not a list, present and mistyped.
+ *
+ * The query is planned inside the API's limits. Its cost (5,000 per request)
+ * counts each entry once, each single relation once more, and each array
+ * relation FAN_OUT_CAP times, since it is read as at most that many ids, plus
+ * COUNT_COST for its totalCount; the page size shrinks to fit. At most
+ * MAX_EXPANSIONS relation fields can be expanded per query, so a model with
+ * more is measured on the first ones and the rest are reported as skipped.
+ */
+import { isNameable, writeName } from "./graphql/names.mjs";
+
+export const FAN_OUT_CAP = 20;
+/** REST's relation items per request, which GraphQL applies per root field. */
+export const MAX_EXPANSIONS = 12;
+const NODE_BUDGET = 5000;
+/** A totalCount's cost: it reads entries the answer does not hold (spec 17, amendment 43). */
+const COUNT_COST = 500;
+const PAGE_MAX = 200;
+const TOP_VALUES = 10;
+const VALUE_CHARS = 60;
+const SAMPLE_CHARS = 120;
+
+const clip = (value, chars) => (typeof value === "string" && value.length > chars ? `${value.slice(0, chars)}...` : value);
+
+const isExpansion = (field) => field.kind === "relation" || field.kind === "relationList";
+
+/** Nodes one entry can cost: itself, one per single relation, FAN_OUT_CAP per array relation. */
+export function nodesPerEntry(fields) {
+  const singles = fields.filter((f) => f.kind === "relation").length;
+  const lists = fields.filter((f) => f.kind === "relationList").length;
+  return 1 + singles + lists * FAN_OUT_CAP;
+}
+
+/** Entries per request that keeps `fields`, and the query's totalCount, inside the API's budget. */
+export function pageSizeFor(fields) {
+  return Math.max(1, Math.min(PAGE_MAX, Math.floor((NODE_BUDGET - COUNT_COST) / nodesPerEntry(fields))));
+}
+
+/**
+ * `fields` split into what one query can measure and what it cannot: every
+ * field up to MAX_EXPANSIONS relations, in schema order, and the relations
+ * past that.
+ */
+export function measurableFields(fields) {
+  const measured = [];
+  const skipped = [];
+  let expansions = 0;
+  for (const field of fields) {
+    if (isExpansion(field) && expansions++ >= MAX_EXPANSIONS) skipped.push(field);
+    else measured.push(field);
+  }
+  return { measured, skipped };
+}
+
+function selectionFor(field) {
+  switch (field.kind) {
+    case "relation":
+      return `${field.name} { id }`;
+    case "relationList":
+      return `${field.name}(first: ${FAN_OUT_CAP}) { nodes { id } pageInfo { hasNextPage } }`;
+    case "media":
+      return `${field.name} { id type }`;
+    default:
+      return field.name;
+  }
+}
+
+/**
+ * A namespace REST's grammars can name: every one but a namespace starting
+ * with `$`, which only `select=*` returns (names.mjs). One holding a
+ * character the grammars use is written quoted (`"price.usd"`). A field REST
+ * cannot name is measured on what GraphQL returns.
+ */
+export const restNameable = isNameable;
+
+/**
+ * The REST read of one page's stored values: the fields by namespace,
+ * relations unexpanded so a reference comes back whether or not its entry
+ * still resolves, for exactly the ids GraphQL returned (`$id` is the system
+ * id even beside a field called `id`). Null when there is nothing to read.
+ */
+export function storedRequest(model, fields, ids) {
+  const names = [...new Set(fields.map((f) => f.namespace).filter(restNameable))];
+  if (!names.length || !ids.length) return null;
+  return {
+    path: `/api/entries/${encodeURIComponent(model.namespace)}`,
+    query: { select: names.map(writeName).join(","), where: JSON.stringify({ $id: { in: ids } }), limit: ids.length },
+  };
+}
+
+/**
+ * Field kinds REST renders rather than passes through, so a stored value of
+ * the wrong type reads as null over REST (this file's header). A relation
+ * list is not one: REST reads a list it cannot render as empty, as it reads a
+ * list stored empty or not at all (`nullListRequests`).
+ */
+const RENDERED_KINDS = new Set(["media", "relation", "id"]);
+const LIST_KINDS = new Set(["relationList", "idList"]);
+
+/**
+ * The REST reads that find stored values REST shows as null: for each field
+ * REST renders, the entries in `ids` whose stored row reads as null there,
+ * filtered with `{ exists: true }`, which tests what is stored. One read per
+ * field with such entries; `namespace` names the field each one is for.
+ */
+export function hiddenValueRequests(model, fields, storedById, ids) {
+  const requests = [];
+  for (const field of fields) {
+    if (!RENDERED_KINDS.has(field.kind) || !restNameable(field.namespace)) continue;
+    const shownNull = ids.filter((id) => {
+      const row = storedById.get(id);
+      return row !== undefined && Object.hasOwn(row, field.namespace) && isNull(row[field.namespace]);
+    });
+    if (!shownNull.length) continue;
+    requests.push({
+      namespace: field.namespace,
+      path: `/api/entries/${encodeURIComponent(model.namespace)}`,
+      query: {
+        select: writeName(field.namespace),
+        where: JSON.stringify({ $id: { in: shownNull }, [writeName(field.namespace)]: { exists: true } }),
+        limit: shownNull.length,
+      },
+    });
+  }
+  return requests;
+}
+
+/** Whether REST shows an entry's relation list as holding no id. */
+function shownEmpty(storedById, id, namespace) {
+  const row = storedById.get(id);
+  return row !== undefined && Object.hasOwn(row, namespace) && (idsOf(row[namespace]) ?? []).length === 0;
+}
+
+/**
+ * For each relation list, one REST read of the entries in `ids` whose stored
+ * row holds no id, filtered by `condition(name)` beside `$id`, where `name`
+ * is the field's namespace as REST writes it. Each request's `namespace`
+ * names the field it is for.
+ */
+function emptyListRequests(model, fields, storedById, ids, condition) {
+  const requests = [];
+  for (const field of fields) {
+    if (!LIST_KINDS.has(field.kind) || !restNameable(field.namespace)) continue;
+    const shown = ids.filter((id) => shownEmpty(storedById, id, field.namespace));
+    if (!shown.length) continue;
+    requests.push({
+      namespace: field.namespace,
+      path: `/api/entries/${encodeURIComponent(model.namespace)}`,
+      query: { select: writeName(field.namespace), where: JSON.stringify({ $id: { in: shown }, ...condition(writeName(field.namespace)) }), limit: shown.length },
+    });
+  }
+  return requests;
+}
+
+/**
+ * The REST reads that find relation lists stored as null among those REST
+ * shows empty, with `{ null: true }`, which tests what is stored.
+ */
+export function nullListRequests(model, fields, storedById, ids) {
+  return emptyListRequests(model, fields, storedById, ids, (namespace) => ({ [namespace]: { null: true } }));
+}
+
+/**
+ * The id `listRequests` asks a list not to hold. Any UUID would do, as a
+ * relation list filter wants one: a list REST shows empty holds no id in
+ * either form `has` reads.
+ */
+export const NO_ENTRY_ID = "00000000-0000-4000-8000-000000000000";
+
+/**
+ * The REST reads that find relation lists stored as a list among those REST
+ * shows empty, with `not` of `has` NO_ENTRY_ID: every stored list passes it,
+ * and `not` keeps a value that is not a list, or none, unknown, so neither
+ * does (this file's header).
+ */
+export function listRequests(model, fields, storedById, ids) {
+  return emptyListRequests(model, fields, storedById, ids, (namespace) => ({ not: { [namespace]: { has: NO_ENTRY_ID } } }));
+}
+
+/** The exploration query for `fields` of `model`, paged by `$after`. `status` is always the system field (N5 renames a field called status). */
+export function exploreQuery(model, fields, first) {
+  const selections = ["id", "status", ...fields.map(selectionFor)].join(" ");
+  return (
+    `query CapaExplore($after: String) { ${model.listField}(first: ${first}, after: $after) ` +
+    `{ nodes { ${selections} } pageInfo { hasNextPage endCursor } totalCount } }`
+  );
+}
+
+function scalarKind(field) {
+  const base = field.graphqlType.replace(/[[\]!]/g, "");
+  if (field.graphqlType.startsWith("[")) return base === "Float" || base === "Int" ? "numberList" : "list";
+  if (base === "Float" || base === "Int") return "number";
+  if (base === "Boolean") return "boolean";
+  if (base === "DateTime") return "date";
+  if (base === "String") return "string";
+  return "other";
+}
+
+function countTop(values, limit) {
+  const counts = new Map();
+  for (const v of values) counts.set(v, (counts.get(v) ?? 0) + 1);
+  const sorted = [...counts].sort((a, b) => b[1] - a[1] || String(a[0]).localeCompare(String(b[0])));
+  return { distinct: counts.size, top: sorted.slice(0, limit).map(([value, count]) => [clip(value, VALUE_CHARS), count]) };
+}
+
+const round = (n) => Math.round(n * 100) / 100;
+
+/**
+ * ISO 8601 as a stored date can be written: a four-digit or signed six-digit
+ * year, an optional time with any number of fraction digits, and `Z` or an
+ * offset (`+14:00`, `-0800`, `+05`). A date alone is midnight UTC.
+ */
+const ISO_INSTANT =
+  /^([+-]\d{6}|\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2})(?::(\d{2})(?:[.,](\d+))?)?(Z|[+-]\d{2}(?::?\d{2})?)?)?$/i;
+
+/**
+ * A date string as the instant it names, comparable across offsets: whole
+ * seconds since the epoch, then the fraction as nine digits. Text order gets
+ * both wrong: `9999-12-31T23:59:59.999-14:00` is in year 10000 UTC, after
+ * `9999-12-31T23:59:59.999Z`, and `2024-01-01T00:00:00+14:00` is before
+ * `2023-12-31T12:00:00Z`.
+ * Null for text that is not such a date.
+ */
+export function instantOf(text) {
+  const match = typeof text === "string" ? ISO_INSTANT.exec(text.trim()) : null;
+  if (!match) return null;
+  const [, year, month, day, hour = "0", minute = "0", second = "0", fraction = "", zone = "Z"] = match;
+  const date = new Date(0);
+  date.setUTCFullYear(Number(year), Number(month) - 1, Number(day));
+  if (date.getUTCMonth() !== Number(month) - 1 || date.getUTCDate() !== Number(day)) return null;
+  date.setUTCHours(Number(hour), Number(minute), Number(second), 0);
+  let offsetMinutes = 0;
+  if (zone.toUpperCase() !== "Z") {
+    const digits = zone.slice(1).replace(":", "");
+    offsetMinutes = (zone[0] === "-" ? -1 : 1) * (Number(digits.slice(0, 2)) * 60 + Number(digits.slice(2) || 0));
+  }
+  const seconds = date.getTime() / 1000 - offsetMinutes * 60;
+  return Number.isFinite(seconds) ? { seconds, fraction: fraction.padEnd(9, "0").slice(0, 9) } : null;
+}
+
+function compareInstants(a, b) {
+  return a.seconds - b.seconds || (a.fraction < b.fraction ? -1 : a.fraction > b.fraction ? 1 : 0);
+}
+
+/** The earliest and latest of `dates` by instant, each as written; text that is no date is left out. */
+export function dateRange(dates) {
+  const instants = dates.map((text) => ({ text, at: instantOf(text) })).filter((d) => d.at !== null);
+  if (!instants.length) return {};
+  instants.sort((a, b) => compareInstants(a.at, b.at) || (a.text < b.text ? -1 : a.text > b.text ? 1 : 0));
+  return { earliest: instants[0].text, latest: instants[instants.length - 1].text };
+}
+
+function numberStats(numbers) {
+  if (!numbers.length) return {};
+  const sorted = [...numbers].sort((a, b) => a - b);
+  const mid = sorted.length >> 1;
+  return {
+    min: sorted[0],
+    max: sorted[sorted.length - 1],
+    mean: round(numbers.reduce((a, b) => a + b, 0) / numbers.length),
+    median: sorted.length % 2 ? sorted[mid] : round((sorted[mid - 1] + sorted[mid]) / 2),
+  };
+}
+
+/** What the counts of every field mean, once per answer. */
+export const COUNTS_LEGEND =
+  "Counts are of what is stored, so they agree with filters. present: has a value. empty: blank text or an empty list. " +
+  "missing: null or absent, what { null: true } matches; { exists: true } matches present and empty. The three add up to scanned. " +
+  "A relation list stored as something other than a list is present and mistyped. dangling: stored references a query returns nothing for " +
+  "(deleted, draft only, another model). mistyped: stored values that do not fit the field's type, which a query returns as null. " +
+  "items and fanOut count the ids stored.";
+
+const isNull = (value) => value === null || value === undefined;
+
+/** The ids of a relation list or id list, as stored (REST `{ items }`) or as served (a connection's `nodes`, or ids). */
+function idsOf(value) {
+  if (Array.isArray(value)) return value.map((item) => (typeof item === "string" ? item : item?.id));
+  if (Array.isArray(value?.items)) return value.items.map((item) => item.id);
+  if (Array.isArray(value?.nodes)) return value.nodes.map((node) => node.id);
+  return null;
+}
+
+/** Blank text, an empty list, or a relation list with no ids: a value that holds nothing. */
+function isEmpty(value) {
+  if (typeof value === "string") return value.trim() === "";
+  if (Array.isArray(value)) return value.length === 0;
+  if (Array.isArray(value?.items)) return value.items.length === 0;
+  if (value && typeof value === "object" && Array.isArray(value.nodes)) return value.nodes.length === 0 && value.pageInfo?.hasNextPage !== true;
+  return false;
+}
+
+/** The id a single reference holds: REST's `{ id, model }`, a served entry, or a bare id. */
+const idOf = (value) => (typeof value === "string" ? value : value?.id);
+
+/** A stored value a query returns as null, or a list some of whose stored items it returns as null. */
+function mistyped({ stored, served }) {
+  if (isNull(served)) return true;
+  return Array.isArray(stored) && Array.isArray(served) && served.some((item, i) => isNull(item) && !isNull(stored[i]));
+}
+
+/** Text as a query returns it: a number or true/false stored in a text field is still text. */
+const asText = (value) => (typeof value === "string" ? value : typeof value === "number" || typeof value === "boolean" ? String(value) : null);
+
+/**
+ * Statistics for one field over the scanned entries. Pure.
+ *
+ * `nodes` are the entries as GraphQL returned them, and `storedById` each
+ * one's fields as REST says they are stored, by namespace. Every count comes
+ * from the stored value, so it agrees with a filter; the served value beside
+ * it says which references resolve (`dangling`) and which values fit their
+ * type (`mistyped`). `hiddenIds` are the entries whose value REST shows as
+ * null although `{ exists: true }` matches it (`hiddenValueRequests`): each
+ * stores a value of the wrong type, counted present and mistyped, since what
+ * it holds cannot be read. `nullIds` are the entries whose relation list REST
+ * shows empty and `{ null: true }` matches (`nullListRequests`): missing. An
+ * entry or field with no stored value read is measured on what GraphQL
+ * returned. `listIds`, when given, are the entries whose relation list REST
+ * shows empty that store a list (`listRequests`): any other entry REST shows
+ * empty and `{ null: true }` does not match stores something that is not a
+ * list, which is hidden like a wrong-typed media value. `present`, `empty`
+ * and `missing` are COUNTS_LEGEND's.
+ */
+export function fieldStats(field, nodes, storedById = new Map(), hiddenIds = new Set(), nullIds = new Set(), listIds = null) {
+  const notAList = (id) => listIds !== null && LIST_KINDS.has(field.kind) && shownEmpty(storedById, id, field.namespace) && !nullIds.has(id) && !listIds.has(id);
+  const entries = nodes.map((node) => {
+    const served = node[field.name] ?? null;
+    const row = storedById.get(node.id);
+    const read = row && Object.hasOwn(row, field.namespace) ? row[field.namespace] ?? null : served;
+    const stored = nullIds.has(node.id) ? null : read;
+    return { stored, served, hidden: (isNull(stored) && hiddenIds.has(node.id)) || notAList(node.id) };
+  });
+  const given = entries.filter((e) => e.hidden || !isNull(e.stored));
+  const present = given.filter((e) => e.hidden || !isEmpty(e.stored));
+  // The values REST can show; a hidden one is only known to be there.
+  const readable = present.filter((e) => !e.hidden);
+  const withHidden = (out) => (present.length > readable.length ? { ...out, mistyped: present.length - readable.length } : out);
+  const stats = {
+    field: field.name,
+    type: field.arrayType ? `${field.capaType} of ${field.arrayType}` : field.capaType,
+    present: present.length,
+    empty: given.length - present.length,
+    missing: entries.length - given.length,
+  };
+  if (field.namespace !== field.name) stats.namespace = field.namespace;
+
+  switch (field.kind) {
+    case "relation":
+      return withHidden({ ...stats, target: field.target, dangling: readable.filter((e) => isNull(e.served)).length, ...countTop(readable.map((e) => idOf(e.stored)), 5) });
+    case "id":
+      // A target GraphQL cannot expand (unreadable, or left out of GraphQL) is still named, as for a list of ids.
+      return withHidden({ ...stats, target: field.target ?? undefined, ...countTop(readable.map((e) => idOf(e.stored)), 5) });
+    case "relationList":
+    case "idList": {
+      // A list stored as something else holds no id, so it points at nothing and dangles nowhere.
+      const lists = given.map((e) => ({ ids: e.hidden ? [] : idsOf(e.stored) ?? [], served: e.served }));
+      // fanOut is how many entries each scanned entry points at, so a list stored as null (missing) counts as 0.
+      const fanOut = numberStats(entries.map((e) => (e.hidden ? [] : idsOf(e.stored) ?? []).length));
+      const out = { ...stats, target: field.target ?? undefined, items: lists.reduce((sum, l) => sum + l.ids.length, 0), fanOut };
+      if (field.kind === "idList") return withHidden(out);
+      // A query reads the first FAN_OUT_CAP stored ids of each entry and leaves out those that do not resolve.
+      out.dangling = lists.reduce((sum, l) => sum + Math.min(l.ids.length, FAN_OUT_CAP) - (l.served?.nodes?.length ?? 0), 0);
+      const longer = lists.filter((l) => l.ids.length > FAN_OUT_CAP).length;
+      if (longer) out.danglingChecked = `the first ${FAN_OUT_CAP} ids of each entry; ${longer} store more`;
+      return withHidden(out);
+    }
+    case "media": {
+      // The kinds of file across every item: a list of media is an array of { id, type }.
+      const items = readable.flatMap((e) => (Array.isArray(e.stored) ? e.stored : [e.stored])).filter((v) => !isNull(v));
+      const out = { ...stats, mediaTypes: countTop(items.map((v) => v.type ?? "unknown"), TOP_VALUES).top };
+      if (!field.arrayType) return withHidden(out);
+      return withHidden({ ...out, items: items.length, perEntry: numberStats(given.map((e) => e.stored).filter(Array.isArray).map((l) => l.length)) });
+    }
+    case "json":
+      return stats;
+    default:
+      break;
+  }
+
+  const wrong = present.filter(mistyped).length;
+  const withMistyped = (out) => (wrong ? { ...out, mistyped: wrong } : out);
+  const values = present.map((e) => e.stored);
+  switch (scalarKind(field)) {
+    case "number":
+      return withMistyped({ ...stats, ...numberStats(values.filter((v) => typeof v === "number" && Number.isFinite(v))) });
+    case "boolean":
+      return withMistyped({ ...stats, true: values.filter((v) => v === true).length, false: values.filter((v) => v === false).length });
+    case "date":
+      return withMistyped({ ...stats, ...dateRange(values) });
+    case "string": {
+      const strings = values.map(asText).filter((v) => v !== null);
+      const { distinct, top } = countTop(strings, TOP_VALUES);
+      const lengths = strings.map((s) => s.length);
+      const out = { ...stats, distinct };
+      const enumLike = field.capaType === "enum" || (distinct <= TOP_VALUES * 2 && distinct <= Math.max(1, strings.length / 2));
+      if (enumLike) out.values = top;
+      else out.examples = top.slice(0, 3).map(([value]) => value);
+      if (lengths.length) out.length = { min: Math.min(...lengths), max: Math.max(...lengths) };
+      return withMistyped(out);
+    }
+    case "list":
+    case "numberList": {
+      const arrays = given.map((e) => e.stored).filter(Array.isArray);
+      const items = arrays.flat().filter((v) => !isNull(v));
+      const { distinct, top } = countTop(items, TOP_VALUES);
+      return withMistyped({ ...stats, items: items.length, perEntry: numberStats(arrays.map((l) => l.length)), distinct, values: top });
+    }
+    default:
+      return withMistyped(stats);
+  }
+}
+
+/** How many of the scanned entries are in each status, in the order an entry moves through them. */
+export function statusCounts(nodes) {
+  const counts = {};
+  for (const status of ["published", "changed", "draft"]) {
+    const count = nodes.filter((node) => node.status === status).length;
+    if (count) counts[status] = count;
+  }
+  return counts;
+}
+
+/** The first `count` entries, long strings clipped, relations as ids. */
+export function samples(nodes, fields, count) {
+  return nodes.slice(0, count).map((node) => {
+    const out = { id: node.id };
+    for (const field of fields) {
+      const value = node[field.name];
+      if (field.kind === "relation") out[field.name] = value?.id ?? null;
+      else if (field.kind === "relationList") out[field.name] = value?.nodes?.map((n) => n.id) ?? [];
+      else if (Array.isArray(value)) out[field.name] = value.map((v) => clip(v, SAMPLE_CHARS));
+      else out[field.name] = clip(value, SAMPLE_CHARS);
+    }
+    return out;
+  });
+}