@shortlink-org/portolan 0.2.4 → 0.4.0

package/src/lib/problem-rules-cel.mjs

@@ -0,0 +1,407 @@
+// The CEL half of a problem rule: what an expression may read, and whether
+// it reads it correctly.
+//
+// Plain JavaScript on purpose, like django-aggregates.mjs: the manifest is
+// checked by scripts/manifest.mjs under Node, where nothing strips types, and
+// the same expression runs in the browser over the merged catalog. One file
+// says what `event.consumers` is so that the two cannot disagree. Types live
+// in problem-rules-cel.d.mts.
+//
+// A rule is written over ONE subject and reads a flat view of it: strings,
+// ints, bools and lists of strings. Everything a rule would have to join -
+// who else writes the table, who publishes on the channel, whether a call's
+// peer is in the estate - is a field on the row, computed once by
+// problem-subjects.ts. The rule decides; the projection only describes
+// (portolan.0017).
+
+import { Environment } from "@marcbachmann/cel-js";
+
+/**
+ * What each subject shows to an expression. The field list is the contract a
+ * rule is type-checked against: `event.nme` is refused when the manifest is
+ * read, not discovered as an empty page.
+ */
+export const SUBJECTS = {
+  service: {
+    description: "One service of the estate, with counts of what it declares.",
+    schema: {
+      id: "string",
+      slug: "string",
+      name: "string",
+      context: "string",
+      kind: "string",
+      repo: "string",
+      path: "string",
+      technologies: "list<string>",
+      owners: "list<string>",
+      provides: "int",
+      methods: "int",
+      calls: "int",
+      unresolvedCalls: "int",
+      aggregates: "int",
+      events: "int",
+      stores: "list<string>",
+      channels: "list<string>",
+      hosts: "list<string>",
+      dials: "list<string>",
+    },
+  },
+  call: {
+    description: "One call a service makes on an interface, and whether the estate answers it.",
+    schema: {
+      id: "string",
+      interface: "string",
+      method: "string",
+      peer: "string",
+      status: "string",
+      resolved: "bool",
+      peerKnown: "bool",
+      methodDeclared: "bool",
+      service: "string",
+      context: "string",
+      source: "string",
+      module: "string",
+      note: "string",
+    },
+  },
+  copy: {
+    description: "One vendored copy of an interface, held against what the provider publishes.",
+    schema: {
+      id: "string",
+      service: "string",
+      context: "string",
+      provider: "string",
+      methods: "int",
+      differences: "list<string>",
+      source: "string",
+    },
+  },
+  event: {
+    description: "One domain event, as its owning aggregate declares it.",
+    schema: {
+      id: "string",
+      slug: "string",
+      name: "string",
+      aggregate: "string",
+      service: "string",
+      context: "string",
+      versions: "int",
+      deprecated: "bool",
+      consumers: "list<string>",
+      unresolvedConsumers: "int",
+      wireName: "string",
+      channel: "string",
+      declaredChannels: "list<string>",
+      fields: "list<string>",
+    },
+  },
+  consumer: {
+    description: "One consumer an event names, from the publisher's side.",
+    schema: {
+      event: "string",
+      name: "string",
+      aggregate: "string",
+      owner: "string",
+      context: "string",
+      consumer: "string",
+      status: "string",
+      resolved: "bool",
+      note: "string",
+    },
+  },
+  channel: {
+    description: "One channel a service touches, declared or named by its events, and who else publishes there.",
+    schema: {
+      address: "string",
+      kind: "string",
+      title: "string",
+      service: "string",
+      context: "string",
+      declared: "bool",
+      publishes: "bool",
+      sends: "list<string>",
+      receives: "list<string>",
+      events: "list<string>",
+      otherPublishers: "list<string>",
+      otherClaims: "list<string>",
+      source: "string",
+    },
+  },
+  subscription: {
+    description: "One message a service's document says it receives, and who in the estate sends it.",
+    schema: {
+      service: "string",
+      context: "string",
+      channel: "string",
+      kind: "string",
+      name: "string",
+      encoding: "string",
+      published: "bool",
+      publishers: "list<string>",
+      mismatched: "list<string>",
+      mismatchedEncodings: "list<string>",
+      source: "string",
+    },
+  },
+  table: {
+    description: "One table or view of a store, with what the model and the code say of it.",
+    schema: {
+      id: "string",
+      name: "string",
+      kind: "string",
+      store: "string",
+      storeKind: "string",
+      owner: "string",
+      context: "string",
+      role: "string",
+      aggregate: "string",
+      aggregateOwner: "string",
+      aggregateFields: "int",
+      mappedColumns: "int",
+      claimedColumns: "int",
+      block: "string",
+      columns: "list<string>",
+      primaryKey: "list<string>",
+      foreignKeys: "list<string>",
+      foreignReads: "list<string>",
+      hasPayload: "bool",
+      indexes: "int",
+      reads: "int",
+      writes: "int",
+      deletes: "int",
+    },
+  },
+  column: {
+    description: "One column of a table or view, with where its value points and where it came from.",
+    schema: {
+      id: "string",
+      name: "string",
+      relation: "string",
+      kind: "string",
+      store: "string",
+      owner: "string",
+      context: "string",
+      type: "string",
+      nullable: "bool",
+      pk: "bool",
+      fk: "string",
+      fkOwner: "string",
+      from: "list<string>",
+      foreignFrom: "list<string>",
+      foreignOwners: "list<string>",
+      maps: "string",
+      fieldType: "string",
+      typeMatches: "bool",
+    },
+  },
+  deployment: {
+    description: "One Application the deployer manages, and whether the catalog can place it.",
+    schema: {
+      id: "string",
+      name: "string",
+      project: "string",
+      environment: "string",
+      cluster: "string",
+      namespace: "string",
+      repo: "string",
+      path: "string",
+      chart: "string",
+      from: "string",
+      targetRevision: "string",
+      revision: "string",
+      tool: "string",
+      service: "string",
+      labelled: "string",
+      claimed: "bool",
+      context: "string",
+      runsIn: "string",
+      images: "list<string>",
+      basis: "string",
+      drifted: "bool",
+      drift: "string",
+      url: "string",
+    },
+  },
+  flow: {
+    description: "One flow, with how much of it has been seen running.",
+    schema: {
+      id: "string",
+      slug: "string",
+      name: "string",
+      owner: "string",
+      trigger: "string",
+      triggerConfidence: "string",
+      participants: "list<string>",
+      contexts: "list<string>",
+      crossContext: "bool",
+      steps: "int",
+      verifiedSteps: "int",
+      declaredSteps: "int",
+      unresolvedSteps: "int",
+      seenSteps: "int",
+      events: "list<string>",
+      stores: "list<string>",
+      examples: "int",
+      source: "string",
+    },
+  },
+  aggregate: {
+    description: "One aggregate, with what its domain model declares.",
+    schema: {
+      id: "string",
+      slug: "string",
+      name: "string",
+      root: "string",
+      modelGroup: "bool",
+      service: "string",
+      context: "string",
+      entities: "int",
+      valueObjects: "int",
+      enums: "int",
+      commands: "int",
+      queries: "int",
+      exposedOperations: "int",
+      deprecatedOperations: "int",
+      events: "list<string>",
+      states: "list<string>",
+      transitions: "int",
+      tables: "list<string>",
+    },
+  },
+};
+
+/** What every rule sees beside its subject: the estate's names, for `in`. */
+export const ESTATE_SCHEMA = {
+  services: "list<string>",
+  contexts: "list<string>",
+  stores: "list<string>",
+  channels: "list<string>",
+  externals: "list<string>",
+};
+
+export const SUBJECT_NAMES = Object.keys(SUBJECTS);
+
+export const SEVERITIES = ["error", "warning"];
+
+/** A rule id: dotted or dashed lower-case segments, the way a warning rule's is. */
+export const RULE_ID = /^[a-z0-9]+(?:[-.][a-z0-9]+)*$/;
+
+const LIMITS = {
+  maxAstNodes: 512,
+  maxDepth: 32,
+  maxListElements: 64,
+  maxMapEntries: 64,
+  maxCallArguments: 8,
+};
+
+const environments = new Map();
+
+/** One environment per subject, built on first use and kept. */
+export function environmentFor(subject) {
+  const known = environments.get(subject);
+  if (known) return known;
+  const definition = SUBJECTS[subject];
+  if (!definition) throw new Error(`unknown subject "${subject}"; one of ${SUBJECT_NAMES.join(", ")}`);
+  const environment = new Environment({ unlistedVariablesAreDyn: false, limits: LIMITS })
+    .registerVariable({ name: subject, schema: definition.schema })
+    .registerVariable({ name: "estate", schema: ESTATE_SCHEMA });
+  environments.set(subject, environment);
+  return environment;
+}
+
+const compiled = new Map();
+
+/**
+ * Parses and type-checks one expression over one subject, and refuses one
+ * that does not come out as `type`. The error names the field: it is what the
+ * manifest check prints and what the page shows under the text box.
+ */
+export function compileExpression(subject, source, type) {
+  const key = `${subject}\0${type}\0${source}`;
+  const known = compiled.get(key);
+  if (known) return known;
+  let expression;
+  try {
+    expression = environmentFor(subject).parse(source);
+  } catch (cause) {
+    throw new Error(`invalid CEL: ${firstLine(cause)}`);
+  }
+  const checked = expression.check();
+  if (!checked.valid) throw new Error(`invalid CEL: ${firstLine(checked.error ?? "type check failed")}`);
+  if (checked.type !== type) throw new Error(`CEL expression must return ${type}, got ${checked.type}`);
+  compiled.set(key, expression);
+  return expression;
+}
+
+/**
+ * Everything wrong with one rule entry as written, as lines. Empty when the
+ * entry is good. `builtinIds` are the rules the package ships: an entry
+ * naming one may only switch it, re-grade it and say why; any other id must
+ * carry a whole rule.
+ */
+export function problemRuleProblems(entries, builtinIds, path = "portolan.json") {
+  const problems = [];
+  if (entries === undefined) return problems;
+  if (!Array.isArray(entries)) return [`${path} problemRules: must be an array`];
+  const builtin = new Set(builtinIds);
+  const seen = new Set();
+  entries.forEach((entry, index) => {
+    const at = `${path} problemRules/${index}`;
+    if (!entry || typeof entry !== "object") {
+      problems.push(`${at}: must be an object`);
+      return;
+    }
+    if (typeof entry.id !== "string" || !RULE_ID.test(entry.id)) {
+      problems.push(`${at}/id: must match ${RULE_ID}`);
+      return;
+    }
+    if (seen.has(entry.id)) problems.push(`${at}/id: "${entry.id}" appears twice`);
+    seen.add(entry.id);
+    if (entry.severity !== undefined && !SEVERITIES.includes(entry.severity)) {
+      problems.push(`${at}/severity: must be one of ${SEVERITIES.join(", ")}`);
+    }
+    if (builtin.has(entry.id)) {
+      for (const key of ["over", "when", "message", "peer", "title", "note", "description", "action"]) {
+        if (entry[key] !== undefined) problems.push(`${at}/${key}: "${entry.id}" is a built-in rule; only enabled, severity and reason may be set`);
+      }
+      if (entry.enabled === false && !entry.reason) problems.push(`${at}/reason: a disabled rule needs a reason`);
+      return;
+    }
+    problems.push(...ruleExpressionProblems(entry, at));
+  });
+  return problems;
+}
+
+/** The expressions of one whole rule, checked over its subject; lines for what is wrong. */
+export function ruleExpressionProblems(rule, at = rule.id) {
+  const problems = [];
+  if (!SUBJECT_NAMES.includes(rule.over)) return [`${at}/over: must be one of ${SUBJECT_NAMES.join(", ")}`];
+  if (typeof rule.title !== "string" || !rule.title.trim()) problems.push(`${at}/title: required`);
+  for (const [key, type] of [
+    ["when", "bool"],
+    ["message", "string"],
+  ]) {
+    if (typeof rule[key] !== "string" || !rule[key].trim()) {
+      problems.push(`${at}/${key}: required`);
+      continue;
+    }
+    try {
+      compileExpression(rule.over, rule[key], type);
+    } catch (cause) {
+      problems.push(`${at}/${key}: ${firstLine(cause)}`);
+    }
+  }
+  if (rule.peer !== undefined) {
+    try {
+      compileExpression(rule.over, rule.peer, "string");
+    } catch (cause) {
+      problems.push(`${at}/peer: ${firstLine(cause)}`);
+    }
+  }
+  return problems;
+}
+
+function firstLine(cause) {
+  const message = cause instanceof Error ? cause.message : String(cause);
+  return message.split("\n", 1)[0];
+}

package/src/lib/problem-rules.test.ts

@@ -0,0 +1,287 @@
+// The rules behind the Problems page: every shipped rule compiles over its
+// subject and runs clean on the frozen estate, the manifest's switches and
+// re-grades take effect, and a rule of the estate's own sees what the schema
+// says it sees. What each shipped rule finds is held by the tests beside the
+// subjects they read - rules-data, rules-wire, rules-proto, rules-deploy and rules-edges.
+
+import { describe, expect, it } from "vitest";
+import { rawCatalog } from "../test-catalog";
+import { buildIndex, validateCatalog } from "../catalog";
+import type { Catalog } from "../catalog";
+import { allProblems, evaluateProblems } from "./all-problems";
+import {
+  BUILTIN_RULES,
+  builtinProblems,
+  estateOf,
+  evaluateRules,
+  resolveRules,
+  runRule,
+  subjectsOf,
+  SUBJECTS,
+} from "./problem-rules";
+import type { ProblemRule, ProblemRuleEntry, RuleSubject } from "./problem-rules";
+import { problemRuleProblems, ruleExpressionProblems, SUBJECT_NAMES } from "./problem-rules-cel.mjs";
+
+const catalog = validateCatalog(JSON.parse(JSON.stringify(rawCatalog)) as unknown as Catalog);
+const index = buildIndex(catalog);
+const builtinIds = BUILTIN_RULES.map((rule) => rule.id);
+
+const custom = (entry: Partial<ProblemRuleEntry> & { id: string }): ProblemRule => ({
+  over: "event",
+  severity: "warning",
+  defaultSeverity: "warning",
+  title: entry.id,
+  note: entry.id,
+  description: "",
+  action: "",
+  when: "true",
+  message: "'m'",
+  builtin: false,
+  enabled: true,
+  ...entry,
+});
+
+describe("rules/builtin.json", () => {
+  it("gives every rule a subject, a severity and words", () => {
+    for (const rule of BUILTIN_RULES) {
+      expect(SUBJECT_NAMES, rule.id).toContain(rule.over);
+      expect(["error", "warning"], rule.id).toContain(rule.severity);
+      for (const field of ["title", "note", "description", "action"] as const) {
+        expect(rule[field].trim().length, `${rule.id}.${field}`).toBeGreaterThan(0);
+      }
+    }
+  });
+
+  it("is CEL that type-checks over its subject, rule by rule", () => {
+    for (const rule of BUILTIN_RULES) expect(ruleExpressionProblems(rule), rule.id).toEqual([]);
+  });
+
+  it("runs clean over the sample estate", () => {
+    const { failures, problems } = evaluateRules(catalog, index, resolveRules([]));
+    expect(failures).toEqual([]);
+    expect(problems.length).toBeGreaterThan(0);
+    for (const problem of problems) expect(builtinIds, problem.id).toContain(problem.rule);
+  });
+
+  it("has no two rules with one id", () => {
+    expect(new Set(builtinIds).size).toBe(builtinIds.length);
+  });
+});
+
+describe("resolveRules", () => {
+  it("is every shipped rule, on, without a manifest entry", () => {
+    const rules = resolveRules([]);
+    expect(rules.map((rule) => rule.id)).toEqual(builtinIds);
+    expect(rules.every((rule) => rule.enabled && rule.builtin)).toBe(true);
+  });
+
+  it("switches and re-grades a shipped rule, and keeps the passport's own severity", () => {
+    const rules = resolveRules([
+      { id: "shared-store", enabled: false, reason: "one database by design" },
+      { id: "cross-service-lineage", severity: "error" },
+    ]);
+    const shared = rules.find((rule) => rule.id === "shared-store")!;
+    expect(shared.enabled).toBe(false);
+    expect(shared.reason).toBe("one database by design");
+    const lineage = rules.find((rule) => rule.id === "cross-service-lineage")!;
+    expect(lineage.severity).toBe("error");
+    expect(lineage.defaultSeverity).toBe("warning");
+  });
+
+  it("appends the manifest's own rules after the shipped ones", () => {
+    const rules = resolveRules([{ id: "team.quiet-event", over: "event", when: "true", message: "'x'", title: "Quiet event" }]);
+    const last = rules[rules.length - 1]!;
+    expect(last.id).toBe("team.quiet-event");
+    expect(last.builtin).toBe(false);
+    expect(last.note).toBe("Quiet event");
+    expect(last.severity).toBe("warning");
+  });
+});
+
+describe("evaluateRules", () => {
+  const all = builtinProblems(catalog, index);
+
+  it("drops a switched-off rule's rows and still counts them", () => {
+    const fk = all.filter((problem) => problem.rule === "cross-service-fk");
+    expect(fk.length).toBeGreaterThan(0);
+    const { problems, matches } = evaluateRules(catalog, index, resolveRules([{ id: "cross-service-fk", enabled: false, reason: "by design" }]));
+    expect(problems.some((problem) => problem.rule === "cross-service-fk")).toBe(false);
+    expect(problems.length).toBe(all.length - fk.length);
+    expect(matches.get("cross-service-fk")).toBe(fk.length);
+  });
+
+  it("re-grades a rule's rows", () => {
+    const { problems } = evaluateRules(catalog, index, resolveRules([{ id: "cross-service-fk", severity: "warning" }]));
+    const fk = problems.filter((problem) => problem.rule === "cross-service-fk");
+    expect(fk.length).toBeGreaterThan(0);
+    expect(fk.every((problem) => problem.severity === "warning")).toBe(true);
+  });
+
+  it("sorts errors first after the rules have spoken", () => {
+    const { problems } = evaluateProblems(catalog, index, resolveRules([{ id: "cross-service-fk", severity: "warning" }]));
+    const firstWarning = problems.findIndex((problem) => problem.severity === "warning");
+    expect(problems.slice(firstWarning).every((problem) => problem.severity === "warning")).toBe(true);
+    expect(allProblems(catalog, index, resolveRules([])).length).toBe(all.length);
+  });
+});
+
+describe("a rule of the estate's own", () => {
+  it("runs over every event and says which it matched", () => {
+    const rule = custom({
+      id: "team.quiet-event",
+      when: "size(event.consumers) == 0",
+      message: "'nothing consumes ' + event.name",
+      peer: "event.service",
+    });
+    const { problems, failure } = runRule(rule, catalog, index);
+    expect(failure).toBeUndefined();
+    const quiet = subjectsOf(catalog, index, "event").filter((subject) => (subject.row.consumers as string[]).length === 0);
+    expect(problems.length).toBe(quiet.length);
+    expect(problems.length).toBeGreaterThan(0);
+    const first = problems[0]!;
+    expect(first.rule).toBe("team.quiet-event");
+    expect(first.id).toBe(quiet[0]!.id);
+    expect(first.note).toBe(`nothing consumes ${quiet[0]!.row.name}`);
+    expect(first.peer).toBe(quiet[0]!.service);
+    expect(first.context).toBe(quiet[0]!.context);
+  });
+
+  it("sees the estate beside its subject", () => {
+    const inside = runRule(custom({ id: "a", over: "call", when: "call.peer in estate.services", message: "'in'" }), catalog, index);
+    const outside = runRule(custom({ id: "b", over: "call", when: "!(call.peer in estate.services)", message: "'out'" }), catalog, index);
+    expect(inside.failure).toBeUndefined();
+    expect(outside.failure).toBeUndefined();
+    expect(inside.problems.length + outside.problems.length).toBe(subjectsOf(catalog, index, "call").length);
+    expect(estateOf(catalog).services).toContain("shop.cart");
+  });
+
+  it("runs over flows and aggregates, and lands on the page each has", () => {
+    const flows = runRule(
+      custom({ id: "team.unproven-flow", over: "flow", when: "flow.verifiedSteps == 0 && flow.steps > 0", message: "flow.name + ' has never been seen running'" }),
+      catalog,
+      index,
+    );
+    expect(flows.failure).toBeUndefined();
+    const unproven = subjectsOf(catalog, index, "flow").filter((subject) => subject.row.verifiedSteps === 0n && (subject.row.steps as bigint) > 0n);
+    expect(flows.problems.length).toBe(unproven.length);
+    expect(flows.problems.length).toBeGreaterThan(0);
+    expect(flows.problems[0]!.context).toBe(unproven[0]!.row.owner);
+
+    const aggregates = runRule(
+      custom({ id: "team.silent-aggregate", over: "aggregate", when: "size(aggregate.events) == 0 && !aggregate.modelGroup", message: "aggregate.name + ' raises no event'", peer: "aggregate.service" }),
+      catalog,
+      index,
+    );
+    expect(aggregates.failure).toBeUndefined();
+    const silent = subjectsOf(catalog, index, "aggregate").filter((subject) => (subject.row.events as string[]).length === 0 && subject.row.modelGroup === false);
+    expect(aggregates.problems.length).toBe(silent.length);
+    for (const problem of aggregates.problems) expect(index.aggregateById.has(problem.id), problem.id).toBe(true);
+  });
+
+  it("refuses a field the subject does not have, before any row", () => {
+    const { problems, failure } = runRule(custom({ id: "a", when: "event.nme == 'x'", message: "'m'" }), catalog, index);
+    expect(problems).toEqual([]);
+    expect(failure?.message).toMatch(/nme/);
+  });
+
+  it("refuses a condition that is not a bool and a message that is not a string", () => {
+    expect(runRule(custom({ id: "a", when: "event.id", message: "'m'" }), catalog, index).failure?.message).toMatch(/bool/);
+    expect(runRule(custom({ id: "b", when: "true", message: "event.versions" }), catalog, index).failure?.message).toMatch(/string/);
+  });
+
+  it("is counted but not shown while switched off", () => {
+    const rule = custom({ id: "team.off", when: "true", message: "'m'", enabled: false });
+    const { problems, matches, failures } = evaluateRules(catalog, index, [rule]);
+    expect(problems).toEqual([]);
+    expect(failures).toEqual([]);
+    expect(matches.get("team.off")).toBe(subjectsOf(catalog, index, "event").length);
+  });
+});
+
+describe("subjects", () => {
+  const shapes: Record<string, (value: unknown) => boolean> = {
+    string: (value) => typeof value === "string",
+    int: (value) => typeof value === "bigint",
+    bool: (value) => typeof value === "boolean",
+    "list<string>": (value) => Array.isArray(value) && value.every((item) => typeof item === "string"),
+  };
+
+  // The frozen estate declares no AsyncAPI channel, no deployment and no
+  // vendored copy; one of each is added so every subject is proved on a
+  // row, not on an absence.
+  const furnished: Catalog = JSON.parse(JSON.stringify(rawCatalog)) as unknown as Catalog;
+  const first = furnished.contexts[0]!.services[0]!;
+  first.channels = [
+    { address: "shop.cart.basket", kind: "event", messages: [{ name: "cart.BasketCreated", direction: "send" }, { name: "payments.PaymentAuthorized", direction: "receive" }], source: "asyncapi.yaml" },
+  ];
+  first.copies = [{ id: "pricing.v1.Pricing", methods: [{ name: "GetQuote", request: "GetQuoteRequest", response: "Quote" }], source: "vendor/pricing.proto" }];
+  furnished.deployments = [
+    { id: "argocd/cart", name: "cart", project: "shop", environment: "prod", cluster: "in-cluster", namespace: "shop", repo: first.repo, path: first.path, targetRevision: "main", revision: "abc", tool: "kustomize", url: "https://argocd/cart" },
+  ];
+  const furnishedCatalog = validateCatalog(furnished);
+  const furnishedIndex = buildIndex(furnishedCatalog);
+
+  for (const over of SUBJECT_NAMES as RuleSubject[]) {
+    it(`${over}: every row has exactly the schema's fields, typed as it says`, () => {
+      const rows = subjectsOf(furnishedCatalog, furnishedIndex, over);
+      expect(rows.length, over).toBeGreaterThan(0);
+      const schema = SUBJECTS[over].schema;
+      for (const subject of rows) {
+        expect(Object.keys(subject.row).sort(), subject.id).toEqual(Object.keys(schema).sort());
+        for (const [field, type] of Object.entries(schema)) {
+          expect(shapes[type]!(subject.row[field]), `${over} ${subject.id}.${field} as ${type}`).toBe(true);
+        }
+        expect(subject.id.length).toBeGreaterThan(0);
+      }
+    });
+  }
+});
+
+describe("problemRuleProblems", () => {
+  it("accepts a switch, a re-grade and a whole rule", () => {
+    expect(
+      problemRuleProblems(
+        [
+          { id: "shared-store", enabled: false, reason: "by design" },
+          { id: "cross-service-fk", severity: "warning" },
+          { id: "team.quiet-event", over: "event", when: "size(event.consumers) == 0", message: "'quiet ' + event.id", title: "Quiet event" },
+        ],
+        builtinIds,
+      ),
+    ).toEqual([]);
+  });
+
+  it("names what is wrong, by entry and field", () => {
+    const problems = problemRuleProblems(
+      [
+        { id: "shared-store", enabled: false },
+        { id: "rpc", when: "true" },
+        { id: "team.a", over: "event", when: "event.nme == 'x'", message: "'m'", title: "A" },
+        { id: "team.b", over: "event", when: "true", message: "event.versions", title: "B" },
+        { id: "team.c", over: "nowhere", when: "true", message: "'m'", title: "C" },
+        { id: "team.d", over: "event", when: "true", message: "'m'" },
+        { id: "Bad Id", over: "event", when: "true", message: "'m'", title: "D" },
+        { id: "team.a", over: "event", when: "true", message: "'m'", title: "again" },
+      ],
+      builtinIds,
+      "x.json",
+    );
+    expect(problems).toEqual(
+      expect.arrayContaining([
+        expect.stringMatching(/^x\.json problemRules\/0\/reason: /),
+        expect.stringMatching(/^x\.json problemRules\/1\/when: "rpc" is a built-in rule/),
+        expect.stringMatching(/^x\.json problemRules\/2\/when: .*nme/),
+        expect.stringMatching(/^x\.json problemRules\/3\/message: CEL expression must return string/),
+        expect.stringMatching(/^x\.json problemRules\/4\/over: /),
+        expect.stringMatching(/^x\.json problemRules\/5\/title: required/),
+        expect.stringMatching(/^x\.json problemRules\/6\/id: /),
+        expect.stringMatching(/^x\.json problemRules\/7\/id: "team\.a" appears twice/),
+      ]),
+    );
+    expect(problems.length).toBe(8);
+  });
+
+  it("is nothing to say about a manifest without the key", () => {
+    expect(problemRuleProblems(undefined, builtinIds)).toEqual([]);
+  });
+});