@canonical/pragma-cli 0.34.0 → 0.36.0

package/src/testing/eval/cases/readNouns.ts

@@ -86,6 +86,181 @@ export const readNounEvalCases: readonly EvalCase[] = [
       });
     },
   },
+  {
+    // The journey the reported failure actually walked: browse a category, take
+    // a published `name` VERBATIM, and ask for its content. Every row `list`
+    // publishes must be addressable by `lookup` — otherwise the two-step
+    // grammar the whole read surface is built on has a hole in it, and an agent
+    // that follows the tools' own output gets ENTITY_NOT_FOUND with empty
+    // suggestions and a recovery pointing back at the list it just read.
+    id: "tool-standard-list-names-are-addressable-by-lookup",
+    kind: "tool",
+    input:
+      "every name standard_list {category:react} publishes resolves through standard_lookup {name:[…], detail:detailed}, dos/donts included.",
+    async expect() {
+      await withCanonicalFixture(CANONICAL_CONFIG, async (mcp) => {
+        const list = await mcp.callTool("standard_list", { category: "react" });
+        assert.equal(list.ok, true);
+        const names = (list.data as { name: string }[]).map((row) => row.name);
+        assert.ok(names.length > 0, "expected standard_list to publish rows");
+        let withExamples = 0;
+        for (const name of names) {
+          const result = await mcp.callTool("standard_lookup", {
+            name: [name],
+            detail: "detailed",
+          });
+          assert.equal(
+            result.ok,
+            true,
+            `standard_lookup rejected the name standard_list published: ${name}`,
+          );
+          const entity = (result.data as { results: Record<string, unknown>[] })
+            .results[0];
+          assert.equal(entity?.name, name);
+          if (
+            (entity?.dos as unknown[] | undefined)?.length &&
+            (entity?.donts as unknown[] | undefined)?.length
+          ) {
+            withExamples += 1;
+          }
+        }
+        assert.ok(
+          withExamples > 0,
+          "expected at least one resolved standard to carry both dos and donts",
+        );
+      });
+    },
+  },
+  {
+    // The glob the lookup tool's own description advertises. It matched nothing
+    // on the shipped graph, because the candidate pool was the `cs:name`
+    // population and no asserted name contains a slash.
+    id: "tool-standard-lookup-advertised-glob-matches",
+    kind: "tool",
+    input:
+      'standard_lookup {name:["react/component/*"]} — the glob the tool description advertises — resolves at least one standard.',
+    async expect() {
+      await withCanonicalFixture(CANONICAL_CONFIG, async (mcp) => {
+        const result = await mcp.callTool("standard_lookup", {
+          name: ["react/component/*"],
+        });
+        assert.equal(result.ok, true);
+        const results = (result.data as { results: unknown[] }).results;
+        assert.ok(
+          results.length > 0,
+          "the advertised glob must match at least one standard",
+        );
+      });
+    },
+  },
+  {
+    // The MIRROR of the addressability case above, and the reason the IRI-name
+    // fallback is DECLARED per story rather than inferred from a class
+    // constraint. `token list` requires `ds:tokenId`, so a `ds:Token` without
+    // one is a row it never publishes — and it must therefore be reachable by
+    // no name at all, and drawn by no sample. Inferring the fallback from the
+    // presence of `type: ds:Token` made it addressable and sampleable under an
+    // IRI-derived name the list never handed out: the same list/lookup
+    // disagreement as the standards defect, pointing the other way.
+    id: "tool-token-lookup-addresses-only-what-list-publishes",
+    kind: "tool",
+    input:
+      "a ds:Token carrying no ds:tokenId appears in no token_list row, resolves through no token_lookup name, and is drawn by no token_sample.",
+    async expect() {
+      await withCanonicalFixture(ALL_VISIBLE_CONFIG, async (mcp) => {
+        const list = await mcp.callTool("token_list");
+        assert.equal(list.ok, true);
+        const published = new Set(
+          (list.data as { name: string }[]).map((row) => row.name),
+        );
+        assert.ok(published.size > 0, "expected token_list to publish rows");
+        // The IRI-derived spellings the fallback would have minted for
+        // `ds:token.legacy.borderRadius` (local name, dots as slashes).
+        for (const derived of [
+          "token.legacy.borderRadius",
+          "token/legacy/borderRadius",
+        ]) {
+          assert.ok(
+            !published.has(derived),
+            `token_list must not publish ${derived}`,
+          );
+          const result = await mcp.callTool("token_lookup", {
+            name: [derived],
+          });
+          assert.equal(
+            result.ok,
+            false,
+            `token_lookup addressed ${derived}, a name token_list never published`,
+          );
+          assert.equal(
+            (result.error as { code: string }).code,
+            "ENTITY_NOT_FOUND",
+          );
+        }
+        // `sample`'s draw pool is the same population, so its reported total is
+        // the published population — not one entity wider.
+        const sample = await mcp.callTool("token_sample", {});
+        assert.equal(sample.ok, true);
+        assert.equal(
+          (sample.data as { totalCount: number }).totalCount,
+          published.size,
+        );
+      });
+    },
+  },
+  {
+    // A slug the graph declares with no standards filed under it. `standard
+    // categories` lists it at count 0 and the tool descriptions document a
+    // filtered list as a plain list, so asking for it is a calm empty answer —
+    // an INVALID_INPUT here tells an agent its own valid slug was a typo.
+    // Validity read off the RETURNED ROWS produced exactly that, because a
+    // category with no standards appears in no row.
+    id: "tool-standard-list-empty-category-is-a-calm-empty-list",
+    kind: "tool",
+    input:
+      "standard_list {category} for a real slug that no standard uses is ok:true with an empty list, not INVALID_INPUT.",
+    async expect() {
+      await withCanonicalFixture(CANONICAL_CONFIG, async (mcp) => {
+        const categories = await mcp.callTool("standard_categories");
+        assert.equal(categories.ok, true);
+        const empty = (categories.data as { name: string; count: string }[])
+          .filter((row) => Number(row.count) === 0)
+          .map((row) => row.name);
+        assert.ok(
+          empty.length > 0,
+          "expected standard_categories to report a zero-count category",
+        );
+        for (const slug of empty) {
+          const result = await mcp.callTool("standard_list", {
+            category: slug,
+          });
+          assert.equal(
+            result.ok,
+            true,
+            `standard_list rejected ${slug}, a slug standard_categories publishes`,
+          );
+          assert.deepEqual(result.data, []);
+        }
+        // The rejection itself still works, and still names the graph's whole
+        // vocabulary — including the categories no standard uses.
+        const rejected = await mcp.callTool("standard_list", {
+          category: "zzz-not-a-slug",
+        });
+        assert.equal(rejected.ok, false);
+        const error = rejected.error as {
+          code: string;
+          validOptions: string[];
+        };
+        assert.equal(error.code, "INVALID_INPUT");
+        for (const slug of empty) {
+          assert.ok(
+            error.validOptions.includes(slug),
+            `validOptions omitted the declared category ${slug}`,
+          );
+        }
+      });
+    },
+  },
   {
     id: "content-canonical-graph-has-4-components",
     kind: "content",
@@ -146,24 +321,85 @@ export const readNounEvalCases: readonly EvalCase[] = [
     },
   },
   {
-    id: "content-standard-list-length-equals-category-count-sum",
+    // The cross-surface count-parity invariant, stated for a HIERARCHY. Summing
+    // the category counts stopped being the right arithmetic the moment a
+    // parent counted its branch (a standard under `testing-unit` is counted by
+    // both `testing-unit` and `testing`); what must hold — and what actually
+    // catches the reported defect — is that each category's count and the rows
+    // `--category` returns for it are the same set. Before the roll-up,
+    // `testing` reported 1 and `standard list --category testing` returned 1 of
+    // the 8 standards in the branch: a silently wrong answer, exit 0.
+    id: "content-standard-category-counts-match-filtered-list",
     kind: "content",
     input:
-      "standard_list length equals the sum of standard_categories counts (the cross-surface count-parity invariant).",
+      "every standard_categories count equals the length of standard_list {category: <slug>}, and a parent category answers for its whole branch.",
     async expect() {
       await withCanonicalFixture(CANONICAL_CONFIG, async (mcp) => {
         const list = await mcp.callTool("standard_list");
         const categories = await mcp.callTool("standard_categories");
-        const listLength = (list.data as unknown[]).length;
-        const categorySum = (categories.data as { count: string }[]).reduce(
-          (sum, row) => sum + Number(row.count),
-          0,
+        const rows = categories.data as { name: string; count: string }[];
+        assert.ok(
+          (list.data as unknown[]).length > 0 && rows.length > 0,
+          "both surfaces must be non-empty for the parity invariant to bite",
+        );
+        let rolledUp = 0;
+        for (const row of rows) {
+          const filtered = await mcp.callTool("standard_list", {
+            category: row.name,
+          });
+          assert.equal(
+            (filtered.data as unknown[]).length,
+            Number(row.count),
+            `standard_list --category ${row.name} disagrees with its count`,
+          );
+          // A parent whose branch is bigger than its own direct membership:
+          // the roll-up is doing something, so this case cannot pass vacuously.
+          const direct = (list.data as { category?: string }[]).filter(
+            (r) => r.category === row.name,
+          ).length;
+          if (Number(row.count) > direct) rolledUp += 1;
+        }
+        assert.ok(
+          rolledUp > 0,
+          "expected at least one parent category to answer for its descendants",
+        );
+      });
+    },
+  },
+  {
+    // The reflexive half of `skos:broader*`. `broader+` looks equivalent and
+    // silently drops every standard filed DIRECTLY on the category asked for —
+    // one of the 8 under `testing` in the shipped graph.
+    id: "content-standard-category-rollup-keeps-the-direct-member",
+    kind: "content",
+    input:
+      "standard_list {category: <parent>} includes both the standard filed directly on the parent and those filed on its child.",
+    async expect() {
+      await withCanonicalFixture(CANONICAL_CONFIG, async (mcp) => {
+        const parent = await mcp.callTool("standard_list", {
+          category: "testing",
+        });
+        const child = await mcp.callTool("standard_list", {
+          category: "testing-unit",
+        });
+        const parentRows = parent.data as { name: string; category: string }[];
+        const childNames = new Set(
+          (child.data as { name: string }[]).map((r) => r.name),
+        );
+        assert.ok(
+          childNames.size > 0,
+          "expected the child category to be used",
         );
+        for (const name of childNames) {
+          assert.ok(
+            parentRows.some((r) => r.name === name),
+            `the parent category dropped its descendant ${name}`,
+          );
+        }
         assert.ok(
-          listLength > 0,
-          "standard_list must be non-empty for the count-parity invariant to be meaningful",
+          parentRows.some((r) => r.category === "testing"),
+          "the parent category dropped the standard filed directly on it",
         );
-        assert.equal(listLength, categorySum);
       });
     },
   },

package/src/testing/fixtures/blockGraph.ts

@@ -4,13 +4,26 @@
  *
  * Models the live ontology's shape faithfully but compactly: a `ds:UIBlock`
  * interface with `Component`/`Pattern`/`Subcomponent` subclasses, the block
- * detail properties (summary, whenToUse/whenNotToUse, guidelines, anatomy,
- * figmaLink), modifier families with values asserted ONLY in the reverse
+ * detail properties (summary, usage, guidelines, anatomy, figmaLink),
+ * modifier families with values asserted ONLY in the reverse
  * `ds:modifierFamily` direction (so the compiled inverse-union resolver is
  * exercised), block properties, and `ds:hasSubcomponent` scoped to `ds:Component`
  * (reached via a subtype-scoped fragment). Two blocks — Button and Modal —
- * carry the full spec so block content-parity asserts on a graph that DECLARES
- * whenToUse/whenNotToUse (which the live graph superseded with ds:usage).
+ * carry the full spec so block content-parity asserts on a graph shaped like
+ * the live one.
+ *
+ * The usage narrative is modelled as the live graph models it: ONE `ds:usage`
+ * literal per block, free-text Markdown carrying its own `### When to use` /
+ * `### When not to use` sub-sections. It used to declare the retired
+ * `ds:whenToUse`/`ds:whenNotToUse` pair instead, which is why the block parity
+ * suite went on asserting a "### When to use" heading that no real install had
+ * rendered since the ontology conflated the two into `ds:usage`.
+ *
+ * Both shapes the live pack has are here on purpose: Button carries a
+ * non-empty literal (126 of the 264 live blocks) and Modal an EMPTY one (the
+ * other 138), so the fixture proves the renderer both nests a body's own
+ * headings under the section heading AND prints no heading at all for an empty
+ * literal. A fixture that only ever carried prose could not tell those apart.
  */
 
 /** The prefixes the fixture store is built and queried with. */
@@ -21,6 +34,18 @@ export const BLOCK_PREFIXES: Readonly<Record<string, string>> = {
   xsd: "http://www.w3.org/2001/XMLSchema#",
 };
 
+/**
+ * A membership roster deep enough to cross the entity read's fan-in threshold,
+ * so the fixture exercises the SAMPLED branch and not only the listed one.
+ * Generated rather than written out: the count is the whole point of the case,
+ * and 22 hand-copied lines invite someone to "tidy" one away.
+ */
+const ROSTER_MEMBER_COUNT = 22;
+const ROSTER_MEMBERS = Array.from(
+  { length: ROSTER_MEMBER_COUNT },
+  (_, index) => `ds:probe.member${index} a ds:Probe ; ds:tier ds:rosterHub .`,
+).join("\n");
+
 /** The fixture ontology + individuals as Turtle. */
 export const BLOCK_TTL = `
 @prefix ds: <https://ds.canonical.com/> .
@@ -44,8 +69,7 @@ ds:name a owl:DatatypeProperty ;
 ds:tier a owl:ObjectProperty, owl:FunctionalProperty ;
   rdfs:domain ds:UIBlock ; rdfs:range ds:Tier .
 ds:summary a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
-ds:whenToUse a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
-ds:whenNotToUse a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
+ds:usage a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
 ds:guidelines a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
 ds:anatomyDsl a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
 ds:anatomyClassic a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
@@ -70,8 +94,7 @@ ds:button a ds:Component ;
   ds:name "Button" ;
   ds:tier ds:global ;
   ds:summary "Primary action trigger with optional icon and label." ;
-  ds:whenToUse "Use for the primary action on a view." ;
-  ds:whenNotToUse "Do not use for navigation between pages." ;
+  ds:usage "Buttons trigger actions; links go places.\\n\\n### When to use\\n\\n- For the primary action on a view.\\n\\n### When not to use\\n\\n- For navigation between pages." ;
   ds:guidelines "Keep labels short and action-oriented." ;
   ds:anatomyDsl "root: button; children: label, icon" ;
   ds:anatomyClassic "Button > Label, Icon" ;
@@ -84,8 +107,7 @@ ds:modal a ds:Component ;
   ds:name "Modal" ;
   ds:tier ds:global ;
   ds:summary "Focused overlay dialog for a single task." ;
-  ds:whenToUse "Use to interrupt for a critical confirmation." ;
-  ds:whenNotToUse "Do not use for non-blocking notifications." ;
+  ds:usage "" ;
   ds:guidelines "Always provide an explicit close affordance." ;
   ds:anatomyDsl "root: dialog; children: header, body, footer" ;
   ds:hasModifierFamily ds:family.size ;
@@ -109,4 +131,76 @@ ds:mod.importance.secondary a ds:Modifier ; ds:name "secondary" ; ds:modifierFam
 ds:mod.density.compact a ds:Modifier ; ds:name "compact" ; ds:modifierFamily ds:family.density .
 ds:mod.size.small a ds:Modifier ; ds:name "small" ; ds:modifierFamily ds:family.size .
 ds:mod.size.large a ds:Modifier ; ds:name "large" ; ds:modifierFamily ds:family.size .
+
+# ---- Neighbourhood-read probes ----
+# Typed \`ds:Probe\`, which is deliberately NOT declared as an owl:Class and is
+# outside the block VALUES set, so these individuals reach the entity reader
+# without entering any block list or the ontology class listing.
+#
+# literalTrap holds a STRING that begins with the ds: namespace. Read through
+# the lossy string view it was indistinguishable from an IRI and got compacted
+# into something that read back as one; the term view is what tells them apart.
+ds:literalTrap a ds:Probe ;
+  ds:name "literal trap" ;
+  ds:figmaLink "https://ds.canonical.com/not-an-iri" .
+
+# blankHolder carries blank-node objects — store-local handles that re-mint on
+# every load — so the read must inline them as records AND order those records
+# by content rather than by the label that grouped them.
+ds:blankHolder a ds:Probe ;
+  ds:name "blank holder" ;
+  ds:changeLog [ a ds:ChangeLogEntry ; ds:changeType "decision" ; ds:change "Split the button." ] ;
+  ds:changeLog [ a ds:ChangeLogEntry ; ds:changeType "revision" ; ds:change "Renamed the slot." ] .
+
+# One blank node reached by TWO predicates — keyed by node alone, the second
+# edge silently vanished.
+ds:doubleLinked a ds:Probe ;
+  ds:name "double linked" ;
+  ds:changeLog [ a ds:ChangeLogEntry ; ds:changeType "decision" ] ;
+  ds:usageNote [ a ds:ChangeLogEntry ; ds:changeType "note" ] .
+
+# An RDF 1.2 directional literal — dropped, it reads as a plain @ar literal.
+ds:directional a ds:Probe ;
+  ds:name "\u0645\u0631\u062D\u0628\u0627"@ar--rtl .
+
+# The roster hub also carries ONE edge under a different predicate, so a read
+# can prove that a 22-deep neighbour does not evict its quieter sibling.
+ds:rosterHub a ds:Probe ; ds:name "roster hub" .
+ds:probe.quiet a ds:Probe ; ds:hasSubcomponent ds:rosterHub .
+${ROSTER_MEMBERS}
+`;
+
+/**
+ * An OPT-IN overlay for the lookup-addressing suite, appended to
+ * {@link BLOCK_TTL} by the tests that need it and by nothing else — the block
+ * parity, ontology and GraphQL-engine suites assert over the base graph and
+ * must keep seeing exactly two components.
+ *
+ * It models the three shapes the addressing path gets wrong: two components
+ * sharing one `ds:name` (declared zeta-first so the store's enumeration order
+ * and IRI order DISAGREE), and a component carrying no `ds:name` at all
+ * (reachable only by IRI, the way 131 of the 144 live code standards are).
+ *
+ * The shared name is a SHAPE probe, not the live ambiguity: neither chip's tier
+ * outranks the other, so this fixture can only prove that both are returned in a
+ * total order — never which of them a reader means. Deciding that is a judgement
+ * about real tiers, and it is asserted where the real tiers are, against the
+ * shipped pack (`capabilities/block.tierRank.exec.test.ts`).
+ */
+export const AMBIGUOUS_TTL = `
+ds:apps a ds:Tier ; ds:name "apps" .
+
+ds:zeta.chip a ds:Component ;
+  ds:name "Chip" ;
+  ds:tier ds:global ;
+  ds:summary "The zeta chip — declared first, sorts last." .
+
+ds:alpha.chip a ds:Component ;
+  ds:name "Chip" ;
+  ds:tier ds:apps ;
+  ds:summary "The alpha chip — declared last, sorts first." .
+
+ds:nameless.widget a ds:Component ;
+  ds:tier ds:global ;
+  ds:summary "Carries no ds:name; addressable only by IRI." .
 `;