@canonical/pragma-cli 0.31.0 → 0.33.0

package/src/testing/eval/cases/stable.ts

@@ -0,0 +1,163 @@
+/**
+ * Stable-now eval seed cases — need no PR3 noun (PR1/PR2 foundation +
+ * PR4's own harness only): the embedded pack, the live MCP catalog (whatever
+ * it currently contains — no noun is hard-coded), and one small self-contained
+ * disclosure fixture. `cases/readNouns.ts` (commit 4) extends this list once
+ * PR3's read nouns are live.
+ */
+
+import assert from "node:assert/strict";
+import { compilePack } from "../../../kernel/packs/compile.js";
+import type { PackDefinition } from "../../../kernel/packs/types.js";
+import type { CapabilityModule } from "../../../kernel/spec/types.js";
+import { bootFixtureRuntime } from "../../helpers/fixtureGraph.js";
+import { projectMcp } from "../../helpers/projectMcp.js";
+import type { EvalCase } from "../harness.js";
+
+/** A tiny, self-contained fixture — NOT the shared canonical graph, so this
+ * case stays independent of PR3's noun set (parameterized elsewhere). */
+const WIDGET_PREFIXES = {
+  ex: "https://example.org/eval-widgets#",
+  owl: "http://www.w3.org/2002/07/owl#",
+  rdfs: "http://www.w3.org/2000/01/rdf-schema#",
+  xsd: "http://www.w3.org/2001/XMLSchema#",
+};
+const WIDGET_TTL = `
+@prefix ex: <https://example.org/eval-widgets#> .
+@prefix owl: <http://www.w3.org/2002/07/owl#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+ex:Widget a owl:Class .
+ex:name a owl:DatatypeProperty ; rdfs:domain ex:Widget ; rdfs:range xsd:string .
+ex:description a owl:DatatypeProperty ; rdfs:domain ex:Widget ; rdfs:range xsd:string .
+ex:hasPart a owl:ObjectProperty ; rdfs:domain ex:Widget ; rdfs:range ex:Widget .
+ex:button a ex:Widget ; ex:name "Button" ; ex:description "A button." ; ex:hasPart ex:label .
+ex:label a ex:Widget ; ex:name "Label" .
+`;
+const WIDGET_PACK: PackDefinition = {
+  noun: "widget",
+  lookup: {
+    source: "sparql",
+    by: "ex:name",
+    type: "ex:Widget",
+    fields: [
+      { name: "description", property: "ex:description", level: "standard" },
+    ],
+    expand: [
+      {
+        name: "parts",
+        relation: "ex:hasPart",
+        level: "detailed",
+        select: [{ name: "name", property: "ex:name" }],
+      },
+    ],
+    disclosure: {
+      levels: ["summary", "standard", "detailed"],
+      default: "summary",
+    },
+  },
+};
+
+/** The condensed-SDL token-budget ceiling the covenant reserves (seeded, not enforced). */
+const CONDENSED_SDL_TOKEN_BUDGET = 8000;
+/** The old suite's one token anchor: ~1 token per 4 characters. */
+const CHARS_PER_TOKEN = 4;
+
+export const stableEvalCases: readonly EvalCase[] = [
+  {
+    id: "tool-sparql-count-positive",
+    kind: "tool",
+    input:
+      "SPARQL COUNT(*) over the embedded pack resolves > 0 — the same escape-hatch facade the live `graph query` verb (PR6, tool graph_query) delegates to (PARITY_GAPS: graph-query-deferred).",
+    async expect({ runtime }) {
+      const result = await runtime.query.sparql(
+        "SELECT (COUNT(*) AS ?c) WHERE { ?s ?p ?o }",
+      );
+      assert.equal(result.type, "select");
+      const count =
+        result.type === "select" ? Number(result.bindings[0]?.c ?? 0) : 0;
+      assert.ok(count > 0, `expected count > 0, got ${count}`);
+    },
+  },
+  {
+    id: "content-condensed-sdl-token-budget",
+    kind: "content",
+    input:
+      "the aggregate live MCP tool catalog (name + description + inputSchema, whatever it currently contains) stays under the condensedSDL budget (<=8000 tokens, ~1 token/4 chars).",
+    async expect({ mcp }) {
+      const tools = await mcp.listTools();
+      assert.ok(tools.length > 0, "expected at least one registered tool");
+      const sdlText = tools
+        .map(
+          (tool) =>
+            `${tool.name}\n${tool.description ?? ""}\n${JSON.stringify(tool.inputSchema ?? {})}`,
+        )
+        .join("\n");
+      const estimatedTokens = Math.ceil(sdlText.length / CHARS_PER_TOKEN);
+      assert.ok(
+        estimatedTokens <= CONDENSED_SDL_TOKEN_BUDGET,
+        `estimated ${estimatedTokens} tokens (${tools.length} tools) exceeds the ${CONDENSED_SDL_TOKEN_BUDGET}-token condensedSDL budget`,
+      );
+    },
+  },
+  {
+    id: "disclosure-summary-omits-standard-and-detailed-fields",
+    kind: "disclosure",
+    input:
+      "widget_lookup {detail:summary} omits standard/detailed-gated fields; {detail:detailed} includes them — proving the harness can express a disclosure case (the cross-noun sweep over PR3 nouns is B6, commit 4).",
+    async expect() {
+      const fixture = await bootFixtureRuntime({ ttl: WIDGET_TTL });
+      try {
+        const widgetModule: CapabilityModule = {
+          name: "widget",
+          verbs: compilePack(WIDGET_PACK, "test:widget", WIDGET_PREFIXES),
+        };
+        const mcp = await projectMcp([widgetModule], fixture.cwd);
+        try {
+          const summary = (await mcp.callTool("widget_lookup", {
+            name: ["Button"],
+            detail: "summary",
+          })) as { data: { results: Record<string, unknown>[] } };
+          const detailed = (await mcp.callTool("widget_lookup", {
+            name: ["Button"],
+            detail: "detailed",
+          })) as { data: { results: Record<string, unknown>[] } };
+
+          assert.equal(summary.data.results[0]?.description, undefined);
+          assert.equal(detailed.data.results[0]?.description, "A button.");
+          assert.deepEqual(detailed.data.results[0]?.parts, [
+            { name: "Label" },
+          ]);
+        } finally {
+          await mcp.cleanup();
+        }
+      } finally {
+        await fixture.dispose();
+      }
+    },
+  },
+  {
+    id: "prompt-version-check-maps-to-info",
+    kind: "prompt",
+    input:
+      '"What version of pragma am I running?" should map to the `info` tool.',
+    async expect({ mcp }) {
+      const tools = await mcp.listTools();
+      const info = tools.find((tool) => tool.name === "info");
+      assert.ok(info, "expected an `info` tool in the catalog");
+      assert.match(String(info?.description ?? ""), /version/i);
+    },
+  },
+  {
+    id: "prompt-config-question-maps-to-config-show",
+    kind: "prompt",
+    input:
+      '"How is pragma configured right now?" should map to the `config_show` tool.',
+    async expect({ mcp }) {
+      const tools = await mcp.listTools();
+      const configShow = tools.find((tool) => tool.name === "config_show");
+      assert.ok(configShow, "expected a `config_show` tool in the catalog");
+      assert.match(String(configShow?.description ?? ""), /config/i);
+    },
+  },
+];

package/src/testing/eval/eval.test.ts

@@ -0,0 +1,74 @@
+/**
+ * The eval gate — runs the seed cases as part of `bun run test`, so CI
+ * exercises the harness itself, not just the individual case bodies.
+ *
+ * `env.mcp` is built from the REAL, live `capabilities` array — never a
+ * hard-coded noun subset — so this gate automatically grows to cover PR5/6/7's
+ * tools as they land, with no edit here (R2 discipline).
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { capabilities } from "../../capabilities/index.js";
+import { bootRuntime } from "../../kernel/runtime/boot.js";
+import { TEST_FLAGS } from "../helpers/projectCli.js";
+import { projectMcp } from "../helpers/projectMcp.js";
+import { assemblyEvalCases } from "./cases/assembly.js";
+import { effectfulEvalCases } from "./cases/effectful.js";
+import { readNounEvalCases } from "./cases/readNouns.js";
+import { stableEvalCases } from "./cases/stable.js";
+import { type EvalCaseResult, type EvalEnv, runEvals } from "./harness.js";
+
+// PR7 populates the full MCP eval matrix on PR4's seed: read nouns (PR3),
+// effectful nouns (PR6 + PR7 mutations/graph/prompt content), and the assembly
+// orientation surface (PR7 capabilities/instructions/native prompts).
+const allSeedCases = [
+  ...stableEvalCases,
+  ...readNounEvalCases,
+  ...effectfulEvalCases,
+  ...assemblyEvalCases,
+];
+
+/** Strip the (only-populated-on-failure) `detail` so a green report snapshots
+ * as pure signal — a failure's message is diagnostic noise in the golden, not
+ * something worth pinning byte-for-byte. */
+function normalize(
+  results: readonly EvalCaseResult[],
+): { id: string; kind: string; passed: boolean }[] {
+  return [...results]
+    .sort((a, b) => a.id.localeCompare(b.id))
+    .map(({ id, kind, passed }) => ({ id, kind, passed }));
+}
+
+describe("eval harness — seed gate", () => {
+  let env: EvalEnv;
+  let cleanup: () => Promise<void>;
+
+  beforeAll(async () => {
+    const runtime = bootRuntime(TEST_FLAGS);
+    const mcp = await projectMcp(capabilities);
+    env = { runtime, mcp };
+    cleanup = () => mcp.cleanup();
+  });
+
+  afterAll(async () => {
+    await cleanup();
+  });
+
+  it("runs the full seed (stable + read-noun) cases clean", async () => {
+    const report = await runEvals(allSeedCases, env);
+    if (report.failed > 0) {
+      const failures = report.cases.filter((c) => !c.passed);
+      throw new Error(
+        `${report.failed} eval case(s) failed:\n${failures
+          .map((c) => `  - ${c.id}: ${c.detail}`)
+          .join("\n")}`,
+      );
+    }
+    expect(report.failed).toBe(0);
+    expect(report.passed).toBe(allSeedCases.length);
+    // PR7 grows the seed into the fuller MCP matrix (kinds × PR3/6/7 nouns).
+    expect(report.cases.length).toBeGreaterThanOrEqual(35);
+    expect(report.cases.length).toBeLessThanOrEqual(60);
+    expect(normalize(report.cases)).toMatchSnapshot();
+  });
+});

package/src/testing/eval/harness.ts

@@ -0,0 +1,91 @@
+/**
+ * The eval harness — the runner PR4 seeds and PR7 populates.
+ *
+ * No formal eval harness existed anywhere in the old shell (confirmed — the
+ * only `eval` hits there are the config `evaluate` verb, an unrelated
+ * homonym). This is new: a minimal, typed runner over a fixture-graph-backed
+ * `PragmaRuntime`/`McpHarness`, producing a machine-readable report so both a
+ * vitest gate (`eval.test.ts`) and a standalone script (`report.ts`) can drive
+ * the SAME cases.
+ */
+
+import type { PragmaRuntime } from "../../kernel/runtime/types.js";
+import type { McpHarness } from "../helpers/projectMcp.js";
+
+/** The four eval kinds PR4 seeds; PR7 populates the full MCP matrix on top. */
+export type EvalKind = "tool" | "content" | "disclosure" | "prompt";
+
+/** What an eval case's `expect` is handed to probe the system under test. */
+export interface EvalEnv {
+  /** A booted runtime (default: the embedded pack) for direct verb/query calls. */
+  readonly runtime: PragmaRuntime;
+  /** An in-process MCP harness over the full live capability catalog. */
+  readonly mcp: McpHarness;
+}
+
+/**
+ * One eval case: a described input/expectation pair. `expect` throws (via
+ * `vitest#expect` or a plain assertion) to signal failure — {@link runEvals}
+ * catches it and records the case as failed rather than aborting the run.
+ */
+export interface EvalCase {
+  /** A stable, distinct id (used in the report and the golden). */
+  readonly id: string;
+  readonly kind: EvalKind;
+  /** A short human-readable description of what's being evaluated (for the report). */
+  readonly input: string;
+  expect(env: EvalEnv): void | Promise<void>;
+}
+
+/** One case's outcome. */
+export interface EvalCaseResult {
+  readonly id: string;
+  readonly kind: EvalKind;
+  readonly input: string;
+  readonly passed: boolean;
+  /** The failure message, when `passed` is false. */
+  readonly detail?: string;
+}
+
+/** The full report: pass/fail counts plus every case's outcome. */
+export interface EvalReport {
+  readonly passed: number;
+  readonly failed: number;
+  readonly cases: readonly EvalCaseResult[];
+}
+
+/**
+ * Run every case against the shared env, collecting pass/fail — never
+ * throwing (a failing case is recorded, not fatal to the run).
+ *
+ * @param cases - The eval cases to run.
+ * @param env - The shared runtime/mcp handles cases probe.
+ * @returns The aggregate report.
+ */
+export async function runEvals(
+  cases: readonly EvalCase[],
+  env: EvalEnv,
+): Promise<EvalReport> {
+  const results: EvalCaseResult[] = [];
+  for (const evalCase of cases) {
+    try {
+      await evalCase.expect(env);
+      results.push({
+        id: evalCase.id,
+        kind: evalCase.kind,
+        input: evalCase.input,
+        passed: true,
+      });
+    } catch (error) {
+      results.push({
+        id: evalCase.id,
+        kind: evalCase.kind,
+        input: evalCase.input,
+        passed: false,
+        detail: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+  const passed = results.filter((result) => result.passed).length;
+  return { passed, failed: results.length - passed, cases: results };
+}

package/src/testing/eval/report.ts

@@ -0,0 +1,58 @@
+#!/usr/bin/env bun
+/**
+ * Standalone eval runner — `bun run eval` (see package.json).
+ *
+ * Emits the {@link EvalReport} as JSON to stdout (machine-parseable, matching
+ * the CLI's own stdout/stderr discipline) and a Markdown summary to stderr.
+ * Exits non-zero when any case failed, so it composes into CI the same way
+ * the vitest gate (`eval.test.ts`) does.
+ */
+
+import { capabilities } from "../../capabilities/index.js";
+import { bootRuntime } from "../../kernel/runtime/boot.js";
+import { TEST_FLAGS } from "../helpers/projectCli.js";
+import { projectMcp } from "../helpers/projectMcp.js";
+import { readNounEvalCases } from "./cases/readNouns.js";
+import { stableEvalCases } from "./cases/stable.js";
+import type { EvalReport } from "./harness.js";
+import { runEvals } from "./harness.js";
+
+const allSeedCases = [...stableEvalCases, ...readNounEvalCases];
+
+const KINDS = ["tool", "content", "disclosure", "prompt"] as const;
+
+/** Render the report as a Markdown summary, grouped by kind. */
+function toMarkdown(report: EvalReport): string {
+  const lines = [
+    "# Eval report",
+    "",
+    `${report.passed} passed, ${report.failed} failed, ${report.cases.length} total.`,
+    "",
+  ];
+  for (const kind of KINDS) {
+    const inKind = report.cases.filter((c) => c.kind === kind);
+    if (inKind.length === 0) continue;
+    lines.push(`## ${kind}`, "");
+    for (const c of inKind) {
+      lines.push(`- [${c.passed ? "PASS" : "FAIL"}] \`${c.id}\` — ${c.input}`);
+      if (!c.passed) lines.push(`  - ${c.detail}`);
+    }
+    lines.push("");
+  }
+  return lines.join("\n");
+}
+
+async function main(): Promise<void> {
+  const runtime = bootRuntime(TEST_FLAGS);
+  const mcp = await projectMcp(capabilities);
+  try {
+    const report = await runEvals(allSeedCases, { runtime, mcp });
+    process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+    process.stderr.write(`${toMarkdown(report)}\n`);
+    if (report.failed > 0) process.exitCode = 1;
+  } finally {
+    await mcp.cleanup();
+  }
+}
+
+await main();

package/src/testing/fixtures/blockGraph.ts

@@ -0,0 +1,112 @@
+/**
+ * A self-contained design-system fixture graph for the pack GraphQL engine and
+ * the block lookup parity suites.
+ *
+ * Models the live ontology's shape faithfully but compactly: a `ds:UIBlock`
+ * interface with `Component`/`Pattern`/`Subcomponent` subclasses, the block
+ * detail properties (summary, whenToUse/whenNotToUse, guidelines, anatomy,
+ * figmaLink), modifier families with values asserted ONLY in the reverse
+ * `ds:modifierFamily` direction (so the compiled inverse-union resolver is
+ * exercised), block properties, and `ds:hasSubcomponent` scoped to `ds:Component`
+ * (reached via a subtype-scoped fragment). Two blocks — Button and Modal —
+ * carry the full spec so block content-parity asserts on a graph that DECLARES
+ * whenToUse/whenNotToUse (which the live graph superseded with ds:usage).
+ */
+
+/** The prefixes the fixture store is built and queried with. */
+export const BLOCK_PREFIXES: Readonly<Record<string, string>> = {
+  ds: "https://ds.canonical.com/",
+  owl: "http://www.w3.org/2002/07/owl#",
+  rdfs: "http://www.w3.org/2000/01/rdf-schema#",
+  xsd: "http://www.w3.org/2001/XMLSchema#",
+};
+
+/** The fixture ontology + individuals as Turtle. */
+export const BLOCK_TTL = `
+@prefix ds: <https://ds.canonical.com/> .
+@prefix owl: <http://www.w3.org/2002/07/owl#> .
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+
+# ---- Ontology (TBox) ----
+ds:UIBlock a owl:Class .
+ds:Component a owl:Class ; rdfs:subClassOf ds:UIBlock .
+ds:Pattern a owl:Class ; rdfs:subClassOf ds:UIBlock .
+ds:Subcomponent a owl:Class ; rdfs:subClassOf ds:UIBlock .
+ds:Tier a owl:Class .
+ds:ModifierFamily a owl:Class .
+ds:Modifier a owl:Class .
+ds:BlockProperty a owl:Class .
+
+ds:name a owl:DatatypeProperty ;
+  rdfs:domain ds:UIBlock ; rdfs:domain ds:ModifierFamily ;
+  rdfs:domain ds:Modifier ; rdfs:domain ds:BlockProperty ; rdfs:range xsd:string .
+ds:tier a owl:ObjectProperty, owl:FunctionalProperty ;
+  rdfs:domain ds:UIBlock ; rdfs:range ds:Tier .
+ds:summary a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
+ds:whenToUse a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
+ds:whenNotToUse a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
+ds:guidelines a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
+ds:anatomyDsl a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
+ds:anatomyClassic a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
+ds:figmaLink a owl:DatatypeProperty ; rdfs:domain ds:UIBlock ; rdfs:range xsd:string .
+
+ds:hasModifierFamily a owl:ObjectProperty ; rdfs:domain ds:UIBlock ; rdfs:range ds:ModifierFamily .
+ds:modifierFamily a owl:ObjectProperty ; rdfs:domain ds:Modifier ; rdfs:range ds:ModifierFamily .
+ds:hasModifier a owl:ObjectProperty ; rdfs:domain ds:ModifierFamily ; rdfs:range ds:Modifier ;
+  owl:inverseOf ds:modifierFamily ; rdfs:label "hasModifier" .
+
+ds:hasProperty a owl:ObjectProperty ; rdfs:domain ds:UIBlock ; rdfs:range ds:BlockProperty .
+ds:propertyType a owl:DatatypeProperty ; rdfs:domain ds:BlockProperty ; rdfs:range xsd:string .
+ds:optional a owl:DatatypeProperty ; rdfs:domain ds:BlockProperty ; rdfs:range xsd:boolean .
+
+# Domain is ds:Component (not ds:UIBlock) — reached via subtype scoping.
+ds:hasSubcomponent a owl:ObjectProperty ; rdfs:domain ds:Component ; rdfs:range ds:Subcomponent .
+
+# ---- Individuals (ABox) ----
+ds:global a ds:Tier ; ds:name "global" .
+
+ds:button a ds:Component ;
+  ds:name "Button" ;
+  ds:tier ds:global ;
+  ds:summary "Primary action trigger with optional icon and label." ;
+  ds:whenToUse "Use for the primary action on a view." ;
+  ds:whenNotToUse "Do not use for navigation between pages." ;
+  ds:guidelines "Keep labels short and action-oriented." ;
+  ds:anatomyDsl "root: button; children: label, icon" ;
+  ds:anatomyClassic "Button > Label, Icon" ;
+  ds:figmaLink "https://figma.com/design/example/Button" ;
+  ds:hasModifierFamily ds:family.importance, ds:family.density ;
+  ds:hasProperty ds:button.prop.disabled ;
+  ds:hasSubcomponent ds:button.icon .
+
+ds:modal a ds:Component ;
+  ds:name "Modal" ;
+  ds:tier ds:global ;
+  ds:summary "Focused overlay dialog for a single task." ;
+  ds:whenToUse "Use to interrupt for a critical confirmation." ;
+  ds:whenNotToUse "Do not use for non-blocking notifications." ;
+  ds:guidelines "Always provide an explicit close affordance." ;
+  ds:anatomyDsl "root: dialog; children: header, body, footer" ;
+  ds:hasModifierFamily ds:family.size ;
+  ds:hasProperty ds:modal.prop.open .
+
+ds:button.icon a ds:Subcomponent ; ds:name "Button Icon" .
+
+ds:button.prop.disabled a ds:BlockProperty ;
+  ds:name "disabled" ; ds:propertyType "boolean" ; ds:optional true .
+ds:modal.prop.open a ds:BlockProperty ;
+  ds:name "open" ; ds:propertyType "boolean" ; ds:optional false .
+
+# Families carry NO forward ds:hasModifier; values assert only the reverse
+# ds:modifierFamily edge, so the inverse-union resolver must find them.
+ds:family.importance a ds:ModifierFamily ; ds:name "importance" .
+ds:family.density a ds:ModifierFamily ; ds:name "density" .
+ds:family.size a ds:ModifierFamily ; ds:name "size" .
+
+ds:mod.importance.primary a ds:Modifier ; ds:name "primary" ; ds:modifierFamily ds:family.importance .
+ds:mod.importance.secondary a ds:Modifier ; ds:name "secondary" ; ds:modifierFamily ds:family.importance .
+ds:mod.density.compact a ds:Modifier ; ds:name "compact" ; ds:modifierFamily ds:family.density .
+ds:mod.size.small a ds:Modifier ; ds:name "small" ; ds:modifierFamily ds:family.size .
+ds:mod.size.large a ds:Modifier ; ds:name "large" ; ds:modifierFamily ds:family.size .
+`;