@canonical/pragma-cli 0.31.0 → 0.32.0

package/src/testing/behavioral/mcpServe.e2e.test.ts

@@ -0,0 +1,21 @@
+/**
+ * A7 — `pragma mcp` serve boot, observed through the real compiled binary.
+ *
+ * Replaces the old mocked-SIGINT unit test: `runCli` spawns with no `input`,
+ * which gives the child an immediately-EOF stdin (verified equivalent to
+ * `< /dev/null`) — the shape a host that closes the pipe on shutdown produces.
+ * The server must connect and exit cleanly, not hang or crash.
+ */
+
+import { describe, expect, it } from "vitest";
+import { runCli } from "../helpers/runCli.js";
+
+describe("mcp serve — stdin close boots and exits cleanly (A7, e2e)", () => {
+  it("exits 0 with no output and no signal", () => {
+    const result = runCli(["mcp"]);
+    expect(result.exitCode).toBe(0);
+    expect(result.signal).toBeNull();
+    expect(result.stdout).toBe("");
+    expect(result.stderr).toBe("");
+  });
+});

package/src/testing/behavioral/mcpSurface.test.ts

@@ -0,0 +1,139 @@
+/**
+ * B4 — sweep every MCP tool the live catalog exposes: callable with
+ * representative args, returns a well-formed envelope, and its
+ * `readOnlyHint`/`destructiveHint` annotations match the verb's capability.
+ *
+ * This IS PR4's "surface ⊆ covenant" proof for the read layer: rather than
+ * re-asserting the static shape `capabilities/surface.test.ts` (PR3-owned)
+ * already pins, this exercises every one of those blessed tools end-to-end.
+ * Iterates `liveReadSurface.ts` (derived from `emitSurface(capabilities)`) —
+ * no tool name is hard-coded, so this sweep covers PR5/6/7's tools with no
+ * edit here once they land.
+ *
+ * Representative args are built structurally: a lookup/sample's positional
+ * resolves through its noun's OWN `list` (browse-then-address, same as B1); a
+ * `prefix`/`uri`-named positional (the grammar's TBox/entity-inspection
+ * convention, used today by `ontology show`/`graph inspect`) resolves from the
+ * pack index generically, not by noun name. A mutating tool is called WITHOUT
+ * `confirm` (a safe plan-only preview).
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { capabilities } from "../../capabilities/index.js";
+import { readPackIndex } from "../../kernel/completion/entitySource.js";
+import {
+  ALL_VISIBLE_CONFIG,
+  CANONICAL_TTL,
+} from "../fixtures/graph/canonical.js";
+import {
+  bootFixtureRuntime,
+  type FixtureGraph,
+} from "../helpers/fixtureGraph.js";
+import { projectMcp } from "../helpers/projectMcp.js";
+import { type LiveVerb, listVerbs, liveVerbs } from "./liveReadSurface.js";
+
+/** Every verb exposed as an MCP tool — read AND mutating. */
+const everyExposedVerb = liveVerbs.filter((v) => v.tool !== false);
+
+let fixture: FixtureGraph;
+let mcp: Awaited<ReturnType<typeof projectMcp>>;
+/** noun -> first `list` row's name, for lookup/sample positionals. */
+let firstNameByNoun: Map<string, string>;
+/** The registered tool list, fetched once (annotations are call-invariant). */
+let registeredTools: Awaited<ReturnType<typeof mcp.listTools>>;
+
+beforeAll(async () => {
+  fixture = await bootFixtureRuntime({
+    ttl: CANONICAL_TTL,
+    config: ALL_VISIBLE_CONFIG,
+  });
+  mcp = await projectMcp(capabilities, fixture.cwd);
+  registeredTools = await mcp.listTools();
+
+  firstNameByNoun = new Map();
+  for (const v of listVerbs) {
+    const result = await mcp.callTool(v.tool as string);
+    const rows = result.data as { name: string }[] | undefined;
+    const first = rows?.[0]?.name;
+    if (first) firstNameByNoun.set(v.noun, first);
+  }
+});
+
+afterAll(async () => {
+  await mcp.cleanup();
+  await fixture.dispose();
+});
+
+/** Build representative args for EVERY required positional (e.g. config_set's
+ * `<key>` + `<value>`), so a multi-positional tool's zod schema is satisfied. */
+function representativeArgs(v: LiveVerb): Record<string, unknown> {
+  const required = v.spec.params.filter((p) => p.positional && p.required);
+  const args: Record<string, unknown> = {};
+  for (const param of required) {
+    args[param.name] = representativeValue(v, param);
+  }
+  return args;
+}
+
+/** A representative value for one required positional. */
+function representativeValue(
+  v: LiveVerb,
+  param: LiveVerb["spec"]["params"][number],
+): unknown {
+  if (param.name === "prefix") {
+    const prefix = Object.keys(readPackIndex(fixture.cwd)?.prefixes ?? {}).find(
+      (p) => p !== "owl" && p !== "rdfs" && p !== "rdf" && p !== "xsd",
+    );
+    return prefix ?? "ds";
+  }
+  if (param.name === "uri") {
+    const entity = readPackIndex(fixture.cwd)?.entities.find(
+      (e) => e.box === "abox",
+    );
+    return entity?.prefixed ?? entity?.name ?? "";
+  }
+  // An enum positional (e.g. `config channel <name>`, `config set <key>`) must
+  // get a real member — a placeholder would fail the tool's zod schema at the
+  // SDK layer, before the handler, yielding a protocol error rather than a
+  // well-formed envelope.
+  if (param.kind === "enum") {
+    return param.values[0];
+  }
+
+  const known = firstNameByNoun.get(v.noun) ?? "placeholder";
+  return param.kind === "string[]" ? [known] : known;
+}
+
+describe("every MCP tool is callable and returns a well-formed envelope (B4)", () => {
+  it("has tools to sweep", () => {
+    expect(everyExposedVerb.length).toBeGreaterThan(0);
+  });
+
+  it.each(
+    everyExposedVerb,
+  )("$tool: callable, well-formed envelope, annotations match capability.mutates", async (v) => {
+    const tool = v.tool as string;
+    const args = representativeArgs(v);
+    const result = await mcp.callTool(tool, args);
+
+    expect(typeof result.ok).toBe("boolean");
+    if (result.ok) {
+      expect("data" in result).toBe(true);
+      expect(typeof result.meta).toBe("object");
+    } else {
+      const error = result.error as { code?: unknown; message?: unknown };
+      expect(typeof error.code).toBe("string");
+      expect(typeof error.message).toBe("string");
+    }
+
+    const registered = registeredTools.find((t) => t.name === tool);
+    expect(registered).toBeDefined();
+    const annotations = registered?.annotations as
+      | { readOnlyHint?: boolean; destructiveHint?: boolean }
+      | undefined;
+    expect(annotations?.readOnlyHint).toBe(!v.mutates);
+    if (v.spec.capability.destructive !== undefined) {
+      expect(annotations?.destructiveHint).toBe(v.spec.capability.destructive);
+    }
+  });
+});

package/src/testing/behavioral/parity.test.ts

@@ -0,0 +1,224 @@
+/**
+ * A5/A6 (stable-now) — the uniform `assertCliMcpParity` helper proven over the
+ * STORELESS, PR1/PR2 nouns; B5 (commit 4, below) extends it to every read noun
+ * PR3 ships, parameterized over the live surface.
+ *
+ * `info`/`config show` CLI-json==MCP parity is already asserted content-wise by
+ * `capabilities/roundtrip.test.ts` (PR1-owned); this file's job is different —
+ * it is the first REAL exercise of the new shared `helpers/parity.ts` asserter
+ * (so its own correctness is proven before B5 leans on it across every read
+ * noun), plus the plan-first/confirm uniform proof (A6).
+ */
+
+import { existsSync, mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { configModule } from "../../capabilities/config/index.js";
+import { capabilities } from "../../capabilities/index.js";
+import { infoModule } from "../../capabilities/info/index.js";
+import { sourcesModule } from "../../capabilities/sources/index.js";
+import { executeVerb } from "../../kernel/project/cli/dispatch.js";
+import { bootRuntime } from "../../kernel/runtime/boot.js";
+import type { VerbSpec } from "../../kernel/spec/types.js";
+import {
+  fixtureEffectsModule,
+  touchPath,
+} from "../fixtures/fixtureCapability.js";
+import {
+  ALL_VISIBLE_CONFIG,
+  CANONICAL_TTL,
+} from "../fixtures/graph/canonical.js";
+import {
+  bootFixtureRuntime,
+  type FixtureGraph,
+} from "../helpers/fixtureGraph.js";
+import {
+  assertCliMcpParity,
+  JSON_FLAGS,
+  NO_MUTATION,
+} from "../helpers/parity.js";
+import { projectMcp } from "../helpers/projectMcp.js";
+import { listVerbs, liveVerbs, lookupVerbs } from "./liveReadSurface.js";
+
+const freshCwd = (): string => mkdtempSync(join(tmpdir(), "pragma-parity-"));
+
+describe("assertCliMcpParity — the shared helper, over stable-now nouns (A5)", () => {
+  it("info: CLI --format json == MCP tool data", async () => {
+    const envelope = await assertCliMcpParity({
+      modules: [infoModule],
+      verb: infoModule.verbs[0] as VerbSpec,
+      tool: "info",
+      cwd: freshCwd(),
+    });
+    expect(envelope.ok).toBe(true);
+  });
+
+  it("config show: CLI --format json == MCP tool data", async () => {
+    const envelope = await assertCliMcpParity({
+      modules: [configModule],
+      verb: configModule.verbs[0] as VerbSpec,
+      tool: "config_show",
+      cwd: freshCwd(),
+    });
+    expect(envelope.ok).toBe(true);
+  });
+
+  it("sources status: CLI --format json == MCP tool data (storeless)", async () => {
+    const statusVerb = sourcesModule.verbs.find(
+      (v) => v.path[1] === "status",
+    ) as VerbSpec;
+    const envelope = await assertCliMcpParity({
+      modules: [sourcesModule],
+      verb: statusVerb,
+      tool: "sources_status",
+      cwd: freshCwd(),
+    });
+    expect(envelope.ok).toBe(true);
+  });
+});
+
+describe("plan-first is uniform across surfaces (A6)", () => {
+  it("CLI --dry-run and MCP no-confirm describe the SAME effects, in different meta shapes", async () => {
+    const cwd = freshCwd();
+    const touchVerb = fixtureEffectsModule.verbs.find(
+      (v) => v.path[1] === "touch",
+    ) as VerbSpec;
+    const name = `parity-plan-${Date.now()}`;
+
+    const cli = await executeVerb(
+      touchVerb,
+      { name },
+      { dryRun: true, undo: false, yes: false },
+      bootRuntime(JSON_FLAGS, cwd),
+    );
+    const cliEnvelope = JSON.parse(cli.stdout as string) as {
+      data: { plan: string[] };
+      meta: Record<string, unknown>;
+    };
+
+    const mcp = await projectMcp([fixtureEffectsModule], cwd);
+    const mcpEnvelope = (await mcp.callTool("probe_touch", { name })) as {
+      data: { plan: string[] };
+      meta: Record<string, unknown>;
+    };
+    await mcp.cleanup();
+
+    // Same described effects...
+    expect(cliEnvelope.data.plan).toEqual(mcpEnvelope.data.plan);
+    // ...under intentionally DIFFERENT meta shapes (PARITY_GAPS: plan-first-meta-differs) —
+    // a CLI --dry-run and an unconfirmed MCP call are different REQUESTS (one
+    // explicit, one implicit), so the envelope names the mode differently.
+    expect(cliEnvelope.meta).toEqual({ dryRun: true });
+    expect(mcpEnvelope.meta).toEqual({ planOnly: true, confirmRequired: true });
+    expect(existsSync(touchPath(name))).toBe(false);
+  });
+
+  it("both surfaces perform the SAME real effect once confirmed", async () => {
+    const cwd = freshCwd();
+    const touchVerb = fixtureEffectsModule.verbs.find(
+      (v) => v.path[1] === "touch",
+    ) as VerbSpec;
+
+    const cliName = `parity-confirm-cli-${Date.now()}`;
+    await executeVerb(
+      touchVerb,
+      { name: cliName },
+      NO_MUTATION,
+      bootRuntime(JSON_FLAGS, cwd),
+    );
+    expect(existsSync(touchPath(cliName))).toBe(true);
+
+    const mcpName = `parity-confirm-mcp-${Date.now()}`;
+    const mcp = await projectMcp([fixtureEffectsModule], cwd);
+    const mcpEnvelope = await mcp.callTool("probe_touch", {
+      name: mcpName,
+      confirm: true,
+    });
+    await mcp.cleanup();
+    expect(mcpEnvelope.ok).toBe(true);
+    expect(existsSync(touchPath(mcpName))).toBe(true);
+  });
+});
+
+/**
+ * B5 — the uniform parity helper extended across EVERY read noun the live
+ * surface exposes, over the shared canonical fixture. Parameterized via
+ * `liveReadSurface.ts` (never a noun copied from the plan); also sweeps any
+ * extra list-shaped verb (e.g. `categories`) and `sample`, found the same way.
+ *
+ * RETIRES the old byte-exact `condensed === fmt.llm(...)` parity (R4 — v2 has
+ * no condensed MCP shape, PARITY_GAPS `no-condensed-mcp-envelope`): this is
+ * structural `--format json` deep-equality via `assertCliMcpParity`, exactly
+ * like A5/A6 above — B5 is that same proof, just swept over every read noun.
+ */
+describe("read-noun parity — every list/lookup/extra verb (B5)", () => {
+  let fixture: FixtureGraph;
+  let firstNameByNoun: Map<string, string>;
+
+  // `sample` draws an independent RANDOM selection per call (`pickRandom`) —
+  // two separate invocations (CLI, MCP) are not expected to return identical
+  // entities, so it is excluded from content-equality parity (its STRUCTURE
+  // is covered by B4's callable-envelope sweep instead). Only deterministic
+  // extra list-shaped verbs (e.g. `categories`) belong here.
+  const extraListShapedVerbs = liveVerbs.filter(
+    (v) => v.verb === "categories" && v.tool !== false && v.needsStore,
+  );
+
+  beforeAll(async () => {
+    fixture = await bootFixtureRuntime({
+      ttl: CANONICAL_TTL,
+      config: ALL_VISIBLE_CONFIG,
+    });
+    firstNameByNoun = new Map();
+    for (const v of listVerbs) {
+      const out = await executeVerb(
+        v.spec,
+        {},
+        NO_MUTATION,
+        bootRuntime(JSON_FLAGS, fixture.cwd),
+      );
+      const rows = JSON.parse(out.stdout as string).data as { name: string }[];
+      const first = rows[0]?.name;
+      if (first) firstNameByNoun.set(v.noun, first);
+    }
+  });
+
+  afterAll(async () => {
+    await fixture.dispose();
+  });
+
+  it.each(listVerbs)("$noun list: CLI --format json == MCP", async (v) => {
+    await assertCliMcpParity({
+      modules: capabilities,
+      verb: v.spec,
+      tool: v.tool as string,
+      cwd: fixture.cwd,
+    });
+  });
+
+  it.each(
+    lookupVerbs,
+  )("$noun lookup: CLI --format json == MCP, for a known name", async (v) => {
+    const name = firstNameByNoun.get(v.noun);
+    if (!name) return;
+    await assertCliMcpParity({
+      modules: capabilities,
+      verb: v.spec,
+      tool: v.tool as string,
+      cwd: fixture.cwd,
+      params: { name: [name] },
+    });
+  });
+
+  it.each(
+    extraListShapedVerbs,
+  )("$noun $verb: CLI --format json == MCP", async (v) => {
+    await assertCliMcpParity({
+      modules: capabilities,
+      verb: v.spec,
+      tool: v.tool as string,
+      cwd: fixture.cwd,
+    });
+  });
+});

package/src/testing/behavioral/rootCli.e2e.test.ts

@@ -0,0 +1,62 @@
+/**
+ * A2 — `--version` at every argv position; A3 — unknown-verb suggestions and
+ * global-flag-before-noun ordering. Both are `bin.ts` argv-shape behaviors,
+ * observed only correctly through the real process boundary (Commander parse
+ * + the bin's early-exit ladder), so both are spawn-e2e.
+ */
+
+import { describe, expect, it } from "vitest";
+import { VERSION } from "../../constants.js";
+import { runCli } from "../helpers/runCli.js";
+
+describe("--version at every level (A2, e2e)", () => {
+  it("prints the same semver at root, noun, and verb position", () => {
+    const root = runCli(["--version"]);
+    const atNoun = runCli(["config", "--version"]);
+    const atVerb = runCli(["config", "show", "--version"]);
+
+    for (const result of [root, atNoun, atVerb]) {
+      expect(result.exitCode).toBe(0);
+      expect(result.stdout.trim()).toBe(VERSION);
+    }
+    expect(VERSION).toMatch(/^\d+\.\d+\.\d+/);
+  });
+
+  it("-v is equivalent to --version", () => {
+    expect(runCli(["-v"]).stdout.trim()).toBe(VERSION);
+  });
+});
+
+describe("unknown verb / noun → suggestion + exit 2 (A3, e2e)", () => {
+  it("an unknown noun ranks the nearest known noun", () => {
+    const result = runCli(["blck"]);
+    expect(result.exitCode).toBe(2);
+    expect(result.stderr).toContain('Unknown command "blck"');
+    expect(result.stderr).toContain("Did you mean?");
+    expect(result.stderr).toContain("- block");
+  });
+
+  it("an unknown verb under a known noun ranks that noun's verbs", () => {
+    // "sho" is a PREFIX of config's one verb, "show" — guaranteed to rank
+    // (suggestNames scores a prefix match ahead of edit-distance matches).
+    const result = runCli(["config", "sho"]);
+    expect(result.exitCode).toBe(2);
+    expect(result.stderr).toContain('Unknown command "sho"');
+    expect(result.stderr).toContain("- show");
+  });
+});
+
+describe("a global flag before the noun still works (A3, e2e)", () => {
+  it("--format json ahead of the command is not mistaken for the noun", () => {
+    const result = runCli(["--format", "json", "config", "show"]);
+    expect(result.exitCode).toBe(0);
+    const envelope = JSON.parse(result.stdout) as { ok: boolean };
+    expect(envelope.ok).toBe(true);
+  });
+
+  it("matches the equivalent flag-after-command invocation", () => {
+    const before = runCli(["--format", "json", "info"]);
+    const after = runCli(["info", "--format", "json"]);
+    expect(JSON.parse(before.stdout)).toEqual(JSON.parse(after.stdout));
+  });
+});

package/src/testing/eval/__snapshots__/eval.test.ts.snap

@@ -0,0 +1,201 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`eval harness — seed gate > runs the full seed (stable + read-noun) cases clean 1`] = `
+[
+  {
+    "id": "content-canonical-graph-has-4-components",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-capabilities-counts-derived-from-categories",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-capabilities-covers-listtools",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-capabilities-selfconsistent-with-emitsurface",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-code-function-purity-description-mentions-pure",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-condensed-sdl-token-budget",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-handshake-advertises-tools-resources-prompts",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-importance-family-includes-primary",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-instructions-present-and-mentions-discovery",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-native-prompt-get-fills-arguments",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-native-prompts-list-nonempty",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-normal-channel-drops-beta-only-block",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "content-standard-list-length-equals-category-count-sum",
+    "kind": "content",
+    "passed": true,
+  },
+  {
+    "id": "disclosure-block-lookup-detailed-adds-anatomy",
+    "kind": "disclosure",
+    "passed": true,
+  },
+  {
+    "id": "disclosure-summary-omits-standard-and-detailed-fields",
+    "kind": "disclosure",
+    "passed": true,
+  },
+  {
+    "id": "prompt-block-question-maps-to-block-lookup",
+    "kind": "prompt",
+    "passed": true,
+  },
+  {
+    "id": "prompt-browse-prompts-maps-to-prompt-list",
+    "kind": "prompt",
+    "passed": true,
+  },
+  {
+    "id": "prompt-config-question-maps-to-config-show",
+    "kind": "prompt",
+    "passed": true,
+  },
+  {
+    "id": "prompt-environment-maps-to-doctor",
+    "kind": "prompt",
+    "passed": true,
+  },
+  {
+    "id": "prompt-scaffold-maps-to-create-component",
+    "kind": "prompt",
+    "passed": true,
+  },
+  {
+    "id": "prompt-set-tier-maps-to-config-tier",
+    "kind": "prompt",
+    "passed": true,
+  },
+  {
+    "id": "prompt-sparql-maps-to-graph-query",
+    "kind": "prompt",
+    "passed": true,
+  },
+  {
+    "id": "prompt-tokens-config-maps-to-token-add-config",
+    "kind": "prompt",
+    "passed": true,
+  },
+  {
+    "id": "prompt-version-check-maps-to-info",
+    "kind": "prompt",
+    "passed": true,
+  },
+  {
+    "id": "tool-block-lookup-button-has-modifier-families",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-block-sample-returns-exemplars",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-capabilities-annotates-every-live-tool",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-config-tier-is-plan-first",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-doctor-reports-check-tallies",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-graph-query-counts-components",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-graph-query-invalid-is-invalid-input",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-modifier-sample-returns-exemplars",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-prompt-list-resolves-authored-prompts",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-sparql-count-positive",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-standard-lookup-has-dos-and-donts",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-tier-lookup-lists-scoped-blocks",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-token-add-config-is-plan-first",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-token-sample-returns-exemplars",
+    "kind": "tool",
+    "passed": true,
+  },
+  {
+    "id": "tool-upgrade-is-plan-first",
+    "kind": "tool",
+    "passed": true,
+  },
+]
+`;