@canonical/pragma-cli 0.31.0 → 0.33.0

package/src/testing/behavioral/journeys.livePack.e2e.test.ts

@@ -0,0 +1,129 @@
+/**
+ * E3 (AV-231, Backlog E) — the network-gated, NON-BLOCKING live-pack job.
+ *
+ * A handful of behaviours can only be confirmed against a REAL oxigraph store
+ * built from the REAL default pack: result ordering (A3/A9), which language a
+ * multilingual label resolves to under real store order (A7), ke truncation of
+ * long literals, and `buildIndex`'s `Promise.all` re-entrancy when the same pack
+ * is built concurrently. Those depend on the network (fetching the shipped pack)
+ * and on real build cost, so this suite is GATED OFF by default — it never runs
+ * (or flakes) in the ordinary `bun test` pass. Set `PRAGMA_LIVE_PACK=1` to run
+ * it (a dedicated, non-required CI job).
+ *
+ * The assertions here run against the VENDORED default pack as a faithful local
+ * proxy for the shipped one, so a maintainer who flips the gate gets real signal
+ * today. Each carries a TODO for wiring the ACTUAL network pack fetch — the one
+ * piece this environment cannot provide.
+ *
+ * TODO(AV-231/E3): resolve the real shipped default pack (git/npm source) here
+ * instead of the vendored TTL, so ordering/truncation are asserted against the
+ * bytes agents actually receive.
+ */
+
+import { describe, expect, it } from "vitest";
+import { buildPack } from "../../kernel/runtime/graphpack/build.js";
+import {
+  DEFAULT_PACK_PREFIXES,
+  DEFAULT_PACK_TTL,
+} from "../fixtures/graph/defaultPack.js";
+import {
+  bootFixtureRuntime,
+  type FixtureGraph,
+} from "../helpers/fixtureGraph.js";
+
+/** Non-blocking gate: skipped unless a maintainer opts in (or CI's live job). */
+const liveGate = process.env.PRAGMA_LIVE_PACK ? describe : describe.skip;
+
+liveGate(
+  "live-pack journey — real oxigraph ordering + build re-entrancy (E3)",
+  () => {
+    it("a SELECT ... ORDER BY returns a STABLE, deterministic ordering (A3/A9)", async () => {
+      const fixture = await bootFixtureRuntime({ ttl: DEFAULT_PACK_TTL });
+      try {
+        const query =
+          "SELECT ?name WHERE { ?c a ds:Component ; ds:name ?name } ORDER BY ?name";
+        const first = await fixture.runtime.query.sparql(query);
+        const second = await fixture.runtime.query.sparql(query);
+        const names = (result: typeof first): string[] =>
+          result.type === "select"
+            ? result.bindings.map((binding) => String(binding.name))
+            : [];
+        // Repeated evaluation is identical, and ORDER BY is honoured.
+        expect(names(first)).toEqual(names(second));
+        expect(names(first)).toEqual(["Beta Badge", "Button", "Orphan Widget"]);
+      } finally {
+        await fixture.dispose();
+      }
+    });
+
+    it("a multilingual label resolves to a SINGLE, tag-stripped value under real store order (A7)", async () => {
+      // TODO(AV-231/E3): against the live pack, pin the EXACT language the store
+      // yields first, so the index-label choice is a confirmed contract rather than
+      // the store-order accident it is today (see the A7 note in the E1 journey).
+      const fixture = await bootFixtureRuntime({ ttl: DEFAULT_PACK_TTL });
+      try {
+        const result = await fixture.runtime.query.sparql(
+          "SELECT ?l WHERE { ds:button rdfs:label ?l } ORDER BY ?l",
+        );
+        const labels =
+          result.type === "select"
+            ? result.bindings.map((binding) => String(binding.l))
+            : [];
+        expect(labels).toEqual(["Bouton", "Button"]);
+      } finally {
+        await fixture.dispose();
+      }
+    });
+
+    it("building the same sources CONCURRENTLY is re-entrant — one hash, no torn pack (buildIndex Promise.all)", async () => {
+      // buildIndex fans out four bulk SPARQL queries under Promise.all, and buildPack
+      // publishes atomically; N concurrent builds of identical sources must converge
+      // on ONE content hash with every pack complete (no half-written cache).
+      const inputs = [
+        { path: "concurrent/pack.ttl", content: DEFAULT_PACK_TTL },
+      ];
+      const options = {
+        name: "pragma",
+        version: "0.0.0-e3",
+        sourceRef: "e3-reentrancy",
+        prefixes: DEFAULT_PACK_PREFIXES,
+      };
+      const results = await Promise.all(
+        Array.from({ length: 5 }, () => buildPack(inputs, options)),
+      );
+      const hashes = new Set(results.map((result) => result.contentHash));
+      expect(hashes.size).toBe(1);
+      for (const result of results) {
+        expect(result.contentHash).toMatch(/^[0-9a-f]{64}$/);
+      }
+    });
+
+    it("a long literal survives the build round-trip without silent ke truncation", async () => {
+      // TODO(AV-231/E3): confirm against the live pack, whose real summaries/guidelines
+      // are long enough to trip any downstream ke truncation (A7/A9 tail).
+      const longText = "x".repeat(20_000);
+      const ttl = `@prefix ds: <https://ds.canonical.com/> .
+@prefix owl: <http://www.w3.org/2002/07/owl#> .
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
+ds:Component a owl:Class .
+ds:summary a owl:DatatypeProperty ; rdfs:range xsd:string .
+ds:big a ds:Component ; ds:summary "${longText}" .`;
+      let fixture: FixtureGraph | undefined;
+      try {
+        fixture = await bootFixtureRuntime({ ttl });
+        const result = await fixture.runtime.query.sparql(
+          "SELECT ?s WHERE { ds:big ds:summary ?s }",
+        );
+        const value =
+          result.type === "select"
+            ? String(
+                (result.bindings.at(0) as { s?: string } | undefined)?.s ?? "",
+              )
+            : "";
+        expect(value.length).toBe(longText.length);
+      } finally {
+        await fixture?.dispose();
+      }
+    });
+  },
+);

package/src/testing/behavioral/journeys.mcpLifecycle.mcp.test.ts

@@ -0,0 +1,176 @@
+/**
+ * E2 (AV-231, Backlog E) — the MCP lifecycle journey over a LONG-LIVED,
+ * in-process server.
+ *
+ * Fresh-runtime unit tests structurally cannot catch the MCP memoization bugs:
+ * they build a new runtime per assertion, so a store/config memo that outlives a
+ * mutation is never exercised. The real MCP server boots ONE runtime for its
+ * whole lifetime and shares it across every tool call — the exact condition
+ * under which a cold-store rejection could be memoized forever, or a stale config
+ * layer could survive a `config_set`. This file drives that server the way an
+ * agent does, over a single connection:
+ *
+ *  - COLD read → STORE_UNAVAILABLE → `sources_update` (confirm) → the SAME
+ *    server's retry read SUCCEEDS. Guards C1: a rejected boot must NOT be
+ *    memoized, and a real mutation must invalidate the shared store so the next
+ *    read re-boots against the freshly built pack.
+ *  - `config_set` → a subsequent `config_show` on the same server sees the new
+ *    value. Guards C2: the mutation must drop the config memo the store boot
+ *    depends on.
+ *  - The discovery sequence resolves from a COLD store (the storeless
+ *    `capabilities` catalog boots with no pack), then browse → inspect works once
+ *    the store is warm.
+ *
+ * Uses the in-process `projectMcp` harness (SDK in-memory transport) — no stdio,
+ * no network — over the vendored default pack.
+ */
+
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { capabilities } from "../../capabilities/index.js";
+import { readLock } from "../../kernel/runtime/lock.js";
+import { DEFAULT_PACK_TTL } from "../fixtures/graph/defaultPack.js";
+import { type McpHarness, projectMcp } from "../helpers/projectMcp.js";
+
+/** The transient roots one test creates; removed in `afterEach`. */
+let roots: string[] = [];
+const tempDir = (prefix: string): string => {
+  const dir = mkdtempSync(join(tmpdir(), prefix));
+  roots.push(dir);
+  return dir;
+};
+
+/**
+ * A COLD project directory: a `pragma.config.ts` pointing at a real file
+ * package, but NO lock yet — so the store boots cold until `sources_update`.
+ */
+function coldProject(ttl: string): string {
+  const pkg = tempDir("e2-pkg-");
+  mkdirSync(join(pkg, "definitions"), { recursive: true });
+  writeFileSync(join(pkg, "definitions", "pack.ttl"), ttl);
+  const cwd = tempDir("e2-proj-");
+  writeFileSync(
+    join(cwd, "pragma.config.ts"),
+    `export default { packages: [{ name: "default", source: "file://${pkg}" }] };\n`,
+  );
+  return cwd;
+}
+
+let mcp: McpHarness | undefined;
+beforeEach(() => {
+  roots = [];
+});
+afterEach(async () => {
+  await mcp?.cleanup();
+  mcp = undefined;
+  for (const dir of roots) rmSync(dir, { recursive: true, force: true });
+});
+
+describe("MCP lifecycle — cold store recovers after sources_update (E2, C1)", () => {
+  it("a cold read fails STORE_UNAVAILABLE with the sources_update recovery, then the retry succeeds", async () => {
+    const cwd = coldProject(DEFAULT_PACK_TTL);
+    mcp = await projectMcp(capabilities, cwd);
+
+    // 1) COLD: the store has no pack — a store-backed read must fail cleanly,
+    //    pointing the agent at the recovery TOOL (not just a CLI string).
+    const cold = await mcp.callTool("block_list");
+    expect(cold.ok).toBe(false);
+    const error = cold.error as {
+      code: string;
+      recovery?: { mcp?: { tool?: string } };
+    };
+    expect(error.code).toBe("STORE_UNAVAILABLE");
+    expect(error.recovery?.mcp?.tool).toBe("sources_update");
+    expect(readLock(cwd)).toBeUndefined();
+
+    // 2) The agent runs the recovery tool for real (confirm) — it builds the pack.
+    const update = await mcp.callTool("sources_update", { confirm: true });
+    expect(update.ok).toBe(true);
+    // A real execution, not a plan-only preview.
+    expect(update.meta).not.toMatchObject({ planOnly: true });
+    expect((update.data as { contentHash: string }).contentHash).toMatch(
+      /^[0-9a-f]{64}$/,
+    );
+    expect(readLock(cwd)?.contentHash).toMatch(/^[0-9a-f]{64}$/);
+
+    // 3) RETRY on the SAME server: the shared store was invalidated by the
+    //    mutation, so the memoized cold rejection is gone and the read re-boots
+    //    against the new pack. This is the guarantee a fresh-runtime test cannot
+    //    make.
+    const warm = await mcp.callTool("block_list");
+    expect(warm.ok).toBe(true);
+    const names = (warm.data as { name: string }[])
+      .map((row) => row.name)
+      .sort();
+    expect(names).toEqual(["Button", "Card"]);
+  });
+});
+
+describe("MCP lifecycle — config_set is visible to the next read (E2, C2)", () => {
+  it("config_set updates the layer a subsequent config_show reads on the same server", async () => {
+    const cwd = tempDir("e2-cfg-");
+    mcp = await projectMcp(capabilities, cwd);
+
+    // Before: the default channel, from the default (unset) origin.
+    const before = await mcp.callTool("config_show");
+    const beforeData = before.data as {
+      config: { channel: string };
+      origins: { channel: string };
+    };
+    expect(beforeData.config.channel).toBe("normal");
+    expect(beforeData.origins.channel).toBe("default");
+
+    // Mutate: set the channel for real (confirm).
+    const set = await mcp.callTool("config_set", {
+      key: "channel",
+      value: "experimental",
+      confirm: true,
+    });
+    expect(set.ok).toBe(true);
+
+    // After: the SAME long-lived server's read reflects the write — proving the
+    // mutation dropped the config memo the boot depends on (not a stale layer).
+    const after = await mcp.callTool("config_show");
+    const afterData = after.data as {
+      config: { channel: string };
+      origins: { channel: string };
+    };
+    expect(afterData.config.channel).toBe("experimental");
+    // The write landed in the global layer, so the origin flips accordingly.
+    expect(afterData.origins.channel).toBe("global");
+  });
+});
+
+describe("MCP lifecycle — discovery from a cold store (E2)", () => {
+  it("the capabilities catalog resolves cold, then browse -> inspect works once warm", async () => {
+    const cwd = coldProject(DEFAULT_PACK_TTL);
+    mcp = await projectMcp(capabilities, cwd);
+
+    // Discovery is storeless: the capabilities catalog answers even with a cold
+    // store, so an agent can orient before the pack exists.
+    const catalog = await mcp.callTool("capabilities");
+    expect(catalog.ok).toBe(true);
+    const catalogData = catalog.data as {
+      discovery_sequence: unknown;
+      tools: unknown;
+    };
+    expect(catalogData.discovery_sequence).toBeDefined();
+    expect(catalogData.tools).toBeDefined();
+
+    // Warm the store, then follow a browse -> inspect discovery step end to end.
+    await mcp.callTool("sources_update", { confirm: true });
+    const list = await mcp.callTool("block_list");
+    expect(list.ok).toBe(true);
+    const first = (list.data as { name: string }[]).at(0);
+    expect(first?.name).toBeDefined();
+
+    const lookup = await mcp.callTool("block_lookup", {
+      name: [first?.name ?? ""],
+    });
+    expect(lookup.ok).toBe(true);
+    const results = (lookup.data as { results: { name: string }[] }).results;
+    expect(results.at(0)?.name).toBe(first?.name);
+  });
+});

package/src/testing/behavioral/liveReadSurface.ts

@@ -0,0 +1,80 @@
+/**
+ * The live read-noun surface, parameterized (R2 discipline).
+ *
+ * Every B-tier test in this directory drives its assertions over THIS —
+ * never a noun list copied from the plan. It cross-references
+ * `emitSurface(capabilities)` (the authoritative "what's live, what's hidden,
+ * what's MCP-exposed" projection) against the raw `capabilities` array (for
+ * the runnable `VerbSpec`), so the derived lists below automatically grow as
+ * PR5/6/7 add nouns — nothing here names a specific noun.
+ */
+
+import { capabilities } from "../../capabilities/index.js";
+import { verbKey } from "../../kernel/packs/uniqueness.js";
+import { emitSurface } from "../../kernel/spec/emitSurface.js";
+import type { VerbSpec } from "../../kernel/spec/types.js";
+
+/** One live, non-hidden verb: its emitted facts plus the runnable spec. */
+export interface LiveVerb {
+  readonly noun: string;
+  /** The verb label (`emitSurface`'s `v` — the noun itself for a self-verb). */
+  readonly verb: string;
+  /** The MCP tool name, or `false` when withheld from MCP. */
+  readonly tool: string | false;
+  readonly needsStore: boolean;
+  readonly mutates: boolean;
+  readonly spec: VerbSpec;
+}
+
+function buildLiveVerbs(): readonly LiveVerb[] {
+  const specByKey = new Map<string, VerbSpec>();
+  for (const module of capabilities) {
+    for (const verb of module.verbs) {
+      if (verb.hidden) continue;
+      specByKey.set(verbKey(verb.path), verb);
+    }
+  }
+
+  const emitted = emitSurface(capabilities);
+  const live: LiveVerb[] = [];
+  for (const [noun, { verbs }] of Object.entries(emitted.nouns)) {
+    for (const entry of verbs) {
+      const key = entry.v === noun ? noun : `${noun} ${entry.v}`;
+      const spec = specByKey.get(key);
+      if (!spec) continue; // Should not happen — emission mirrors the registry.
+      live.push({
+        noun,
+        verb: entry.v,
+        tool: entry.mcp ?? false,
+        needsStore: entry.needsStore === true,
+        mutates: entry.mutates === true,
+        spec,
+      });
+    }
+  }
+  return live;
+}
+
+/** Every live, non-hidden verb across the whole capability catalog. */
+export const liveVerbs: readonly LiveVerb[] = buildLiveVerbs();
+
+/**
+ * Read nouns' `lookup` verbs, MCP-exposed, store-backed — restricted to the
+ * PACK-lookup contract (a variadic `<name...>` positional), which the B-tier
+ * list→lookup→batch suites drive uniformly. Bespoke single-name lookups
+ * (`tier_lookup`, `prompt_lookup`) have their own tests, so they are excluded
+ * here structurally (by param shape, not by naming the noun — R2).
+ */
+export const lookupVerbs: readonly LiveVerb[] = liveVerbs.filter(
+  (v) =>
+    v.verb === "lookup" &&
+    v.tool !== false &&
+    v.needsStore &&
+    !v.mutates &&
+    v.spec.params.some((p) => p.positional && p.kind === "string[]"),
+);
+
+/** Read nouns' `list` verbs, MCP-exposed, store-backed. */
+export const listVerbs: readonly LiveVerb[] = liveVerbs.filter(
+  (v) => v.verb === "list" && v.tool !== false && v.needsStore && !v.mutates,
+);

package/src/testing/behavioral/mcpServe.e2e.test.ts

@@ -0,0 +1,21 @@
+/**
+ * A7 — `pragma mcp` serve boot, observed through the real compiled binary.
+ *
+ * Replaces the old mocked-SIGINT unit test: `runCli` spawns with no `input`,
+ * which gives the child an immediately-EOF stdin (verified equivalent to
+ * `< /dev/null`) — the shape a host that closes the pipe on shutdown produces.
+ * The server must connect and exit cleanly, not hang or crash.
+ */
+
+import { describe, expect, it } from "vitest";
+import { runCli } from "../helpers/runCli.js";
+
+describe("mcp serve — stdin close boots and exits cleanly (A7, e2e)", () => {
+  it("exits 0 with no output and no signal", () => {
+    const result = runCli(["mcp"]);
+    expect(result.exitCode).toBe(0);
+    expect(result.signal).toBeNull();
+    expect(result.stdout).toBe("");
+    expect(result.stderr).toBe("");
+  });
+});

package/src/testing/behavioral/mcpSurface.test.ts

@@ -0,0 +1,139 @@
+/**
+ * B4 — sweep every MCP tool the live catalog exposes: callable with
+ * representative args, returns a well-formed envelope, and its
+ * `readOnlyHint`/`destructiveHint` annotations match the verb's capability.
+ *
+ * This IS PR4's "surface ⊆ covenant" proof for the read layer: rather than
+ * re-asserting the static shape `capabilities/surface.test.ts` (PR3-owned)
+ * already pins, this exercises every one of those blessed tools end-to-end.
+ * Iterates `liveReadSurface.ts` (derived from `emitSurface(capabilities)`) —
+ * no tool name is hard-coded, so this sweep covers PR5/6/7's tools with no
+ * edit here once they land.
+ *
+ * Representative args are built structurally: a lookup/sample's positional
+ * resolves through its noun's OWN `list` (browse-then-address, same as B1); a
+ * `prefix`/`uri`-named positional (the grammar's TBox/entity-inspection
+ * convention, used today by `ontology show`/`graph inspect`) resolves from the
+ * pack index generically, not by noun name. A mutating tool is called WITHOUT
+ * `confirm` (a safe plan-only preview).
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { capabilities } from "../../capabilities/index.js";
+import { readPackIndex } from "../../kernel/completion/entitySource.js";
+import {
+  ALL_VISIBLE_CONFIG,
+  CANONICAL_TTL,
+} from "../fixtures/graph/canonical.js";
+import {
+  bootFixtureRuntime,
+  type FixtureGraph,
+} from "../helpers/fixtureGraph.js";
+import { projectMcp } from "../helpers/projectMcp.js";
+import { type LiveVerb, listVerbs, liveVerbs } from "./liveReadSurface.js";
+
+/** Every verb exposed as an MCP tool — read AND mutating. */
+const everyExposedVerb = liveVerbs.filter((v) => v.tool !== false);
+
+let fixture: FixtureGraph;
+let mcp: Awaited<ReturnType<typeof projectMcp>>;
+/** noun -> first `list` row's name, for lookup/sample positionals. */
+let firstNameByNoun: Map<string, string>;
+/** The registered tool list, fetched once (annotations are call-invariant). */
+let registeredTools: Awaited<ReturnType<typeof mcp.listTools>>;
+
+beforeAll(async () => {
+  fixture = await bootFixtureRuntime({
+    ttl: CANONICAL_TTL,
+    config: ALL_VISIBLE_CONFIG,
+  });
+  mcp = await projectMcp(capabilities, fixture.cwd);
+  registeredTools = await mcp.listTools();
+
+  firstNameByNoun = new Map();
+  for (const v of listVerbs) {
+    const result = await mcp.callTool(v.tool as string);
+    const rows = result.data as { name: string }[] | undefined;
+    const first = rows?.[0]?.name;
+    if (first) firstNameByNoun.set(v.noun, first);
+  }
+});
+
+afterAll(async () => {
+  await mcp.cleanup();
+  await fixture.dispose();
+});
+
+/** Build representative args for EVERY required positional (e.g. config_set's
+ * `<key>` + `<value>`), so a multi-positional tool's zod schema is satisfied. */
+function representativeArgs(v: LiveVerb): Record<string, unknown> {
+  const required = v.spec.params.filter((p) => p.positional && p.required);
+  const args: Record<string, unknown> = {};
+  for (const param of required) {
+    args[param.name] = representativeValue(v, param);
+  }
+  return args;
+}
+
+/** A representative value for one required positional. */
+function representativeValue(
+  v: LiveVerb,
+  param: LiveVerb["spec"]["params"][number],
+): unknown {
+  if (param.name === "prefix") {
+    const prefix = Object.keys(readPackIndex(fixture.cwd)?.prefixes ?? {}).find(
+      (p) => p !== "owl" && p !== "rdfs" && p !== "rdf" && p !== "xsd",
+    );
+    return prefix ?? "ds";
+  }
+  if (param.name === "uri") {
+    const entity = readPackIndex(fixture.cwd)?.entities.find(
+      (e) => e.box === "abox",
+    );
+    return entity?.prefixed ?? entity?.name ?? "";
+  }
+  // An enum positional (e.g. `config set <key>`) must
+  // get a real member — a placeholder would fail the tool's zod schema at the
+  // SDK layer, before the handler, yielding a protocol error rather than a
+  // well-formed envelope.
+  if (param.kind === "enum") {
+    return param.values[0];
+  }
+
+  const known = firstNameByNoun.get(v.noun) ?? "placeholder";
+  return param.kind === "string[]" ? [known] : known;
+}
+
+describe("every MCP tool is callable and returns a well-formed envelope (B4)", () => {
+  it("has tools to sweep", () => {
+    expect(everyExposedVerb.length).toBeGreaterThan(0);
+  });
+
+  it.each(
+    everyExposedVerb,
+  )("$tool: callable, well-formed envelope, annotations match capability.mutates", async (v) => {
+    const tool = v.tool as string;
+    const args = representativeArgs(v);
+    const result = await mcp.callTool(tool, args);
+
+    expect(typeof result.ok).toBe("boolean");
+    if (result.ok) {
+      expect("data" in result).toBe(true);
+      expect(typeof result.meta).toBe("object");
+    } else {
+      const error = result.error as { code?: unknown; message?: unknown };
+      expect(typeof error.code).toBe("string");
+      expect(typeof error.message).toBe("string");
+    }
+
+    const registered = registeredTools.find((t) => t.name === tool);
+    expect(registered).toBeDefined();
+    const annotations = registered?.annotations as
+      | { readOnlyHint?: boolean; destructiveHint?: boolean }
+      | undefined;
+    expect(annotations?.readOnlyHint).toBe(!v.mutates);
+    if (v.spec.capability.destructive !== undefined) {
+      expect(annotations?.destructiveHint).toBe(v.spec.capability.destructive);
+    }
+  });
+});