@canonical/pragma-cli 0.30.0 → 0.32.0

package/src/testing/helpers/runCli.ts

@@ -0,0 +1,128 @@
+/**
+ * Test helper: spawn-and-CAPTURE the real `pragma` binary.
+ *
+ * {@link measureCommand} (the perf helper) discards stdout/stderr — it only
+ * times spawns. This is the complementary helper for tests that need to READ
+ * what the process printed: the true end-to-end boundary (argv parsing,
+ * first-run, the compiled/`bun` entry point, real process exit codes).
+ *
+ * Kept THIN — most behavioral coverage should run in-process via
+ * `executeVerb`/`projectCli`/`projectMcp`; reserve `runCli` for tests where the
+ * PROCESS BOUNDARY itself is under test (first-run banners, `--version`,
+ * real exit codes, `mcp` serve boot).
+ */
+
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+/** The compiled binary the perf `globalSetup` guarantees exists before tests run. */
+const COMPILED_BINARY = fileURLToPath(
+  new URL("../../../dist/pragma", import.meta.url),
+);
+
+/** The `bin.ts` entry point, for fast source-mode spawns (no rebuild). */
+const SOURCE_ENTRY = fileURLToPath(new URL("../../bin.ts", import.meta.url));
+
+/** Options for {@link runCli}. */
+export interface RunCliOptions {
+  /** Working directory for the spawned process (defaults to the current one). */
+  readonly cwd?: string;
+  /**
+   * Extra/overriding environment variables. Wins over the seeded defaults, so a
+   * test that wants a FRESH (unseeded) `XDG_CONFIG_HOME` — e.g. to observe
+   * first-run — passes its own here.
+   */
+  readonly env?: Record<string, string | undefined>;
+  /**
+   * `"compiled"` (default) spawns `dist/pragma` — the true release boundary.
+   * `"source"` spawns `bun src/bin.ts` — faster, no rebuild required, for
+   * journeys that do not test the compiled-binary boundary itself.
+   */
+  readonly mode?: "compiled" | "source";
+  /** Spawn timeout in milliseconds (default 20000). */
+  readonly timeoutMs?: number;
+}
+
+/** The captured outcome of a `runCli` invocation. */
+export interface RunCliResult {
+  readonly stdout: string;
+  readonly stderr: string;
+  readonly exitCode: number;
+  /** The signal that terminated the process, if any (e.g. on a timeout). */
+  readonly signal: string | null;
+}
+
+/**
+ * A shared XDG_CONFIG_HOME seeded with `pragma/config.json` already present, so
+ * a default `runCli` invocation is PAST first-run — otherwise every spawn's
+ * stderr would carry the onboarding banner, making assertions order-dependent
+ * on whether a given test happens to run first. Memoized per process (module
+ * singleton): every test file gets its own worker process, so this is safely
+ * shared within a file without cross-file leakage.
+ */
+let seededConfigHome: string | undefined;
+function seededXdgConfigHome(): string {
+  if (seededConfigHome !== undefined) return seededConfigHome;
+  const dir = mkdtempSync(join(tmpdir(), "pragma-runcli-seeded-xdg-"));
+  mkdirSync(join(dir, "pragma"), { recursive: true });
+  writeFileSync(join(dir, "pragma", "config.json"), "{}\n");
+  seededConfigHome = dir;
+  return dir;
+}
+
+/**
+ * Spawn the real `pragma` CLI and capture its output.
+ *
+ * @param args - Argv passed to the binary (no `pragma`/`bun` prefix).
+ * @param options - cwd, env overrides, mode, and timeout.
+ * @returns The captured stdout/stderr/exitCode/signal.
+ * @note Impure — spawns a child process.
+ */
+export function runCli(
+  args: readonly string[],
+  options: RunCliOptions = {},
+): RunCliResult {
+  const mode = options.mode ?? "compiled";
+  const command = mode === "compiled" ? COMPILED_BINARY : "bun";
+  const spawnArgs = mode === "compiled" ? [...args] : [SOURCE_ENTRY, ...args];
+
+  const result = spawnSync(command, spawnArgs, {
+    cwd: options.cwd,
+    env: {
+      ...process.env,
+      XDG_CONFIG_HOME: seededXdgConfigHome(),
+      ...options.env,
+    },
+    encoding: "utf-8",
+    timeout: options.timeoutMs ?? 20_000,
+  });
+
+  if (result.error) {
+    throw new Error(
+      `runCli: failed to spawn ${command} ${spawnArgs.join(" ")} — ${result.error.message}`,
+    );
+  }
+
+  return {
+    stdout: result.stdout ?? "",
+    stderr: result.stderr ?? "",
+    exitCode: result.status ?? (result.signal ? 1 : 0),
+    signal: result.signal ?? null,
+  };
+}
+
+/**
+ * Build a fresh, unseeded XDG env overlay — for tests that WANT first-run
+ * (a config home with no `pragma/config.json` yet).
+ *
+ * @returns An env overlay a caller spreads into `RunCliOptions.env`.
+ * @note Impure — creates a temp directory.
+ */
+export function freshXdgEnv(): Record<string, string> {
+  const configHome = mkdtempSync(join(tmpdir(), "pragma-runcli-fresh-cfg-"));
+  const stateHome = mkdtempSync(join(tmpdir(), "pragma-runcli-fresh-state-"));
+  return { XDG_CONFIG_HOME: configHome, XDG_STATE_HOME: stateHome };
+}

package/src/testing/perf/budgets.test.ts

@@ -0,0 +1,164 @@
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+import { capabilities } from "../../capabilities/index.js";
+import { evaluateProjectConfig } from "../../kernel/config/evaluateProjectConfig.js";
+import { projectMcp } from "../helpers/projectMcp.js";
+import {
+  BUDGET_COMPLETE_MS,
+  BUDGET_HELP_MS,
+  BUDGET_MCP_P95_WARM_MS,
+  BUDGET_PROJECT_CONFIG_MS,
+  BUDGET_WARM_STORE_MS,
+} from "./budgets.js";
+import { measureCommand, percentile, trimmedMean } from "./measure.js";
+
+/** The compiled binary the perf globalSetup guarantees exists. */
+const BINARY = fileURLToPath(new URL("../../../dist/pragma", import.meta.url));
+
+/** Fresh XDG dirs so spawned runs never touch real state (and first-run is isolated). */
+const perfEnv = {
+  XDG_CONFIG_HOME: mkdtempSync(join(tmpdir(), "pragma-perf-cfg-")),
+  XDG_STATE_HOME: mkdtempSync(join(tmpdir(), "pragma-perf-state-")),
+};
+
+/**
+ * Protected budget suite. Spawns the standalone `dist/pragma`, discards
+ * warmups, and asserts median + p95 against the ceilings in budgets.ts. Given
+ * spawn-time variance, each spawn case retries.
+ */
+describe("perf budgets (PROTECTED)", () => {
+  it("percentile uses nearest-rank", () => {
+    expect(percentile([1, 2, 3, 4, 5], 0.5)).toBe(3);
+    expect(percentile([1, 2, 3, 4, 5], 0.95)).toBe(5);
+    expect(percentile([], 0.5)).toBeNaN();
+  });
+
+  it("trimmedMean discards the extremes", () => {
+    // 10% of 10 → cut 1 each end: mean of [2..9] = 5.5, unmoved by the 100 spike.
+    expect(trimmedMean([1, 2, 3, 4, 5, 6, 7, 8, 9, 100], 0.1)).toBe(5.5);
+    // Too few to trim → falls back to the full-sample mean.
+    expect(trimmedMean([2, 4], 0.1)).toBe(3);
+    expect(trimmedMean([], 0.1)).toBeNaN();
+  });
+
+  it("pragma --help stays under budget", { retry: 2 }, () => {
+    const result = measureCommand(BINARY, ["--help"], {
+      runs: 15,
+      warmups: 3,
+      env: perfEnv,
+    });
+    expect(result.medianMs).toBeLessThanOrEqual(BUDGET_HELP_MS);
+    expect(result.p95Ms).toBeLessThanOrEqual(BUDGET_HELP_MS);
+  });
+
+  it("pragma __complete stays under budget", { retry: 3 }, () => {
+    const result = measureCommand(BINARY, ["__complete", "config"], {
+      runs: 30,
+      warmups: 5,
+      env: perfEnv,
+    });
+    // Enforce BUDGET_COMPLETE_MS on the ROBUST statistic. The 10%-trimmed mean
+    // estimates typical completion latency without being dominated by the single
+    // worst spawn — which a nearest-rank p95 over a small sample effectively is,
+    // so it flaked red under whole-suite CPU contention on slower-than-reference
+    // hardware even though the median sat comfortably under budget (see
+    // BUDGETS.md). p95 is kept as a SOFT signal against gross regressions, with
+    // headroom rather than at the bare ceiling. The 100 ms ceiling is unchanged.
+    expect(result.trimmedMeanMs).toBeLessThanOrEqual(BUDGET_COMPLETE_MS);
+    expect(result.p95Ms).toBeLessThanOrEqual(BUDGET_COMPLETE_MS * 1.5);
+  });
+
+  it("pragma __complete skill lookup stays under budget", { retry: 3 }, () => {
+    // A NAME-source completion (the storeless `skills` filesystem walk) — the
+    // guard the "watch skill discovery" perf caveat calls for. Fresh XDG data
+    // dir → an empty walk, but it exercises the discoverSkills path per spawn;
+    // enforce the same robust statistic as the noun case.
+    const result = measureCommand(
+      BINARY,
+      ["__complete", "skill", "lookup", "do"],
+      {
+        runs: 30,
+        warmups: 5,
+        env: {
+          ...perfEnv,
+          XDG_DATA_HOME: mkdtempSync(join(tmpdir(), "pragma-perf-data-")),
+        },
+      },
+    );
+    expect(result.trimmedMeanMs).toBeLessThanOrEqual(BUDGET_COMPLETE_MS);
+    expect(result.p95Ms).toBeLessThanOrEqual(BUDGET_COMPLETE_MS * 1.5);
+  });
+
+  it("warm store-backed verb stays under budget", { retry: 2 }, () => {
+    // __store-probe boots the embedded pack from its n-quads cache and queries
+    // it — the full store-backed verb cost in the compiled binary. The first
+    // spawn materializes the pack; warmups absorb it, then it is a cache hit.
+    const result = measureCommand(BINARY, ["__store-probe"], {
+      runs: 12,
+      warmups: 3,
+      env: {
+        ...perfEnv,
+        XDG_CACHE_HOME: mkdtempSync(join(tmpdir(), "pragma-perf-cache-")),
+      },
+    });
+    expect(result.medianMs).toBeLessThanOrEqual(BUDGET_WARM_STORE_MS);
+    expect(result.p95Ms).toBeLessThanOrEqual(BUDGET_WARM_STORE_MS);
+  });
+
+  it("warm in-process MCP tool call stays under budget (mcpP95Warm)", {
+    retry: 2,
+  }, async () => {
+    // In-process (NOT a binary spawn): the warm cost of the MCP call path over
+    // the full 38-tool catalog. `capabilities` is storeless + network-free, so
+    // this isolates envelope + dispatch overhead without store/network noise.
+    const mcp = await projectMcp(capabilities);
+    try {
+      await mcp.callTool("capabilities"); // warm-up, excluded from the sample
+      const samples: number[] = [];
+      for (let i = 0; i < 25; i++) {
+        const start = performance.now();
+        await mcp.callTool("capabilities");
+        samples.push(performance.now() - start);
+      }
+      samples.sort((a, b) => a - b);
+      // Enforce on the robust trimmed mean; keep p95 as a signal with headroom
+      // (the __complete precedent) so a lone scheduler spike never flakes red.
+      expect(trimmedMean(samples, 0.1)).toBeLessThanOrEqual(
+        BUDGET_MCP_P95_WARM_MS,
+      );
+      expect(percentile(samples, 0.95)).toBeLessThanOrEqual(
+        BUDGET_MCP_P95_WARM_MS,
+      );
+    } finally {
+      await mcp.cleanup();
+    }
+  });
+
+  it("warm project-config load stays under budget", { retry: 2 }, async () => {
+    const savedState = process.env.XDG_STATE_HOME;
+    process.env.XDG_STATE_HOME = mkdtempSync(join(tmpdir(), "pragma-warm-"));
+    try {
+      const dir = mkdtempSync(join(tmpdir(), "pragma-warm-proj-"));
+      const path = join(dir, "pragma.config.ts");
+      writeFileSync(path, 'export default { tier: "core" };');
+
+      await evaluateProjectConfig(path); // prime the content-hash cache
+
+      const samples: number[] = [];
+      for (let i = 0; i < 20; i++) {
+        const start = performance.now();
+        await evaluateProjectConfig(path);
+        samples.push(performance.now() - start);
+      }
+      samples.sort((a, b) => a - b);
+      expect(percentile(samples, 0.5)).toBeLessThanOrEqual(
+        BUDGET_PROJECT_CONFIG_MS,
+      );
+    } finally {
+      process.env.XDG_STATE_HOME = savedState;
+    }
+  });
+});

package/src/testing/perf/budgets.ts

@@ -0,0 +1,37 @@
+/**
+ * Performance budgets for the compiled `pragma` binary.
+ *
+ * The ceilings the protected budget tests assert against. The surface
+ * covenant's designed target for `--help`/`__complete` is 50 ms; the day-1 perf
+ * spike (commit 6) found that unrealistic on the build hardware (cold Bun
+ * process start alone is ~45 ms here), so — per the plan — each ceiling is set
+ * to roughly 2× the measured median rather than silently weakening the target.
+ * Measured numbers and environment are recorded in BUDGETS.md.
+ */
+
+/** `pragma --help` ceiling (ms). Designed 50; measured median ~61 → 2×. */
+export const BUDGET_HELP_MS = 130;
+
+/** `pragma __complete …` ceiling (ms). Designed 50; measured median ~46 → 2×. */
+export const BUDGET_COMPLETE_MS = 100;
+
+/** Warm project-config (`pragma.config.ts`) load ceiling (ms). Cache hit is sub-ms. */
+export const BUDGET_PROJECT_CONFIG_MS = 10;
+
+/**
+ * Warm store-backed verb ceiling (ms) — a store boot from the cached n-quads
+ * dump plus a query, in the compiled binary. The designed 300 ms holds: the
+ * measured median (`__store-probe`) is ~150 ms here, so 300 is already ~2× the
+ * measured median and needs no relaxation (see BUDGETS.md).
+ */
+export const BUDGET_WARM_STORE_MS = 300;
+
+/**
+ * Warm in-process MCP tool-call ceiling (ms) — PR7 graduates this from seeded to
+ * ENFORCED. Measured over a warm, storeless tool (`capabilities`): pure envelope
+ * + dispatch, no store boot, no network, so it isolates the per-call overhead of
+ * the grown 38-tool catalog. Measured p95 is ~0.4 ms here (huge headroom), so
+ * 100 ms guards against a gross regression without flaking. `info` is
+ * deliberately NOT used — its network update-check makes it ~55 ms (see BUDGETS.md).
+ */
+export const BUDGET_MCP_P95_WARM_MS = 100;

package/src/testing/perf/globalSetup.ts

@@ -0,0 +1,30 @@
+/**
+ * Vitest global setup: build the compiled `dist/pragma` once, before the suite,
+ * if it is missing.
+ *
+ * Shared by both configs, because two suites spawn the binary: the perf budgets
+ * (src/testing/perf/**, `test:perf`) and the storeless-guarantee guards in
+ * src/kernel/completion/safety.test.ts (the main `test:vitest` pass). Wiring it
+ * into both means `bun run test` on a clean checkout provisions the binary with
+ * no manual build step — neither suite may assume it is pre-built.
+ */
+
+import { spawnSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+export default function setup(): void {
+  const root = fileURLToPath(new URL("../../../", import.meta.url));
+  const binary = fileURLToPath(
+    new URL("../../../dist/pragma", import.meta.url),
+  );
+  if (existsSync(binary)) return;
+
+  const result = spawnSync("bun", ["run", "scripts/build.ts"], {
+    cwd: root,
+    stdio: "inherit",
+  });
+  if (result.status !== 0) {
+    throw new Error("perf globalSetup: failed to build dist/pragma");
+  }
+}

package/src/testing/perf/measure.ts

@@ -0,0 +1,97 @@
+/**
+ * Perf measurement helper for the compiled `pragma` binary.
+ *
+ * Spawns the standalone binary N times, discards warmups, and reports the
+ * median and p95 of the wall-clock durations. Kept dependency-free (only
+ * `node:child_process`) so it can run inside vitest without touching the
+ * kernel's import graph. Wired up by the perf-spike commit; the skeleton
+ * lands early so the budget module and its consumers type-check from commit 1.
+ */
+
+import { spawnSync } from "node:child_process";
+
+/** Outcome of a spawn-based measurement run. */
+export interface MeasureResult {
+  /** Median duration across the kept samples, in milliseconds. */
+  readonly medianMs: number;
+  /** 95th-percentile duration across the kept samples, in milliseconds. */
+  readonly p95Ms: number;
+  /**
+   * 10%-trimmed mean across the kept samples, in milliseconds — the robust
+   * central estimate the budget suite enforces. Unlike a nearest-rank p95 over a
+   * small sample (effectively the max), it is not dominated by a single
+   * GC/scheduler spike, so it holds a ceiling reliably under CPU contention.
+   */
+  readonly trimmedMeanMs: number;
+  /** Every kept sample (warmups already discarded), in milliseconds. */
+  readonly samplesMs: readonly number[];
+}
+
+/** Options for {@link measureCommand}. */
+export interface MeasureOptions {
+  /** Total spawns to perform. */
+  readonly runs?: number;
+  /** Leading spawns to discard before sampling. */
+  readonly warmups?: number;
+  /** Extra environment overlaid on the child process env. */
+  readonly env?: Record<string, string>;
+}
+
+/** The percentile `p` (0–1) of a numeric sample, using nearest-rank. */
+export function percentile(sorted: readonly number[], p: number): number {
+  if (sorted.length === 0) return Number.NaN;
+  const rank = Math.ceil(p * sorted.length);
+  const index = Math.min(Math.max(rank, 1), sorted.length) - 1;
+  return sorted[index] as number;
+}
+
+/**
+ * The mean of a sorted sample after trimming the fastest and slowest `trim`
+ * fraction from each end. Far less sensitive than a small-sample p95 (which is
+ * effectively the max) to the occasional spawn spike, so it estimates typical
+ * latency and holds a ceiling reliably under whole-suite CPU contention.
+ */
+export function trimmedMean(sorted: readonly number[], trim = 0.1): number {
+  if (sorted.length === 0) return Number.NaN;
+  const cut = Math.floor(sorted.length * trim);
+  const body =
+    2 * cut < sorted.length ? sorted.slice(cut, sorted.length - cut) : sorted;
+  return body.reduce((sum, value) => sum + value, 0) / body.length;
+}
+
+/**
+ * Measure the wall-clock cost of running `binary args…` repeatedly.
+ *
+ * @param binary - Absolute path to the compiled binary.
+ * @param args - Arguments passed on each spawn.
+ * @param options - Run/warmup counts and env overlay.
+ * @returns Median, p95, and the kept samples.
+ * @note Impure — spawns child processes.
+ */
+export function measureCommand(
+  binary: string,
+  args: readonly string[],
+  options: MeasureOptions = {},
+): MeasureResult {
+  const runs = options.runs ?? 30;
+  const warmups = options.warmups ?? 3;
+  const samples: number[] = [];
+
+  for (let i = 0; i < runs; i++) {
+    const start = performance.now();
+    spawnSync(binary, [...args], {
+      env: { ...process.env, ...options.env },
+      stdio: "ignore",
+    });
+    const elapsed = performance.now() - start;
+    if (i >= warmups) samples.push(elapsed);
+  }
+
+  const sorted = [...samples].sort((a, b) => a - b);
+  return {
+    medianMs: percentile(sorted, 0.5),
+    p95Ms: percentile(sorted, 0.95),
+    trimmedMeanMs: trimmedMean(sorted, 0.1),
+    samplesMs: samples,
+  };
+}

package/src/testing/setupXdgIsolation.ts

@@ -1,11 +1,12 @@
 /**
- * Global test setup: isolate the XDG config layer.
+ * Global test setup: isolate the XDG config, data, and state layers.
  *
  * The layered config reader consults `$XDG_CONFIG_HOME/pragma/config.json`
- * on every read, and global-first writes create it. Point XDG at a
- * per-run temp directory so tests neither observe nor pollute the
- * developer's real global configuration. Individual tests that exercise
- * XDG behavior override this value themselves.
+ * on every read, global-first writes create it, and the project-config
+ * evaluator caches compiled configs under `$XDG_STATE_HOME`. Point all three
+ * at per-run temp directories so tests neither observe nor pollute the
+ * developer's real global state. Individual tests that exercise XDG
+ * behaviour override these values themselves.
  */
 
 import { mkdtempSync } from "node:fs";
@@ -13,3 +14,14 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 
 process.env.XDG_CONFIG_HOME = mkdtempSync(join(tmpdir(), "pragma-test-xdg-"));
+process.env.XDG_DATA_HOME = mkdtempSync(
+  join(tmpdir(), "pragma-test-xdg-data-"),
+);
+process.env.XDG_STATE_HOME = mkdtempSync(
+  join(tmpdir(), "pragma-test-xdg-state-"),
+);
+// The store layer writes the content-addressed pack cache under
+// $XDG_CACHE_HOME — isolate it too so tests never touch the real cache.
+process.env.XDG_CACHE_HOME = mkdtempSync(
+  join(tmpdir(), "pragma-test-xdg-cache-"),
+);

package/src/types.ts

@@ -0,0 +1,42 @@
+/**
+ * Cross-cutting type re-exports for the public barrel.
+ *
+ * The kernel keeps its authoritative type definitions co-located with the
+ * modules that own them (spec grammar, config, error, runtime); this file is
+ * the single import hop the types-only `index.ts` barrel re-exports from, so
+ * the public surface never reaches into deep module paths. It grows one
+ * section per PR as each kernel layer lands.
+ */
+
+export type { DetailLevel, OutputFormat } from "./constants.js";
+// — Config authoring ————————————————————————————————————————————————————————
+export type {
+  Channel,
+  PackageDeclaration,
+  PackageEntry,
+  PragmaConfig,
+} from "./kernel/config/types.js";
+// — Error model —————————————————————————————————————————————————————————————
+export type {
+  ErrorCode,
+  ErrorPayload,
+  PragmaErrorData,
+  Recovery,
+} from "./kernel/error/types.js";
+// — Render / envelope ———————————————————————————————————————————————————————
+export type {
+  ErrorEnvelope,
+  SuccessEnvelope,
+} from "./kernel/render/envelope.js";
+// — Spec grammar ———————————————————————————————————————————————————————————
+export type {
+  Capability,
+  CapabilityModule,
+  DisclosureSpec,
+  Example,
+  Formatters,
+  McpAnnotations,
+  ParamComplete,
+  ParamSpec,
+  VerbSpec,
+} from "./kernel/spec/types.js";