@canonical/pragma-cli 0.36.0 → 0.38.0

package/src/capabilities/listQuery.shipped.exec.test.ts

@@ -0,0 +1,400 @@
+/**
+ * Filters, search and pagination compiled into the query, EXECUTED against the
+ * shipped pack, for every list-shaped body the distribution declares
+ * (PROTECTED).
+ *
+ * The kernel change these assertions guard moved a story's declared filters from
+ * a predicate over returned rows into the generated SPARQL, and gave every
+ * list-shaped verb a page applied inside that same query. Both halves are
+ * semantics claims about a corpus, so a fixture cannot falsify them: a fixture
+ * obligingly contains whatever the assertion expects, while the shipped pack
+ * contains 252 blocks, 147 standards with a `GROUP_CONCAT`-ed category set, and
+ * an ordering with 25 tied names.
+ *
+ * Four claims, each on the whole corpus rather than a sample:
+ *
+ * - PAGES PARTITION. Walking a body in small pages returns exactly the rows the
+ *   unpaginated read returns, in exactly that order, with exactly those keys.
+ *   This is what makes the page a page OF the answer, and it is the assertion
+ *   that would catch a store whose sub-select ordering is not stable.
+ * - THE QUERY AGREES WITH THE ROW PREDICATE. Every declared filter's admissible
+ *   values, and a search term, answered by the compiled query, equal what the
+ *   retired row predicate would have answered over the whole population —
+ *   computed here, independently, as the oracle. Exhaustively for a
+ *   choice-sized vocabulary; at a deterministic spread for a catalogue-sized
+ *   one, because the sweep costs a query per value and two of these
+ *   vocabularies are catalogues of 745 and 1,156. See {@link sweep}.
+ * - A REFUSAL IS STILL A REFUSAL. A value the graph's vocabulary does not admit
+ *   is INVALID_INPUT naming the admissible values.
+ * - AN EMPTY ANSWER IS STILL CALM. A value the vocabulary admits that no row
+ *   carries is an empty page at exit 0, and so is a cursor past the last row.
+ */
+
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { MAX_LISTED_OPTIONS } from "../kernel/error/validOptions.js";
+import { compileStoryModule } from "../kernel/packs/compile.js";
+import { MAX_LIST_WINDOW } from "../kernel/packs/paging.js";
+import {
+  distributionSource,
+  type PackFilter,
+  type PackList,
+  type PackPage,
+  type PackRow,
+} from "../kernel/packs/types.js";
+import { verbKey } from "../kernel/packs/uniqueness.js";
+import { bootRuntime } from "../kernel/runtime/boot.js";
+import type { PragmaRuntime } from "../kernel/runtime/types.js";
+import { kebabCase } from "../kernel/spec/emitSurface.js";
+import type { VerbSpec } from "../kernel/spec/types.js";
+import { TEST_FLAGS } from "../testing/helpers/projectCli.js";
+import { declaredStories } from "./distribution.js";
+
+const SOURCE = distributionSource("pragma.conf.ts");
+
+/** One list-shaped body of a declared story, with the verb that runs it. */
+interface Body {
+  readonly label: string;
+  readonly shape: PackList;
+  readonly verb: VerbSpec;
+}
+
+/**
+ * Every list-shaped body the distribution declares — `list` and each extra
+ * verb — DERIVED from the config rather than listed, so a story added tomorrow
+ * is covered by the author who declares it and without their help.
+ */
+const BODIES: readonly Body[] = [...declaredStories].flatMap(
+  ([noun, story]) => {
+    const module = compileStoryModule(story, SOURCE, {});
+    const find = (verb: string): VerbSpec => {
+      const found = module.verbs.find(
+        (v) => verbKey(v.path) === `${noun} ${verb}`,
+      );
+      if (!found) throw new Error(`no compiled "${noun} ${verb}" verb`);
+      return found;
+    };
+    const bodies: Body[] = [];
+    if (story.list) {
+      bodies.push({
+        label: `${noun} list`,
+        shape: story.list,
+        verb: find("list"),
+      });
+    }
+    for (const verb of story.verbs ?? []) {
+      bodies.push({
+        label: `${noun} ${verb.verb}`,
+        shape: verb,
+        verb: find(verb.verb),
+      });
+    }
+    return bodies;
+  },
+);
+
+/** Bodies whose story declares at least one filter or a search. */
+const NARROWABLE = BODIES.filter(
+  (body) => body.shape.filters !== undefined || body.shape.search !== undefined,
+);
+
+let rt: PragmaRuntime;
+beforeAll(async () => {
+  rt = bootRuntime(TEST_FLAGS);
+  await rt.store.get();
+}, 60_000);
+
+afterAll(async () => {
+  (await rt.store.get()).store.dispose();
+});
+
+/** Run one body and return its page. */
+const page = (body: Body, params: Record<string, unknown> = {}) =>
+  body.verb.run(params, rt) as Promise<PackPage>;
+
+/** Every row of a body, in one call — the population the oracles read. */
+async function population(body: Body): Promise<readonly PackRow[]> {
+  // The kernel ceiling, which is far above every declared story's row count —
+  // so this is the whole answer rather than a page of it, asked for the way a
+  // caller who wants everything has to ask.
+  return (await page(body, { limit: MAX_LIST_WINDOW })).rows;
+}
+
+/** The values a value-free filter's declared vocabulary admits. */
+async function vocabulary(filter: PackFilter): Promise<readonly string[]> {
+  if (filter.values) return filter.values;
+  const declared = filter.vocabulary;
+  if (!declared) throw new Error(`filter --${filter.param} declares neither`);
+  const result = await rt.query.sparql(declared.query);
+  if (result.type !== "select") throw new Error("vocabulary must be a SELECT");
+  const variable = declared.variable ?? filter.variable;
+  return result.bindings
+    .map((row) => row[variable] ?? "")
+    .filter((value) => value !== "");
+}
+
+/**
+ * The values this suite actually sweeps for one filter.
+ *
+ * The promise is per-value, so a sweep costs one query per admissible value.
+ * Every filter that predates the token graph had a vocabulary of at most 36 —
+ * a set of choices — and exhausting it cost seconds. Two of the token graph's
+ * filters have vocabularies that are catalogues instead: 745 symbols and 1,156
+ * variables, measured at 1.25s and 0.36s per filtered query. Exhausting those
+ * two took 22 minutes between them, and the suite could not finish at all.
+ *
+ * So a choice-sized vocabulary is still swept exhaustively, and a
+ * catalogue-sized one is swept at a deterministic spread of that same size —
+ * first, last, and evenly spaced between — which stays reproducible across
+ * runs and still exercises every shape of value the filter admits. The
+ * boundary is {@link MAX_LISTED_OPTIONS}, the constant that also decides when
+ * an error stops naming values and starts counting them, because it is the
+ * same distinction: a choice is read, a catalogue is listed.
+ *
+ * @param admissible - Every value the filter's declared vocabulary admits.
+ * @returns The values to sweep, in the vocabulary's own order.
+ */
+function sweep(admissible: readonly string[]): readonly string[] {
+  if (admissible.length <= MAX_LISTED_OPTIONS) return admissible;
+  const step = (admissible.length - 1) / (MAX_LISTED_OPTIONS - 1);
+  const picked: string[] = [];
+  for (let index = 0; index < MAX_LISTED_OPTIONS; index += 1) {
+    const value = admissible[Math.round(index * step)];
+    if (value !== undefined) picked.push(value);
+  }
+  return [...new Set(picked)];
+}
+
+/**
+ * The RETIRED row predicate, reimplemented here as the oracle the compiled
+ * query is held to: whole-cell equality for `exact`, membership of a
+ * whitespace-separated set for `set`, case-insensitive either way, and a cell
+ * that is missing or empty never matches.
+ */
+function rowPredicate(
+  rows: readonly PackRow[],
+  filter: PackFilter,
+  value: string,
+): PackRow[] {
+  const wanted = value.normalize("NFC").toLowerCase();
+  return rows.filter((row) => {
+    const cell = row[filter.variable];
+    if (cell === undefined || cell === "") return false;
+    const normalized = cell.normalize("NFC").toLowerCase();
+    return filter.match === "set"
+      ? normalized.split(/\s+/).includes(wanted)
+      : normalized === wanted;
+  });
+}
+
+/** The retired search predicate, likewise. */
+function searchPredicate(
+  rows: readonly PackRow[],
+  variables: readonly string[],
+  term: string,
+): PackRow[] {
+  const wanted = term.normalize("NFC").toLowerCase();
+  return rows.filter((row) =>
+    variables.some((variable) =>
+      row[variable]?.normalize("NFC").toLowerCase().includes(wanted),
+    ),
+  );
+}
+
+describe("pages partition the answer, whole corpus (PROTECTED)", () => {
+  it("derives every list-shaped body from the config", () => {
+    // A guard, not a no-op: an empty derivation would make every sweep below
+    // vacuously green.
+    expect(BODIES.map((body) => body.label)).toContain("standard list");
+    expect(BODIES.length).toBeGreaterThanOrEqual(9);
+  });
+
+  it.each(BODIES.map((body) => [body.label, body] as const))(
+    "%s: a paged walk is the unpaginated answer, in order",
+    async (_label, body) => {
+      const whole = await population(body);
+      const walked: PackRow[] = [];
+      let after: string | undefined;
+      // Deliberately not a divisor of any story's row count, so the last page
+      // is a partial one and the walk has to notice it is the last.
+      const limit = 7;
+      for (let guard = 0; guard < 500; guard += 1) {
+        const current: PackPage = await page(body, {
+          limit,
+          ...(after ? { after } : {}),
+        });
+        expect(current.rows.length).toBeLessThanOrEqual(limit);
+        walked.push(...current.rows);
+        if (current.nextAfter === undefined) break;
+        after = current.nextAfter;
+      }
+      // Byte-identical, keys included: a row's shape is part of the answer, and
+      // the generated wrapper projects the author's own variables in the
+      // author's own order precisely so it stays so.
+      expect(JSON.stringify(walked)).toBe(JSON.stringify(whole));
+    },
+    60_000,
+  );
+
+  it("standard list: a cursor past the last row is a calm empty page", async () => {
+    const body = BODIES.find((b) => b.label === "standard list");
+    if (!body) throw new Error("no standard list");
+    const whole = await population(body);
+    const limit = whole.length - 1;
+    const first = await page(body, { limit });
+    expect(first.nextAfter).toBeTypeOf("string");
+    const last = await page(body, { limit, after: first.nextAfter });
+    // The tail, then nothing — and nothing is a success, not an error.
+    expect(last.rows).toHaveLength(1);
+    expect(last.nextAfter).toBeUndefined();
+  }, 60_000);
+});
+
+describe("the compiled query agrees with the retired row predicate (PROTECTED)", () => {
+  it("has a narrowable body to sweep", () => {
+    expect([...NARROWABLE.map((body) => body.label)].sort()).toEqual([
+      "concept list",
+      "implementation list",
+      "standard list",
+      "token consumers",
+      "token list",
+      "token values",
+      "variable chain",
+      "variable list",
+    ]);
+  });
+
+  it.each(NARROWABLE.map((body) => [body.label, body] as const))(
+    "%s: every admissible filter value answers what the row predicate would",
+    async (_label, body) => {
+      const whole = await population(body);
+      for (const filter of body.shape.filters ?? []) {
+        const admissible = await vocabulary(filter);
+        expect(admissible.length).toBeGreaterThan(0);
+        for (const value of sweep(admissible)) {
+          const answered = await page(body, {
+            [filter.param]: value,
+            limit: MAX_LIST_WINDOW,
+          });
+          expect(JSON.stringify(answered.rows)).toBe(
+            JSON.stringify(rowPredicate(whole, filter, value)),
+          );
+        }
+      }
+    },
+    120_000,
+  );
+
+  it.each(NARROWABLE.map((body) => [body.label, body] as const))(
+    "%s: a search term answers what the row predicate would",
+    async (_label, body) => {
+      const search = body.shape.search;
+      if (!search) return;
+      const whole = await population(body);
+      for (const term of ["a", "Button", "TEST"]) {
+        const answered = await page(body, {
+          search: term,
+          limit: MAX_LIST_WINDOW,
+        });
+        expect(JSON.stringify(answered.rows)).toBe(
+          JSON.stringify(searchPredicate(whole, search.variables, term)),
+        );
+      }
+    },
+    120_000,
+  );
+
+  it("standard list: a repeated filter is a union, without duplicating a row", async () => {
+    const body = BODIES.find((b) => b.label === "standard list");
+    if (!body) throw new Error("no standard list");
+    const whole = await population(body);
+    const filter = body.shape.filters?.[0];
+    if (!filter) throw new Error("no category filter");
+    const answered = await page(body, {
+      [filter.param]: ["testing", "react"],
+      limit: MAX_LIST_WINDOW,
+    });
+    const union = whole.filter(
+      (row) =>
+        rowPredicate([row], filter, "testing").length > 0 ||
+        rowPredicate([row], filter, "react").length > 0,
+    );
+    expect(JSON.stringify(answered.rows)).toBe(JSON.stringify(union));
+    // The case a `VALUES` block joined into the group would get wrong: a
+    // standard filed under a `testing` descendant AND under `react` would come
+    // back twice.
+    expect(new Set(answered.rows.map((row) => row.uri)).size).toBe(
+      answered.rows.length,
+    );
+  }, 60_000);
+
+  it("standard list: two filters are a conjunction, narrowing each other", async () => {
+    const body = BODIES.find((b) => b.label === "implementation list");
+    if (!body) throw new Error("no implementation list");
+    const whole = await population(body);
+    const [first, second] = body.shape.filters ?? [];
+    if (!first || !second) throw new Error("expected two filters");
+    const a = (await vocabulary(first))[0] as string;
+    const b = (await vocabulary(second))[0] as string;
+    const answered = await page(body, {
+      [first.param]: a,
+      [second.param]: b,
+      limit: MAX_LIST_WINDOW,
+    });
+    expect(JSON.stringify(answered.rows)).toBe(
+      JSON.stringify(rowPredicate(rowPredicate(whole, first, a), second, b)),
+    );
+  }, 60_000);
+});
+
+describe("a refusal and a calm empty answer stay distinguishable (PROTECTED)", () => {
+  it.each(NARROWABLE.map((body) => [body.label, body] as const))(
+    "%s: a value the vocabulary does not admit is INVALID_INPUT, naming the ones it does",
+    async (_label, body) => {
+      for (const filter of body.shape.filters ?? []) {
+        const admissible = await vocabulary(filter);
+        await expect(
+          page(body, { [filter.param]: "zzz-definitely-not-a-value" }),
+        ).rejects.toMatchObject({
+          code: "INVALID_INPUT",
+          validOptions: [...admissible].sort(),
+        });
+      }
+    },
+    60_000,
+  );
+
+  it("a value the vocabulary admits that no row carries is an empty page, not an error", async () => {
+    // The case that separates "the graph is the vocabulary" from "the returned
+    // rows are the vocabulary", found in the corpus rather than named: today it
+    // is a `ds:ConceptType` the ontology declares and no concept uses. Under
+    // the retired rows-as-vocabulary fallback such a value was INVALID_INPUT;
+    // under a PAGE it would have been INVALID_INPUT for merely sorting outside
+    // the window. It is neither — it is the empty answer the graph actually has.
+    const unpopulated: { body: Body; param: string; value: string }[] = [];
+    for (const body of NARROWABLE) {
+      const whole = await population(body);
+      for (const filter of body.shape.filters ?? []) {
+        for (const value of sweep(await vocabulary(filter))) {
+          if (rowPredicate(whole, filter, value).length === 0) {
+            unpopulated.push({ body, param: filter.param, value });
+          }
+        }
+      }
+    }
+    expect(unpopulated.length).toBeGreaterThan(0);
+    for (const { body, param, value } of unpopulated) {
+      const answered = await page(body, { [param]: value });
+      expect(answered.rows).toEqual([]);
+      expect(answered.nextAfter).toBeUndefined();
+      // And the calm sentence rides the same seam it always did — naming the
+      // filter that narrowed to nothing, since a filtered empty answer is a
+      // statement about the filter and not about the population.
+      expect(body.verb.output.formatters.notice?.(answered as never)).toContain(
+        // In the flag's own spelling: a story's `channelOf` param is typed
+        // `--channel-of`, and an answer that named the param would be naming
+        // something no caller can pass.
+        `matches \`--${kebabCase(param)} ${value}\`.`,
+      );
+    }
+  }, 120_000);
+});

package/src/capabilities/resources/resources.test.ts

@@ -61,8 +61,31 @@ const EVERYTHING: McpListable = { sources: [{ as: "entities" }] };
  * only. Payload SIZE is the property that actually regressed (712 entries /
  * ~155 KB on every connect), and it is the one an agent pays for twice: once in
  * transfer, once in the context window it can no longer spend on the task.
+ *
+ * **Raised from 60,000 to 100,000 on 2026-09-01**, when the
+ * `@canonical/token-ontology` pack took the payload to 65,119 bytes. The
+ * listing was kept whole rather than trimmed to fit: the per-class counts are
+ * how an agent gets the shape of the graph before spending a single lookup,
+ * and that is worth more than the bytes.
+ *
+ * **Cost scales with the number of T-Box terms and their metadata, not with
+ * the number of packs.** The `ontology` noun emits one resource per matching
+ * T-Box entity (`provider.ts`), so a pack costs what its own vocabulary
+ * costs — this one happened to add ~23 KB, which is a fact about this
+ * ontology and not a rate. The headroom here does not convert into "one more
+ * pack" of any size.
+ *
+ * It is also the only listing that grows this way: every other noun lists
+ * collections with their counts, so those grow with the number of nouns. When
+ * this ceiling is next reached, the fix is to scope the T-Box slice or
+ * paginate it — not to raise the number again, which would turn a gate into a
+ * formality.
+ *
+ * It is deliberately not removed. The regression it was written for is still
+ * worth catching, and a ceiling with headroom catches it while a deleted one
+ * catches nothing.
  */
-const LISTING_BUDGET_BYTES = 60_000;
+const LISTING_BUDGET_BYTES = 100_000;
 
 describe("resource listing (storeless, over the pack index)", () => {
   it("degrades to a recovery entry on a missing or legacy index", () => {
@@ -322,7 +345,20 @@ describe("resource surface over the server (embedded pack)", () => {
   let harness: Awaited<ReturnType<typeof projectMcp>>;
   beforeAll(async () => {
     harness = await projectMcp(capabilities);
-  });
+    // Boot the store HERE, not in whichever test happens to read first.
+    //
+    // `createLazyStore` memoises the boot PROMISE, and vitest's per-test
+    // timeout aborts the test without aborting the work — so a cold boot that
+    // overruns is charged partly to the test that triggered it and partly to
+    // the next one, which merely awaits the same promise. That reads as a
+    // sudden per-read regression and is nothing of the kind; it cost a real
+    // investigation to find that out.
+    //
+    // Paying it in the hook attributes it where it belongs, keeps every `it`
+    // in this file measuring what it claims to, and bounds it by hookTimeout
+    // rather than by a 5 s default that pack growth will keep outgrowing.
+    await harness.readResource(BUTTON_URI);
+  }, 60_000);
   afterAll(async () => {
     await harness.cleanup();
   });

package/src/capabilities/setup/buildPlan.ts

@@ -11,6 +11,7 @@
  */
 
 import type { PragmaRuntime } from "../../kernel/runtime/index.js";
+import type { FsProbe } from "./operations/writability.js";
 import {
   defaultSelected,
   type PlanRow,
@@ -88,6 +89,10 @@ const messageOf = (reason: unknown): string =>
  * @param rt - The per-invocation runtime.
  * @param ids - The targets to plan (all five for the run-all).
  * @param scope - The resolved scope selection.
+ * @param probe - The writability filesystem seam, handed to every row's
+ *   `detect`. Defaults to the real one; a test injects a fixture to drive the
+ *   two arms no CI host has — a `/nix/store` path, and (as root) a directory
+ *   that refuses `W_OK` — without opening a second way in to the row bodies.
  * @returns One entry per (target, scope) the scope actually runs.
  * @note Impure — every target's `detect` reads the real filesystem.
  */
@@ -95,6 +100,7 @@ export async function detectTargets(
   rt: PragmaRuntime,
   ids: readonly TargetId[],
   scope: ScopeSelection,
+  probe?: FsProbe,
 ): Promise<DetectedRow[]> {
   const wanted = TARGETS.filter((target) => ids.includes(target.id));
   const pairs = scopesForSelection(scope).flatMap((scope) =>
@@ -106,7 +112,7 @@ export async function detectTargets(
       })),
   );
   const settled = await Promise.allSettled(
-    pairs.map(({ target, scope }) => target.detect(rt, scope)),
+    pairs.map(({ target, scope }) => target.detect(rt, scope, probe)),
   );
   return pairs.map(({ target, scope }, index) => {
     const outcome = settled[index] as PromiseSettledResult<unknown>;

package/src/capabilities/setup/operations/index.ts

@@ -26,6 +26,11 @@
  * path — a consumer that routes it through this barrel statically would undo
  * that, and the setup guard test would say so.
  *
+ * Every line here has an importer that goes THROUGH it. The modules also
+ * import each other directly (`targets.ts` reaches `setupLsp.js`, and its
+ * siblings reach `writability.js`), and re-exporting those edges as well would
+ * make this file a second, wider surface that nothing asked for.
+ *
  * Deliberately internal: the config seed literal, the per-target message
  * builders, and the MCP entry writer. They are inputs to the composers above,
  * not operations a caller performs.
@@ -49,8 +54,12 @@ export type { LspDetection } from "./setupLsp.js";
 export {
   composeLsp,
   composeLspRemoval,
+  describeEditorSource,
   detectLsp,
+  firstLspBlock,
+  installableEditors,
   LSP_SKIP_REMEDY,
+  lspBlockReason,
   lspEditorNames,
   lspSkipReason,
   lspUninstallRemedy,
@@ -58,9 +67,13 @@ export {
 } from "./setupLsp.js";
 export type { McpDetection } from "./setupMcp.js";
 export {
+  blockedMcpGroups,
   composeMcp,
   composeMcpRemoval,
   detectMcp,
+  firstMcpBlock,
+  mcpBlockReason,
+  mcpGroupBlock,
   mcpGroupState,
   mcpWriteState,
   ownedMcpGroups,
@@ -77,3 +90,4 @@ export {
   staleSkillLinks,
   withinRoot,
 } from "./setupSkills.js";
+export type { FsProbe } from "./writability.js";

package/src/capabilities/setup/operations/setupCompletions.ts

@@ -204,11 +204,18 @@ export function composeCompletions(d: CompletionsDetection): Task<void> {
  * machine that never installed one reverses zero steps and says so.
  *
  * @param d - The detection gathered up front.
+ * @param undoKey - Correlation key stamped on the reversal(s), echoed on
+ *   their undo outcomes so the caller can read results back per row.
  * @returns A Task whose undo deletes the installed script.
  */
-export function composeCompletionsRemoval(d: CompletionsDetection): Task<void> {
+export function composeCompletionsRemoval(
+  d: CompletionsDetection,
+  undoKey?: string,
+): Task<void> {
   if (d.path === null || d.script === null || d.state === "absent") {
     return sequence_([]);
   }
-  return sequence_([writeFile(d.path, d.script, { undo: deleteFile(d.path) })]);
+  return sequence_([
+    writeFile(d.path, d.script, { undo: deleteFile(d.path), undoKey }),
+  ]);
 }

package/src/capabilities/setup/operations/setupConfig.ts

@@ -110,12 +110,17 @@ export function composeConfigFile(d: ConfigDetection): Task<void> {
  * delete as its `undo`, which is what the undo interpreter executes.
  *
  * @param d - The detection gathered up front.
+ * @param undoKey - Correlation key stamped on the reversal(s), echoed on
+ *   their undo outcomes so the caller can read results back per row.
  * @returns A Task whose undo removes a seed-only config, or an empty Task.
  */
-export function composeConfigRemoval(d: ConfigDetection): Task<void> {
+export function composeConfigRemoval(
+  d: ConfigDetection,
+  undoKey?: string,
+): Task<void> {
   if (!d.exists || !d.isSeed) return sequence_([]);
   return sequence_([
-    writeFile(d.path, SEED_CONFIG, { undo: deleteFile(d.path) }),
+    writeFile(d.path, SEED_CONFIG, { undo: deleteFile(d.path), undoKey }),
   ]);
 }