@canonical/pragma-cli 0.33.0 → 0.35.0

package/src/kernel/packs/collect.test.ts

@@ -2,15 +2,25 @@
  * Dynamic pack precedence + uniqueness (PROTECTED).
  *
  * Default config ⇒ effective == static (the golden holds, no config read cost on
- * the fast path). A config story overrides a bundled-pack noun or introduces a
- * new one; a story claiming a non-pack authored noun, a duplicate noun, or any
- * surviving `(noun, verb)` collision is rejected.
+ * the fast path, and the distribution's own declarations are compiled exactly
+ * once). A config story overrides a story-backed noun or introduces a new one;
+ * a story claiming an authored non-story noun, a duplicate noun within one
+ * config tier, or any surviving `(noun, verb)` collision is rejected. Across the
+ * two config tiers the closer declaration WINS rather than erroring.
  */
 
+import { mkdtempSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
 import { describe, expect, it } from "vitest";
+import { capabilities } from "../../capabilities/index.js";
 import type { ConfigLayers } from "../config/types.js";
 import type { CapabilityModule, VerbSpec } from "../spec/types.js";
-import { assembleEffectiveModules } from "./collect.js";
+import {
+  assembleEffectiveModules,
+  loadEffectiveModules,
+  validateStories,
+} from "./collect.js";
 import { assertUniqueVerbs } from "./uniqueness.js";
 
 /** A trivial storeless verb for a fake authored module. */
@@ -33,7 +43,9 @@ function fakeVerb(noun: string, verb?: string): VerbSpec {
 
 const STATIC: CapabilityModule[] = [
   { name: "config", verbs: [fakeVerb("config", "show")] },
-  { name: "standard", verbs: [fakeVerb("standard", "list")] },
+  // `story: true` marks a module compiled from a declared story — the ONLY
+  // kind a config/package story may replace.
+  { name: "standard", story: true, verbs: [fakeVerb("standard", "list")] },
 ];
 
 function layers(stories: unknown[]): ConfigLayers {
@@ -43,16 +55,36 @@ function layers(stories: unknown[]): ConfigLayers {
       tier: "default",
       channel: "default",
       detail: "default",
-      packages: "default",
+      packs: "default",
       stories: stories.length > 0 ? "project" : "default",
       prefixes: "default",
-      prompts: "default",
     },
     global: { path: "", exists: false },
     project: { exists: false },
   };
 }
 
+/**
+ * Config layers where the PROJECT declares `packs`, with stories on the pack
+ * entry and optionally at the top level (the stronger tier).
+ */
+function packLayers(
+  packStories: unknown[],
+  topLevel: unknown[] = [],
+): ConfigLayers {
+  const base = layers(topLevel);
+  return {
+    ...base,
+    config: {
+      ...base.config,
+      packs: [
+        { name: "recipes", source: "file:///recipes", stories: packStories },
+      ],
+    },
+    origins: { ...base.origins, packs: "project" },
+  };
+}
+
 const validPack = (noun: string) => ({
   noun,
   list: {
@@ -66,13 +98,13 @@ describe("assembleEffectiveModules (PROTECTED)", () => {
     expect(assembleEffectiveModules(STATIC, layers([]))).toBe(STATIC);
   });
 
-  it("a config story overrides a bundled-pack noun", () => {
+  it("a config story overrides a story-backed noun", () => {
     const effective = assembleEffectiveModules(
       STATIC,
       layers([validPack("standard")]),
     );
     const standard = effective.find((m) => m.name === "standard");
-    // The bundled `standard` module was replaced by the config pack (source
+    // The static `standard` module was replaced by the config pack (source
     // "config"); it still owns exactly `standard list`.
     expect(standard?.verbs.map((v) => v.path.join(" "))).toEqual([
       "standard list",
@@ -104,13 +136,13 @@ describe("assembleEffectiveModules (PROTECTED)", () => {
     expect(plain.find((m) => m.name === "stew")?.colophon).toBeUndefined();
   });
 
-  it("rejects a story claiming a non-pack authored noun", () => {
+  it("rejects a story claiming an authored non-story noun", () => {
     expect(() =>
       assembleEffectiveModules(STATIC, layers([validPack("config")])),
     ).toThrow(/built-in command/);
   });
 
-  it("rejects a duplicate story noun", () => {
+  it("rejects a duplicate story noun within one config tier", () => {
     expect(() =>
       assembleEffectiveModules(
         STATIC,
@@ -119,6 +151,34 @@ describe("assembleEffectiveModules (PROTECTED)", () => {
     ).toThrow(/Duplicate/);
   });
 
+  it("a story declared on a config pack contributes its noun", () => {
+    const effective = assembleEffectiveModules(
+      STATIC,
+      packLayers([validPack("recipe")]),
+    );
+    expect(effective.map((m) => m.name).sort()).toEqual([
+      "config",
+      "recipe",
+      "standard",
+    ]);
+  });
+
+  it("the top-level stories tier overrides packs[].stories for one noun", () => {
+    // Two config sources naming one noun is a REFINEMENT, not a conflict: the
+    // more specific (top-level) declaration wins and nothing throws.
+    const effective = assembleEffectiveModules(
+      STATIC,
+      packLayers(
+        [{ ...validPack("recipe"), colophon: "declared on the pack" }],
+        [{ ...validPack("recipe"), colophon: "declared by the project" }],
+      ),
+    );
+    expect(effective.filter((m) => m.name === "recipe")).toHaveLength(1);
+    expect(effective.find((m) => m.name === "recipe")?.colophon).toBe(
+      "declared by the project",
+    );
+  });
+
   it("rejects an invalid story pack (via the zod validator)", () => {
     expect(() =>
       assembleEffectiveModules(STATIC, layers([{ noun: "Bad Noun" }])),
@@ -126,6 +186,130 @@ describe("assembleEffectiveModules (PROTECTED)", () => {
   });
 });
 
+describe("validateStories — package stories NEVER throw (PROTECTED)", () => {
+  const record = (source: string, content: string) => ({ source, content });
+
+  it("drops an unparseable AND a schema-invalid story, keeping the valid one", () => {
+    // Both bad shapes in ONE case, deliberately: guarding the malformed file
+    // and letting the schema-invalid one through is exactly how a third-party
+    // story once bricked every command, `sources update` and `doctor` included.
+    const result = validateStories(
+      [
+        record("pkg/stories/broken.json", "{ not json"),
+        record("pkg/stories/invalid.json", '{"noun":"Bad Noun","list":{}}'),
+        record("pkg/stories/recipe.json", JSON.stringify(validPack("recipe"))),
+      ],
+      STATIC,
+    );
+    expect(result.entries.map((entry) => entry.definition.noun)).toEqual([
+      "recipe",
+    ]);
+    expect(result.problems.map((problem) => problem.source)).toEqual([
+      "pkg/stories/broken.json",
+      "pkg/stories/invalid.json",
+    ]);
+  });
+
+  it("last declaration wins for a noun, and the shadowed file is reported", () => {
+    const result = validateStories(
+      [
+        record("a/stories/recipe.json", JSON.stringify(validPack("recipe"))),
+        record(
+          "b/stories/recipe.json",
+          JSON.stringify({ ...validPack("recipe"), colophon: "from b" }),
+        ),
+      ],
+      STATIC,
+    );
+    expect(result.entries).toHaveLength(1);
+    expect(result.entries.at(0)?.source).toBe("b/stories/recipe.json");
+    expect(result.problems.at(0)?.source).toBe("a/stories/recipe.json");
+  });
+
+  it("refuses ANY noun the CLI already ships, without throwing", () => {
+    // Authored (`config`) and story-backed (`standard`) alike. A package may
+    // only ADD a noun: `assembleEffectiveModules` replaces a noun WHOLESALE, so
+    // a package claiming `standard` would swap this distribution's code-standard
+    // reads for its own in a project that did nothing but declare a dependency.
+    // Overriding a shipped noun stays a config decision.
+    const result = validateStories(
+      [
+        record("pkg/stories/config.json", JSON.stringify(validPack("config"))),
+        record(
+          "pkg/stories/standard.json",
+          JSON.stringify(validPack("standard")),
+        ),
+      ],
+      STATIC,
+    );
+    expect(result.entries).toEqual([]);
+    expect(result.problems.map((problem) => problem.message)).toEqual([
+      'its noun "config" is a command this CLI already ships and cannot be replaced by a package.',
+      'its noun "standard" is a command this CLI already ships and cannot be replaced by a package.',
+    ]);
+  });
+
+  it("leaves a declared noun's shipped verbs intact against a package claim", () => {
+    // The real registry, not a fixture. `token` used to be a COMPOSITE and this
+    // case guarded its hand-written `add-config` mutation; L-OPEN-9 removed that
+    // verb, so the same package-claim mechanics are now asserted against the
+    // story's own verbs — which is the case that still matters, since after the
+    // removal EVERY data noun is story-backed and a wholesale replacement is the
+    // only way one could be lost.
+    const before = capabilities.find((module) => module.name === "token");
+    expect(before?.verbs.map((verb) => verb.path.join(" "))).toEqual([
+      "token list",
+      "token lookup",
+      "token sample",
+    ]);
+    const { entries, problems } = validateStories(
+      [record("pkg/stories/token.json", JSON.stringify(validPack("token")))],
+      capabilities,
+    );
+    expect(entries).toEqual([]);
+    expect(problems.at(0)?.source).toBe("pkg/stories/token.json");
+    const after = assembleEffectiveModules(capabilities, layers([]), entries);
+    expect(
+      after
+        .find((module) => module.name === "token")
+        ?.verbs.map((verb) => verb.path.join(" ")),
+    ).toEqual(before?.verbs.map((verb) => verb.path.join(" ")));
+  });
+
+  it("a config story still REPLACES a package one for the same noun", () => {
+    const { entries } = validateStories(
+      [record("pkg/stories/recipe.json", JSON.stringify(validPack("recipe")))],
+      STATIC,
+    );
+    const effective = assembleEffectiveModules(
+      STATIC,
+      layers([{ ...validPack("recipe"), colophon: "from the project" }]),
+      entries,
+    );
+    expect(effective.filter((m) => m.name === "recipe")).toHaveLength(1);
+    expect(effective.find((m) => m.name === "recipe")?.colophon).toBe(
+      "from the project",
+    );
+  });
+});
+
+describe("the default layer's stories are compiled exactly once (PROTECTED)", () => {
+  it("a fresh cwd's effective modules ARE the static capabilities", async () => {
+    // `pragma.conf.ts` declares the distribution's own stories on its packs,
+    // and `capabilities/distribution.ts` has already compiled them into the
+    // static set — that is what keeps those nouns on the `--help`/`__complete`
+    // fast path. Re-merging them at dispatch would recompile and re-validate
+    // them on every command, and would put the distribution's own declarations
+    // behind a validator whose failure is fatal. Identity (`toBe`), not
+    // equality, is the assertion: nothing was rebuilt.
+    const cwd = mkdtempSync(join(tmpdir(), "pragma-carve-out-"));
+    const { modules, problems } = await loadEffectiveModules(capabilities, cwd);
+    expect(modules).toBe(capabilities);
+    // The embedded snapshot carries no package stories, so nothing to report.
+    expect(problems).toEqual([]);
+  });
+});
+
 describe("assertUniqueVerbs (PROTECTED)", () => {
   it("passes disjoint verbs and catches a collision", () => {
     expect(() =>

package/src/kernel/packs/collect.ts

@@ -1,108 +1,259 @@
 /**
  * Assemble the EFFECTIVE capability modules at dispatch.
  *
- * The `--help`/`__complete` fast paths use the STATIC capabilities (bundled +
- * authored) only. When a real command runs and the project config declares
- * `stories`, this merges them in: a config story overrides a bundled-pack noun
- * (replacing that module) or introduces a new noun; a config story that claims a
- * non-pack authored noun (config, ontology, …) is a hard error, and any
- * surviving `(noun, verb)` collision is caught by uniqueness.
+ * The `--help`/`__complete` fast paths use the STATIC capabilities (the
+ * authored modules plus the stories `pragma.conf.ts` declares, compiled at
+ * module load) only. When a real command runs, this merges in the stories the
+ * PROJECT declares: one may override a story-backed noun (replacing that
+ * module) or introduce a new one; one that claims an authored, non-story noun
+ * (config, ontology, …) is a hard error, and any surviving `(noun, verb)`
+ * collision is caught by uniqueness.
  *
- * Precedence is config > package > bundled. Package-shipped stories are a future
- * source (the new kernel does not yet discover package `stories/*.json`); the
- * seam is here so wiring them later is additive.
+ * Precedence is config > package > static, and within config the closer
+ * declaration wins: `packs[].stories` (the story a declared pack supplies) is
+ * overridden by the top-level `stories` (the project's own, most specific,
+ * statement). PACKAGE stories — the `stories/*.json` the answering pack carries
+ * — are third-party data, so they go through {@link validateStories}, which
+ * drops a bad one with a reported problem, CANNOT throw, and lets a package
+ * ADD a noun but never replace one the CLI ships. Config stories stay fatal and
+ * may override: the user owns those, and a broken `pragma.config.ts` already
+ * fails every command.
  *
  * zod is reached only through {@link parsePackDefinition} (lazy) — this module
  * is imported at dispatch, never on the fast path, so validating config stories
  * costs nothing on `--help`/`__complete`.
  */
 
-import type { ConfigLayers } from "../config/types.js";
-import { PragmaError } from "../error/PragmaError.js";
-import type { CapabilityModule } from "../spec/types.js";
-import { compilePack } from "./compile.js";
+import type { ConfigLayers } from "../config/index.js";
+import { PragmaError } from "../error/index.js";
+import type { PackStoryRecord } from "../runtime/graphpack/stories.js";
+import type { CapabilityModule } from "../spec/index.js";
+import { compileStoryModule } from "./compile.js";
 import { parsePackDefinition } from "./schema.js";
-import type { PackDefinition } from "./types.js";
+import type { PackDefinition, PackEntry } from "./types.js";
 import { assertUniqueVerbs } from "./uniqueness.js";
 
-/** The bundled-pack nouns a config/package story may override. */
-export const BUNDLED_PACK_NOUNS: readonly string[] = [
-  "standard",
-  "tier",
-  "modifier",
-  "token",
-  "block",
-];
+/** One package-declared story that could not be used, and why. */
+export interface StoryProblem {
+  /** The story file, e.g. `@acme/recipes/stories/recipe.json`. */
+  readonly source: string;
+  /** Why it was ignored, in one sentence. */
+  readonly message: string;
+}
+
+/** The outcome of validating the stories a pack carries. */
+interface ValidatedStories {
+  /** The stories that can be used, one per noun. */
+  readonly entries: readonly PackEntry[];
+  /** The ones that were ignored, each with its reason. */
+  readonly problems: readonly StoryProblem[];
+}
 
 /**
- * Merge dynamic (config-declared) story packs into the static capabilities.
+ * Turn pack-carried story records into usable entries, DROPPING (never throwing
+ * on) the ones that cannot be used.
+ *
+ * TOTAL by construction: `JSON.parse` and `parsePackDefinition` happen inside
+ * ONE try per record, so a malformed file and a schema-invalid one are handled
+ * identically. That is not a nicety — package stories reach dispatch before the
+ * command tree exists, so a throw here would fail EVERY command, including
+ * `sources update` and `doctor`, the only two that can recover from it.
  *
- * @param staticModules - The static capabilities (bundled + authored).
- * @param layers - The resolved config layers (its `config.stories`, `prefixes`).
+ * Two packages claiming one noun: the last declared wins and the shadowed one
+ * is reported, through this same channel.
+ *
+ * A package story may only introduce a NOUN THE CLI DOES NOT HAVE. Every static
+ * noun is reserved — the kernel's own AND the ones the distribution declares as
+ * stories — because the merge replaces a noun WHOLESALE: a package claiming
+ * `block` would swap this distribution's design-system reads for its own in a
+ * project that did nothing but declare a dependency. Overriding a shipped noun
+ * stays a CONFIG decision — that file is the user's own.
+ *
+ * @param records - The raw story records the answering pack carries.
+ * @param staticModules - The static capabilities, to detect a story claiming a
+ *   noun the CLI already ships.
+ * @returns The usable entries and the problems, both possibly empty.
+ */
+export function validateStories(
+  records: readonly PackStoryRecord[],
+  staticModules: readonly CapabilityModule[],
+): ValidatedStories {
+  const reserved = new Set(staticModules.map((module) => module.name));
+  const byNoun = new Map<string, PackEntry>();
+  const problems: StoryProblem[] = [];
+  for (const record of records) {
+    try {
+      const definition = parsePackDefinition(
+        JSON.parse(record.content),
+        record.source,
+      );
+      if (reserved.has(definition.noun)) {
+        problems.push({
+          source: record.source,
+          message: `its noun "${definition.noun}" is a command this CLI already ships and cannot be replaced by a package.`,
+        });
+        continue;
+      }
+      const shadowed = byNoun.get(definition.noun);
+      if (shadowed) {
+        problems.push({
+          source: shadowed.source,
+          message: `its "${definition.noun}" story is shadowed by ${record.source}.`,
+        });
+      }
+      byNoun.set(definition.noun, { source: record.source, definition });
+    } catch (error) {
+      problems.push({
+        source: record.source,
+        message: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
+  return { entries: [...byNoun.values()], problems };
+}
+
+/**
+ * The PROJECT's config-declared stories, weakest tier first.
+ *
+ * A tier whose origin is `"default"` is the DISTRIBUTION's own declaration,
+ * which `capabilities/distribution.ts` has already compiled into the static
+ * modules — that is what keeps those nouns on the `--help`/`__complete` fast
+ * path. Merging it again would recompile and re-validate it on every dispatch,
+ * and would put the distribution's own declarations behind a validator whose
+ * failure is fatal. `readConfig`'s per-field pick replaces wholesale, so an
+ * origin off `"default"` means the project (or global layer) owns that field.
+ *
+ * @param layers - The resolved config layers.
+ * @returns The `packs[].stories` tier, then the top-level `stories` tier.
+ */
+function projectStoryTiers(layers: ConfigLayers): readonly unknown[][] {
+  const packStories =
+    layers.origins.packs === "default"
+      ? []
+      : (layers.config.packs ?? []).flatMap((pack) =>
+          typeof pack === "string" ? [] : (pack.stories ?? []),
+        );
+  const topLevel =
+    layers.origins.stories === "default" ? [] : (layers.config.stories ?? []);
+  return [[...packStories], [...topLevel]];
+}
+
+/**
+ * Merge the package- and config-declared story packs into the static capabilities.
+ *
+ * @param staticModules - The static capabilities (authored + declared stories).
+ * @param layers - The resolved config layers (its `packs`, `stories`, `prefixes`).
+ * @param packageStories - The already-validated stories the answering pack
+ *   carries (see {@link validateStories}); weaker than either config tier.
  * @returns The effective modules, uniqueness-checked.
- * @throws PragmaError CONFIG_ERROR on an invalid story, a story claiming a
- *   non-pack authored noun, or a duplicate story noun.
+ * @throws PragmaError CONFIG_ERROR on an invalid CONFIG story, a config story
+ *   claiming an authored non-story noun, or a duplicate noun within one config
+ *   tier. Package stories were already screened and never throw here.
  */
 export function assembleEffectiveModules(
   staticModules: readonly CapabilityModule[],
   layers: ConfigLayers,
+  packageStories: readonly PackEntry[] = [],
 ): readonly CapabilityModule[] {
-  const stories = layers.config.stories ?? [];
-  if (stories.length === 0) return staticModules;
+  const tiers = projectStoryTiers(layers);
+  if (packageStories.length === 0 && tiers.every((tier) => tier.length === 0)) {
+    return staticModules;
+  }
 
   const prefixes = layers.config.prefixes ?? {};
-  const overridable = new Set(BUNDLED_PACK_NOUNS);
+  // Only a module compiled from a story may be replaced by one; the authored
+  // nouns (config, ontology, doctor, …) are the CLI itself.
+  const overridable = new Set(
+    staticModules.filter((module) => module.story).map((module) => module.name),
+  );
   const staticNouns = new Set(staticModules.map((module) => module.name));
 
-  const seen = new Set<string>();
-  const dynamic: CapabilityModule[] = [];
-  for (const raw of stories) {
-    const definition: PackDefinition = parsePackDefinition(raw, "config");
-    if (seen.has(definition.noun)) {
-      throw PragmaError.configError(
-        `Duplicate story noun "${definition.noun}" in config.`,
-      );
-    }
-    if (staticNouns.has(definition.noun) && !overridable.has(definition.noun)) {
-      throw PragmaError.configError(
-        `Story noun "${definition.noun}" collides with a built-in command.`,
+  // Keyed by noun so the stronger tier REPLACES the weaker one — declaring a
+  // story both on its pack and at the top level is a refinement, not an error.
+  const dynamic = new Map<string, CapabilityModule>();
+  for (const entry of packageStories) {
+    dynamic.set(
+      entry.definition.noun,
+      compileStoryModule(
+        entry.definition,
+        { label: entry.source, origin: "package" },
+        prefixes,
+      ),
+    );
+  }
+  for (const tier of tiers) {
+    const seen = new Set<string>();
+    for (const raw of tier) {
+      const definition: PackDefinition = parsePackDefinition(raw, "config");
+      if (seen.has(definition.noun)) {
+        throw PragmaError.configError(
+          `Duplicate story noun "${definition.noun}" in config.`,
+        );
+      }
+      if (
+        staticNouns.has(definition.noun) &&
+        !overridable.has(definition.noun)
+      ) {
+        throw PragmaError.configError(
+          `Story noun "${definition.noun}" collides with a built-in command.`,
+        );
+      }
+      seen.add(definition.noun);
+      dynamic.set(
+        definition.noun,
+        compileStoryModule(
+          definition,
+          { label: "config", origin: "config" },
+          prefixes,
+        ),
       );
     }
-    seen.add(definition.noun);
-    dynamic.push({
-      name: definition.noun,
-      verbs: compilePack(definition, "config", prefixes),
-      colophon: definition.colophon,
-    });
   }
 
-  // Drop the bundled module for any noun a config story overrides, then append
+  // Drop the static module for any noun a config story overrides, then append
   // the dynamic modules.
-  const kept = staticModules.filter((module) => !seen.has(module.name));
-  const effective = [...kept, ...dynamic];
+  const kept = staticModules.filter((module) => !dynamic.has(module.name));
+  const effective = [...kept, ...dynamic.values()];
   assertUniqueVerbs(effective.flatMap((module) => [...module.verbs]));
   return effective;
 }
 
 /**
  * Load the effective modules for a real invocation: read the layered config and
- * merge its story packs into the static capabilities.
+ * the answering pack's carried stories, and merge both into the static
+ * capabilities.
  *
  * Reached only at DISPATCH (real command / MCP serve), never on the
- * `--help`/`__complete` fast path, so the config read and zod validation never
- * cost the storeless paths. The config reader is dynamic-imported to keep even
- * this module's static graph free of it.
+ * `--help`/`__complete` fast path, so the config read, the pack read and zod
+ * validation never cost the storeless paths. The runtime pieces are
+ * dynamic-imported to keep even this module's static graph free of them.
  *
- * @param staticModules - The static capabilities (bundled + authored).
+ * @param staticModules - The static capabilities (authored + declared stories).
  * @param cwd - The directory to resolve project config against.
- * @returns The effective modules (static when no config stories are declared).
- * @note Impure — reads the project/global config.
+ * @returns The effective modules, plus the package stories that were ignored —
+ *   callers surface those on stderr rather than failing the command.
+ * @note Impure — reads the project/global config and the answering pack.
  */
 export async function loadEffectiveModules(
   staticModules: readonly CapabilityModule[],
   cwd: string,
-): Promise<readonly CapabilityModule[]> {
-  const { readConfig } = await import("../config/readConfig.js");
+): Promise<{
+  readonly modules: readonly CapabilityModule[];
+  readonly problems: readonly StoryProblem[];
+}> {
+  const [{ readConfig }, { resolveSources }, { activeStories }] =
+    await Promise.all([
+      import("../config/readConfig.js"),
+      import("../runtime/resolveSources.js"),
+      import("../runtime/graphpack/stories.js"),
+    ]);
   const layers = await readConfig(cwd);
-  return assembleEffectiveModules(staticModules, layers);
+  const { entries, problems } = validateStories(
+    activeStories(resolveSources(layers, cwd)),
+    staticModules,
+  );
+  return {
+    modules: assembleEffectiveModules(staticModules, layers, entries),
+    problems,
+  };
 }