blume 1.1.3 → 1.2.0

package/src/cli/commands/eval.ts

@@ -0,0 +1,291 @@
+import { existsSync } from "node:fs";
+
+import { defineCommand } from "citty";
+import { join } from "pathe";
+
+import {
+  AGENTS,
+  launchAgent,
+  WINDOWS_COMMAND_NOT_FOUND,
+} from "../../audit/agent.ts";
+import type { AgentKind } from "../../audit/agent.ts";
+import { BlumeError } from "../../core/diagnostics.ts";
+import { scanProject } from "../../core/project-graph.ts";
+import { evalFixPrompt, initPrompt } from "../../eval/prompts.ts";
+import {
+  evalReportJson,
+  fixLines,
+  headerLine,
+  questionDetails,
+  questionLine,
+  startLine,
+  summaryLine,
+  warningLines,
+  writeEvalReport,
+} from "../../eval/report.ts";
+import { runEval } from "../../eval/run.ts";
+import type { EvalResult } from "../../eval/run.ts";
+import { EvalsFileError, loadEvalsFile } from "../../eval/schema.ts";
+import { reportInternalError } from "../internal-error.ts";
+import { flushStdout, logger } from "../log.ts";
+
+const DEFAULT_FILE = "evals.yaml";
+
+/** Reader wall-clock ceiling per question, in seconds. */
+const DEFAULT_TIMEOUT_S = 180;
+
+const isAgentKind = (value: string): value is AgentKind => value in AGENTS;
+
+/**
+ * Launch the interactive agent CLI, translating a missing executable into the
+ * Windows not-found sentinel. Only `ENOENT` means "not installed" — any other
+ * spawn failure (`EACCES`, `EMFILE`, …) must surface as itself.
+ */
+const launchAgentCode = async (
+  bin: string,
+  prompt: string
+): Promise<number> => {
+  try {
+    return await launchAgent(bin, prompt);
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") {
+      throw error;
+    }
+    return WINDOWS_COMMAND_NOT_FOUND;
+  }
+};
+
+const notInstalled = (agent: AgentKind): never => {
+  const cli = AGENTS[agent];
+  logger.error(
+    `${cli.name} (\`${cli.bin}\`) was not found on PATH. Install it with \`${cli.install}\`.`
+  );
+  return process.exit(1);
+};
+
+/** The fraction of run (non-skipped) questions that passed. */
+export const passFraction = (result: EvalResult): number => {
+  const ran = result.results.length - result.counts.skip;
+  return ran === 0 ? 1 : result.counts.pass / ran;
+};
+
+interface EvalFlags {
+  action?: string;
+  agent: string;
+  file: string;
+  fix?: boolean;
+  json?: boolean;
+  threshold?: string;
+  timeout?: string;
+}
+
+/** Validate the flag surface, exiting with a message on the first offense. */
+const parseFlags = (
+  args: EvalFlags
+): { agent: AgentKind; threshold: number; timeoutS: number } => {
+  if (!isAgentKind(args.agent)) {
+    logger.error(`Invalid --agent "${args.agent}" (use claude | codex).`);
+    process.exit(1);
+  }
+  if (args.action !== undefined && args.action !== "init") {
+    logger.error(`Unknown action "${args.action}" (did you mean "init"?).`);
+    process.exit(1);
+  }
+  if (args.json && args.fix) {
+    logger.error("--json and --fix are mutually exclusive.");
+    process.exit(1);
+  }
+  const threshold = args.threshold === undefined ? 1 : Number(args.threshold);
+  if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
+    logger.error(`Invalid --threshold "${args.threshold}" (use 0..1).`);
+    process.exit(1);
+  }
+  const timeoutS =
+    args.timeout === undefined ? DEFAULT_TIMEOUT_S : Number(args.timeout);
+  if (!Number.isInteger(timeoutS) || timeoutS <= 0) {
+    logger.error(`Invalid --timeout "${args.timeout}" (whole seconds).`);
+    process.exit(1);
+  }
+  return { agent: args.agent, threshold, timeoutS };
+};
+
+/** `blume eval --fix`: hand the failing report to the interactive agent. */
+const runFixHandoff = async (
+  agent: AgentKind,
+  result: EvalResult,
+  root: string,
+  threshold: number
+): Promise<void> => {
+  const count = result.counts.fail + result.counts.error;
+  if (count === 0) {
+    return;
+  }
+  const cli = AGENTS[agent];
+  const report = await writeEvalReport(result, root, threshold);
+  process.stderr.write(
+    `  Handing ${count} failed question${count === 1 ? "" : "s"} to ${cli.name}…\n\n`
+  );
+  const code = await launchAgentCode(cli.bin, evalFixPrompt(report));
+  if (code === WINDOWS_COMMAND_NOT_FOUND) {
+    notInstalled(agent);
+  }
+  if (code !== 0) {
+    process.exit(code);
+  }
+};
+
+/** `blume eval init`: draft a starter evals file via the interactive agent. */
+const runInit = async (agent: AgentKind, file: string): Promise<void> => {
+  const path = join(process.cwd(), file);
+  if (existsSync(path)) {
+    logger.error(
+      `${file} already exists — edit it directly, or pass --file to draft elsewhere.`
+    );
+    process.exit(1);
+  }
+  const code = await launchAgentCode(AGENTS[agent].bin, initPrompt(file));
+  if (code === WINDOWS_COMMAND_NOT_FOUND) {
+    notInstalled(agent);
+  }
+  if (code !== 0) {
+    process.exit(code);
+  }
+};
+
+export const evalCommand = defineCommand({
+  args: {
+    action: {
+      description: 'Optional action: "init" drafts a starter evals file.',
+      required: false,
+      type: "positional",
+    },
+    agent: {
+      default: "claude",
+      description: "Agent CLI that reads and grades the docs: claude | codex.",
+      type: "string",
+    },
+    file: {
+      default: DEFAULT_FILE,
+      description: "The evals file to run.",
+      type: "string",
+    },
+    fix: {
+      description:
+        "After a failing run, hand the report to the agent to fix the docs interactively.",
+      type: "boolean",
+    },
+    json: {
+      description: "Emit the report as JSON on stdout (for CI/editors).",
+      type: "boolean",
+    },
+    threshold: {
+      description:
+        "Minimum passing fraction (0..1) before the run exits non-zero. Defaults to 1.",
+      type: "string",
+    },
+    timeout: {
+      description: `Reader time limit per question, in seconds. Defaults to ${DEFAULT_TIMEOUT_S}.`,
+      type: "string",
+    },
+    verbose: {
+      description: "Include the reader's full answer under each failure.",
+      type: "boolean",
+    },
+  },
+  meta: {
+    description:
+      "Test the docs: an agent answers your questions using only the documentation.",
+    name: "eval",
+  },
+  async run({ args }) {
+    const root = process.cwd();
+    const { agent, threshold, timeoutS } = parseFlags(args);
+    if (args.action === "init") {
+      await runInit(agent, args.file);
+      return;
+    }
+
+    let result: EvalResult;
+    try {
+      // `scanProject`, not `prepareProject`: the eval reads the content tree
+      // and never regenerates the runtime, so it doesn't contend with a
+      // running dev server. Same reasoning as `blume audit`.
+      const project = await scanProject(root, { mode: "build" });
+      const evalsPath = join(root, args.file);
+      const { evals, raw } = await loadEvalsFile(evalsPath);
+
+      process.stderr.write(`${headerLine(evals.questions.length, agent)}\n\n`);
+      result = await runEval({
+        agent,
+        evals,
+        evalsPath,
+        onProgress: (event) => {
+          // Straight to stderr, not `logger.info` — consola drops info-level
+          // lines in test and CI environments.
+          if (event.kind === "question-start") {
+            process.stderr.write(
+              `${startLine(event.id, event.index, event.total)}\n`
+            );
+            return;
+          }
+          const lines = [
+            questionLine(event.result),
+            ...questionDetails(event.result, Boolean(args.verbose)),
+          ];
+          process.stderr.write(`${lines.join("\n")}\n`);
+        },
+        project,
+        rawEvals: raw,
+        readerTimeoutMs: timeoutS * 1000,
+      });
+    } catch (error) {
+      if (error instanceof EvalsFileError) {
+        logger.error(error.message);
+        process.exit(1);
+      }
+      if (error instanceof BlumeError) {
+        logger.error(error.diagnostic.message);
+        process.exit(1);
+      }
+      if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
+        notInstalled(agent);
+      }
+      reportInternalError(error);
+      process.exit(1);
+    }
+
+    const tail = [
+      "",
+      ...warningLines(result, root),
+      ...fixLines(result, root),
+      "",
+      `  ${summaryLine(result)}`,
+      "",
+    ];
+    process.stderr.write(tail.join("\n"));
+
+    const failed = passFraction(result) < threshold;
+
+    if (args.fix) {
+      // The gate is a CI concern; a handoff run succeeds when the agent
+      // session does, not when the docs already passed.
+      await runFixHandoff(agent, result, root, threshold);
+      return;
+    }
+
+    if (args.json) {
+      process.stdout.write(evalReportJson(result, root, threshold));
+      if (failed) {
+        // `process.exit` doesn't flush a piped stdout — without this the JSON
+        // is truncated mid-write in exactly the CI setups that consume it.
+        await flushStdout();
+        process.exit(1);
+      }
+      return;
+    }
+
+    if (failed) {
+      process.exit(1);
+    }
+  },
+});

package/src/cli/commands/init.ts

@@ -157,10 +157,15 @@ export const initCommand = defineCommand({
     const sink = interactive ? clack.log : logger;
     const { createdPackage } = await applyPlan(buildPlan(root, answers), sink);
 
-    // Keep Blume's generated runtime (`.blume/`) and build output (`dist/`) out
-    // of version control. Idempotent: creates `.gitignore` when absent and skips
-    // entries already present (trailing-slash agnostic).
-    const ignored = await ensureGitignore(root, [".blume/", "dist/"]);
+    // Keep installed dependencies, Blume's generated runtime (`.blume/`), and
+    // build output (`dist/`) out of version control. Idempotent: creates
+    // `.gitignore` when absent and skips entries already present
+    // (trailing-slash agnostic).
+    const ignored = await ensureGitignore(root, [
+      "node_modules/",
+      ".blume/",
+      "dist/",
+    ]);
     if (ignored.length > 0) {
       sink.success(`Added ${ignored.join(", ")} to .gitignore`);
     }

package/src/cli/commands/mcp-stdio.ts

@@ -0,0 +1,36 @@
+import { readFile } from "node:fs/promises";
+
+import { defineCommand } from "citty";
+
+import type { McpData } from "../../ai/mcp/data.ts";
+import { serveMcpStdio } from "../../ai/mcp/stdio.ts";
+
+export const mcpStdioCommand = defineCommand({
+  args: {
+    data: {
+      description: "Path to a serialized MCP data snapshot (JSON).",
+      required: true,
+      type: "string",
+    },
+  },
+  meta: {
+    description:
+      "Serve an MCP data snapshot over stdio (internal, used by `blume eval`).",
+    name: "mcp-stdio",
+  },
+  async run({ args }) {
+    // stdout belongs to the JSON-RPC transport from here on; every diagnostic
+    // must go to stderr or the MCP client chokes on the stray line.
+    let data: McpData;
+    try {
+      data = JSON.parse(await readFile(args.data, "utf-8")) as McpData;
+    } catch (error) {
+      const detail = error instanceof Error ? error.message : String(error);
+      process.stderr.write(
+        `blume mcp-stdio: cannot load the snapshot at ${args.data}: ${detail}\n`
+      );
+      process.exit(1);
+    }
+    await serveMcpStdio(data);
+  },
+});

package/src/cli/index.ts

@@ -8,7 +8,9 @@ import { checkCommand } from "./commands/check.ts";
 import { devCommand } from "./commands/dev.ts";
 import { doctorCommand } from "./commands/doctor.ts";
 import { ejectCommand } from "./commands/eject.ts";
+import { evalCommand } from "./commands/eval.ts";
 import { initCommand } from "./commands/init.ts";
+import { mcpStdioCommand } from "./commands/mcp-stdio.ts";
 import { previewCommand } from "./commands/preview.ts";
 import { syncCommand } from "./commands/sync.ts";
 import { validateCommand } from "./commands/validate.ts";
@@ -29,7 +31,9 @@ const main = defineCommand({
     dev: devCommand,
     doctor: doctorCommand,
     eject: ejectCommand,
+    eval: evalCommand,
     init: initCommand,
+    "mcp-stdio": mcpStdioCommand,
     preview: previewCommand,
     sync: syncCommand,
     validate: validateCommand,

package/src/cli/required-secrets.ts

@@ -24,7 +24,7 @@ export const checkRequiredSecrets = (config: ResolvedConfig): Diagnostic[] => {
     });
   };
 
-  if (config.ai.ask?.enabled) {
+  if (config.ai.ask?.enabled && !config.ai.ask.endpoint) {
     const backend = resolveAskBackend(config.ai.ask);
     if (backend.kind === "gateway") {
       requireSecret(

package/src/components/content/AccordionItem.astro

@@ -22,7 +22,7 @@ const accordionId = id ?? slugify(title);
 ---
 
 <details
-  class="not-prose group"
+  class="not-prose"
   data-blume-accordion
   id={accordionId}
   open={defaultOpen}
@@ -52,7 +52,7 @@ const accordionId = id ?? slugify(title);
       </span>
     </span>
     <Icon
-      class="text-muted-foreground transition-transform group-open:rotate-180"
+      class="text-muted-foreground transition-transform [details[open]>summary_&]:rotate-180"
       name="chevron-down"
       size={16}
     />

package/src/components/content/Frame.astro

@@ -13,7 +13,10 @@ const escapeRawHtml = (value: string): string =>
 
 const unwrapParagraph = (html: string): string => {
   const trimmed = html.trim();
-  const match = trimmed.match(/^<p>(?<content>[\s\S]*)<\/p>$/u);
+  // Only a *single* paragraph is unwrapped: the content must not contain its
+  // own `</p>`, or `<p>a</p>\n<p>b</p>` would "unwrap" to `a</p>\n<p>b` —
+  // unbalanced HTML injected via set:html.
+  const match = trimmed.match(/^<p>(?<content>(?:(?!<\/p>)[\s\S])*)<\/p>$/u);
   return match?.groups?.content ?? trimmed;
 };
 

package/src/components/content/Prompt.astro

@@ -13,7 +13,10 @@ const escapeRawHtml = (value: string): string =>
 
 const unwrapParagraph = (html: string): string => {
   const trimmed = html.trim();
-  const match = trimmed.match(/^<p>(?<content>[\s\S]*)<\/p>$/u);
+  // Only a *single* paragraph is unwrapped: the content must not contain its
+  // own `</p>`, or `<p>a</p>\n<p>b</p>` would "unwrap" to `a</p>\n<p>b` —
+  // unbalanced HTML injected via set:html.
+  const match = trimmed.match(/^<p>(?<content>(?:(?!<\/p>)[\s\S])*)<\/p>$/u);
   return match?.groups?.content ?? trimmed;
 };
 

package/src/components/content/Tooltip.astro

@@ -16,7 +16,10 @@ const external = href?.startsWith("http");
 
 const unwrapParagraph = (html: string): string => {
   const trimmed = html.trim();
-  const match = trimmed.match(/^<p>(?<content>[\s\S]*)<\/p>$/u);
+  // Only a *single* paragraph is unwrapped: the content must not contain its
+  // own `</p>`, or `<p>a</p>\n<p>b</p>` would "unwrap" to `a</p>\n<p>b` —
+  // unbalanced HTML injected via set:html.
+  const match = trimmed.match(/^<p>(?<content>(?:(?!<\/p>)[\s\S])*)<\/p>$/u);
   return match?.groups?.content ?? trimmed;
 };
 

package/src/components/content/TreeFolder.astro

@@ -7,7 +7,6 @@ const { defaultOpen = false, name, openable = true } = Astro.props;
 {
   openable ? (
     <details
-      class="group"
       data-blume-tree-folder
       data-blume-tree-openable="true"
       open={defaultOpen}
@@ -20,7 +19,7 @@ const { defaultOpen = false, name, openable = true } = Astro.props;
         role="treeitem"
       >
         <Icon
-          class="shrink-0 transition-transform group-open:rotate-90"
+          class="shrink-0 transition-transform [details[open]>summary_&]:rotate-90"
           name="chevron-right"
           size={15}
         />

package/src/components/content/Update.astro

@@ -81,3 +81,48 @@ const tagList = Array.isArray(tags) ? tags : tags ? [tags] : [];
     <slot />
   </div>
 </article>
+
+<script>
+  const state = window as Window & { __blumeUpdateIds?: boolean };
+  if (!state.__blumeUpdateIds) {
+    state.__blumeUpdateIds = true;
+
+    // Repeated labels slug to the same id (two "Bug fixes" entries on one
+    // changelog page); suffix the later ones so ids stay unique and each
+    // entry's self-anchor jumps to itself, not the first duplicate. The first
+    // keeps the plain slug, so hash deep-links stay stable. Mirrors the
+    // accordion id dedupe in `AccordionItem.astro`.
+    const dedupeUpdateIds = () => {
+      const seen = new Map<string, number>();
+      for (const update of document.querySelectorAll<HTMLElement>(
+        "[data-blume-update][id]"
+      )) {
+        const previous = update.id;
+        const count = seen.get(previous) ?? 0;
+        seen.set(previous, count + 1);
+        if (count === 0) {
+          continue;
+        }
+        update.id = `${previous}-${count + 1}`;
+        // Re-point the entry's own header anchor when it targets the old id;
+        // an explicit `href` prop (an external permalink) is left alone.
+        const anchor = update.querySelector<HTMLAnchorElement>("header a[href]");
+        const href = anchor?.getAttribute("href");
+        if (anchor && href?.endsWith(`#${previous}`)) {
+          anchor.setAttribute(
+            "href",
+            `${href.slice(0, -previous.length)}${update.id}`
+          );
+        }
+      }
+    };
+
+    if (document.readyState === "loading") {
+      document.addEventListener("DOMContentLoaded", dedupeUpdateIds, {
+        once: true,
+      });
+    } else {
+      dedupeUpdateIds();
+    }
+  }
+</script>

package/src/components/islands/AskAI.astro

@@ -9,11 +9,12 @@ interface Suggestion {
 }
 
 interface Props {
+  endpoint?: string;
   strings?: UIStrings["ask"];
   suggestions?: Suggestion[];
 }
 
-const { strings, suggestions = [] } = Astro.props;
+const { endpoint, strings, suggestions = [] } = Astro.props;
 
 // Icons resolve to inline SVG here (server-only module), so the client island
 // gets ready-to-render markup rather than a name it can't resolve. Bare Lucide
@@ -43,7 +44,13 @@ const icons = {
 };
 ---
 
-<AskAI client:load icons={icons} strings={strings} suggestions={items} />
+<AskAI
+  client:load
+  endpoint={endpoint}
+  icons={icons}
+  strings={strings}
+  suggestions={items}
+/>
 
 <style is:global>
   :root {

package/src/components/islands/ask-ai.tsx

@@ -65,7 +65,7 @@ const nextId = (): number => {
 
 // The endpoint and page path both honor the deployment `base` so grounding works
 // under a non-root base path (the server matches base-less document routes).
-const ASK_ENDPOINT = joinBase(import.meta.env.BASE_URL, "api/ask");
+const DEFAULT_ASK_ENDPOINT = joinBase(import.meta.env.BASE_URL, "api/ask");
 
 /** The current route with the deployment base stripped, for page-context lookup. */
 const currentPath = (): string =>
@@ -133,10 +133,12 @@ const ANSWER_CLASS =
   "prose prose-sm max-w-none text-foreground [&_a]:inline-flex [&_a]:items-center [&_a]:gap-1 [&_a]:rounded-full [&_a]:bg-muted [&_a]:px-2 [&_a]:py-1 [&_a]:align-middle [&_a]:font-medium [&_a]:text-[0.7rem] [&_a]:leading-none [&_a]:text-muted-foreground! [&_a]:no-underline! [&_a:hover]:text-foreground!";
 
 const AskAI = ({
+  endpoint = DEFAULT_ASK_ENDPOINT,
   icons = EMPTY_ICONS,
   strings,
   suggestions = EMPTY_SUGGESTIONS,
 }: {
+  endpoint?: string;
   icons?: AskIcons;
   strings?: UIStrings["ask"];
   suggestions?: Suggestion[];
@@ -180,13 +182,30 @@ const AskAI = ({
     return () => window.removeEventListener("blume:open-ask-ai", handler);
   }, []);
 
-  // ⌘I / Ctrl+I toggles the panel; Escape closes it.
+  // ⌘I / Ctrl+I toggles the panel; Escape closes it. Shift/Alt chords are
+  // left alone — Ctrl+Shift+I is the browser's DevTools shortcut, and
+  // capturing it would flap the panel open alongside them.
   useEffect(() => {
     const onKey = (event: KeyboardEvent) => {
-      if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === "i") {
+      if (
+        (event.metaKey || event.ctrlKey) &&
+        !event.shiftKey &&
+        !event.altKey &&
+        event.key.toLowerCase() === "i"
+      ) {
         event.preventDefault();
         setOpen((value) => !value);
       } else if (event.key === "Escape" && open) {
+        // An Escape aimed at a modal surface stacked on top (the search
+        // dialog traps focus inside itself) dismisses that surface only —
+        // this window listener still fires for it, and closing the panel
+        // underneath too would eat the user's conversation view. Duck-typed
+        // (`closest` presence) rather than `instanceof Element`, which needs
+        // a DOM global the test environment doesn't provide.
+        const target = event.target as Partial<Element> | null;
+        if (typeof target?.closest === "function" && target.closest("dialog")) {
+          return;
+        }
         setOpen(false);
       }
     };
@@ -251,7 +270,7 @@ const AskAI = ({
     abortRef.current = controller;
 
     try {
-      const response = await fetch(ASK_ENDPOINT, {
+      const response = await fetch(endpoint, {
         body: JSON.stringify({
           messages: history.map((m) => ({ content: m.content, role: m.role })),
           page: { path: currentPath() },