blume 1.1.3 → 1.2.0

package/src/eval/findings.ts

@@ -0,0 +1,103 @@
+import type { BlumeProject } from "../core/project-graph.ts";
+import type { Diagnostic } from "../core/types.ts";
+import { locateQuestion } from "./schema.ts";
+import type { EvalQuestion } from "./schema.ts";
+
+const DOCS_URL = "https://useblume.dev/docs/reference/eval";
+
+/** Where a finding should anchor when no route hint matches a page. */
+export interface EvalsAnchor {
+  path: string;
+  raw: string;
+}
+
+/** The manifest route a hint names, if any. */
+const hintedRoute = (question: EvalQuestion, project: BlumeProject) => {
+  for (const hint of question.routes) {
+    const route = project.manifest.routes.find(
+      (candidate) => candidate.path === hint
+    );
+    if (route) {
+      return route;
+    }
+  }
+};
+
+/**
+ * Warnings for route hints that name no manifest route — docs move, and a
+ * finding that silently anchors to the evals file instead of the page it
+ * used to name is a debugging session; a warning is a one-line fix.
+ */
+export const routeFindings = (
+  question: EvalQuestion,
+  project: BlumeProject,
+  anchor: EvalsAnchor
+): Diagnostic[] => {
+  const known = new Set(project.manifest.routes.map((route) => route.path));
+  return question.routes
+    .filter((hint) => !known.has(hint))
+    .map((hint) => ({
+      code: "BLUME_EVAL_ROUTE_UNKNOWN",
+      docsUrl: DOCS_URL,
+      file: anchor.path,
+      line: locateQuestion(anchor.raw, question.id),
+      message: `Question "${question.id}" hints at route "${hint}", which matches no page.`,
+      severity: "warning" as const,
+      suggestion:
+        "Update the question's `routes` to the page's current route, or remove the hint.",
+    }));
+};
+
+/** What the finding needs to know about how the question went. */
+export interface QuestionOutcome {
+  /** Why the run errored, when it did. */
+  detail?: string;
+  /** The expected facts the judge found absent or contradicted. */
+  missing: string[];
+  status: "error" | "fail";
+}
+
+/**
+ * The diagnostic for a failed or errored question, anchored to the page that
+ * should have answered it (via the first matching route hint) or, failing
+ * that, to the question's line in the evals file.
+ */
+export const questionFinding = (
+  question: EvalQuestion,
+  outcome: QuestionOutcome,
+  project: BlumeProject,
+  anchor: EvalsAnchor
+): Diagnostic => {
+  const route = hintedRoute(question, project);
+  const site = route
+    ? { file: route.sourcePath, url: route.path }
+    : { file: anchor.path, line: locateQuestion(anchor.raw, question.id) };
+
+  if (outcome.status === "error") {
+    return {
+      code: "BLUME_EVAL_QUESTION_ERROR",
+      docsUrl: DOCS_URL,
+      message: `Eval run failed for "${question.question}"${
+        outcome.detail ? ` — ${outcome.detail}` : ""
+      }.`,
+      severity: question.severity,
+      suggestion:
+        "Rerun `blume eval`; if it persists, check the agent CLI installation and the failure detail.",
+      ...site,
+    };
+  }
+
+  const missing =
+    outcome.missing.length > 0
+      ? ` — missing: ${outcome.missing.join("; ")}`
+      : "";
+  return {
+    code: "BLUME_EVAL_QUESTION_FAILED",
+    docsUrl: DOCS_URL,
+    message: `Docs could not answer: "${question.question}"${missing}`,
+    severity: question.severity,
+    suggestion:
+      "State the missing facts on this page, then rerun `blume eval`.",
+    ...site,
+  };
+};

package/src/eval/prompts.ts

@@ -0,0 +1,78 @@
+import type { EvalQuestion } from "./schema.ts";
+
+/**
+ * The reader's instructions. The agent is spawned in an empty directory with
+ * its file/shell/web tools disabled, so the prompt's job is to direct it at
+ * the MCP docs tools and forbid the one escape hatch that remains: answering
+ * from prior knowledge of the product.
+ */
+export const readerPrompt = (question: EvalQuestion): string =>
+  `You are evaluating whether a product's documentation can answer a user's question.
+
+Answer the question below using ONLY the connected documentation tools (search_docs, get_page, list_pages, get_navigation). Rules:
+- Do not use prior knowledge about the product. Do not guess.
+- Do not read files, run commands, or access the network.
+- Search first, then read the most relevant pages with get_page.
+- If the documentation does not contain the answer, say exactly what information is missing instead of inventing one.
+
+Question: ${question.question}
+
+Reply with a concise answer containing the specific facts the documentation provides. Plain text only.`;
+
+/** The judge's instructions: grade the answer against the expected facts. */
+export const judgePrompt = (question: EvalQuestion, answer: string): string => {
+  const facts = question.expected.map((fact) => `- ${fact}`).join("\n");
+  return `You are grading an answer against expected facts. Do not use any tools.
+
+Question: ${question.question}
+
+Expected facts — each must be present in substance (paraphrase is fine, contradiction is not):
+${facts}
+
+Answer to grade:
+"""
+${answer}
+"""
+
+An answer that states the documentation lacks the information FAILS.
+
+Reply with ONLY this JSON object on a single line, no markdown fences:
+{"pass": true|false, "score": 0.0-1.0, "missing": ["expected facts absent or contradicted"], "notes": "one sentence"}`;
+};
+
+/** The `--fix` handoff prompt: where the report is and the ground rules. */
+export const evalFixPrompt = (reportPath: string): string =>
+  `Fix the documentation gaps found by \`blume eval\` in this project.
+
+The full report is at ${reportPath}. It is JSON: each entry in \`eval.results\` with status "fail" is one question the documentation could not answer. Each carries the \`question\`, the \`expected\` facts, the judge's \`missing\` facts, and the reader agent's \`answer\` (what the docs currently convey). The matching \`diagnostics\` entry names the source \`file\` of the page that should answer it.
+
+Work through every failed question:
+1. Read the page named in the finding (or choose the best page when none is named).
+2. Edit the documentation so it states the missing facts explicitly. Add prose, not filler; keep the page's voice.
+3. Never delete questions from the evals file or weaken expected facts.
+
+When you are done, run \`blume eval\` to verify, and repeat until every question passes.`;
+
+/** The `eval init` prompt: draft a starter evals file from the docs. */
+export const initPrompt = (evalsPath: string): string =>
+  `Draft a starter evals file for \`blume eval\` in this documentation project.
+
+Read the documentation source pages in this project and write ${evalsPath} with about 10 high-value questions a real user would ask — installation, configuration, deployment, and the project's headline features. For each question, list the expected facts a correct answer must state, grounded in what the documentation actually promises (never invent facts the docs don't state).
+
+The file format is YAML:
+
+questions:
+  - id: kebab-case-slug
+    question: One user question?
+    expected:
+      - a fact the answer must contain
+      - another required fact
+    routes:
+      - /route/of/the/page/that/answers/it
+
+Rules:
+- Every \`expected\` fact must be verifiable in the docs today.
+- Prefer questions whose answers live on one page; set \`routes\` to that page.
+- Keep ids unique and questions short.
+
+When you are done, print the file and suggest running \`blume eval\` to try it.`;

package/src/eval/report.ts

@@ -0,0 +1,214 @@
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+
+import { join, relative } from "pathe";
+
+import { AGENTS } from "../audit/agent.ts";
+import { countBySeverity } from "../core/diagnostics.ts";
+import type { EvalResult, QuestionResult, QuestionStatus } from "./run.ts";
+
+const ESC = String.fromCodePoint(27);
+const COLORS = {
+  bold: `${ESC}[1m`,
+  cyan: `${ESC}[36m`,
+  dim: `${ESC}[2m`,
+  green: `${ESC}[32m`,
+  red: `${ESC}[31m`,
+  reset: `${ESC}[0m`,
+  yellow: `${ESC}[33m`,
+};
+
+const GLYPH: Record<QuestionStatus, string> = {
+  error: "!",
+  fail: "✖",
+  pass: "✔",
+  skip: "⊘",
+};
+
+const STATUS_COLOR: Record<QuestionStatus, string> = {
+  error: COLORS.yellow,
+  fail: COLORS.red,
+  pass: COLORS.green,
+  skip: COLORS.dim,
+};
+
+/** Longest id gets the room; everything shorter aligns to it. */
+const ID_PAD = 28;
+
+const seconds = (ms: number): string => `${(ms / 1000).toFixed(1)}s`;
+
+const money = (cost: number | undefined): string =>
+  cost === undefined ? "" : `$${cost.toFixed(2)}`;
+
+const duration = (ms: number): string => {
+  if (ms < 60_000) {
+    return seconds(ms);
+  }
+  const minutes = Math.floor(ms / 60_000);
+  const rest = Math.round((ms % 60_000) / 1000);
+  return `${minutes}m ${rest}s`;
+};
+
+/** One question's progress/report line: glyph, id, status, score, time, cost. */
+export const questionLine = (result: QuestionResult): string => {
+  const color = STATUS_COLOR[result.status];
+  const glyph = `${color}${GLYPH[result.status]}${COLORS.reset}`;
+  const id = result.id.padEnd(ID_PAD);
+  if (result.status === "skip") {
+    return `  ${glyph} ${id} ${COLORS.dim}skipped${COLORS.reset}`;
+  }
+  const score = result.score === undefined ? "" : result.score.toFixed(2);
+  const cells = [
+    `${color}${result.status}${COLORS.reset}`,
+    score,
+    `${COLORS.dim}${seconds(result.durationMs)}${COLORS.reset}`,
+    `${COLORS.dim}${money(result.costUsd)}${COLORS.reset}`,
+  ]
+    .filter((cell) => cell !== "")
+    .join("  ");
+  return `  ${glyph} ${id} ${cells}`;
+};
+
+/** Indented context under a question's line: missing facts, error detail. */
+export const questionDetails = (
+  result: QuestionResult,
+  verbose: boolean
+): string[] => {
+  const lines: string[] = [];
+  if (result.status === "fail") {
+    for (const fact of result.missing) {
+      lines.push(`      ${COLORS.dim}missing: ${fact}${COLORS.reset}`);
+    }
+  }
+  if (result.status === "error" && result.detail) {
+    lines.push(`      ${COLORS.dim}${result.detail}${COLORS.reset}`);
+  }
+  if (verbose && result.answer && result.status !== "pass") {
+    lines.push(
+      ...result.answer
+        .split("\n")
+        .map((line) => `      ${COLORS.dim}> ${line}${COLORS.reset}`)
+    );
+  }
+  return lines;
+};
+
+/** The one-line totals: `9 passed · 2 failed · 1 skipped · 1m 42s · $0.71`. */
+export const summaryLine = (result: EvalResult): string => {
+  const { counts } = result;
+  const parts = [
+    `${counts.pass} passed`,
+    counts.fail > 0 ? `${counts.fail} failed` : "",
+    counts.error > 0 ? `${counts.error} errored` : "",
+    counts.skip > 0 ? `${counts.skip} skipped` : "",
+    duration(result.durationMs),
+    money(result.costUsd),
+  ].filter((part) => part !== "");
+  return parts.join(" · ");
+};
+
+/** The header line the command prints before the first question runs. */
+export const headerLine = (total: number, agent: EvalResult["agent"]): string =>
+  `${COLORS.bold}blume eval${COLORS.reset}  ${total} question(s) · ${AGENTS[agent].name}`;
+
+/** The dim announce line while a question's agents run. */
+export const startLine = (id: string, index: number, total: number): string =>
+  `  ${COLORS.dim}▸ ${id} (${index + 1}/${total})${COLORS.reset}`;
+
+/** `fix:` pointers for failed questions, naming the file that resolves each. */
+export const fixLines = (result: EvalResult, root: string): string[] =>
+  result.diagnostics
+    .filter((diagnostic) => diagnostic.code !== "BLUME_EVAL_ROUTE_UNKNOWN")
+    .map((finding) => {
+      const site = finding.file
+        ? `${relative(root, finding.file)}${finding.line ? `:${finding.line}` : ""}`
+        : "";
+      return `  ${COLORS.cyan}fix:${COLORS.reset} ${site} ${COLORS.dim}${finding.message}${COLORS.reset}`;
+    });
+
+/** Dim warnings for route hints that no longer match a page. */
+export const warningLines = (result: EvalResult, root: string): string[] =>
+  result.diagnostics
+    .filter((diagnostic) => diagnostic.code === "BLUME_EVAL_ROUTE_UNKNOWN")
+    .map((finding) => {
+      const site = finding.file
+        ? ` ${relative(root, finding.file)}${finding.line ? `:${finding.line}` : ""}`
+        : "";
+      return `  ${COLORS.yellow}⚠${COLORS.reset}${site} ${COLORS.dim}${finding.message}${COLORS.reset}`;
+    });
+
+/** The human report, written to stderr by the command. */
+export const formatEvalReport = (
+  result: EvalResult,
+  root: string,
+  options: { verbose?: boolean } = {}
+): string => {
+  const lines: string[] = [headerLine(result.results.length, result.agent), ""];
+
+  for (const question of result.results) {
+    lines.push(
+      questionLine(question),
+      ...questionDetails(question, Boolean(options.verbose))
+    );
+  }
+  lines.push("");
+
+  // The finding tells the author which file fixes which failure.
+  const failures = fixLines(result, root);
+  lines.push(...failures);
+  if (failures.length > 0) {
+    lines.push("");
+  }
+
+  lines.push(`  ${summaryLine(result)}`, "");
+  return lines.join("\n");
+};
+
+/**
+ * The machine-readable report. The `diagnostics` + `summary` shape matches
+ * `blume validate --json` and `blume audit --json` exactly — anything parsing
+ * those keeps working — with the eval run's own results alongside.
+ */
+export const evalReportJson = (
+  result: EvalResult,
+  root: string,
+  threshold: number
+): string => {
+  const diagnostics = result.diagnostics.map((diagnostic) =>
+    diagnostic.file
+      ? { ...diagnostic, file: relative(root, diagnostic.file) }
+      : diagnostic
+  );
+  return `${JSON.stringify(
+    {
+      diagnostics,
+      eval: {
+        agent: result.agent,
+        costUsd: result.costUsd,
+        counts: result.counts,
+        durationMs: result.durationMs,
+        results: result.results,
+        threshold,
+      },
+      summary: countBySeverity(result.diagnostics),
+    },
+    null,
+    2
+  )}\n`;
+};
+
+/**
+ * Write the full JSON report where a `--fix` agent can read it — a file
+ * rather than inline prompt text, because a long run's answers would exceed
+ * the platform's argv limit.
+ */
+export const writeEvalReport = async (
+  result: EvalResult,
+  root: string,
+  threshold: number
+): Promise<string> => {
+  const dir = await mkdtemp(join(tmpdir(), "blume-eval-"));
+  const path = join(dir, "report.json");
+  await writeFile(path, evalReportJson(result, root, threshold));
+  return path;
+};

package/src/eval/run.ts

@@ -0,0 +1,290 @@
+import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+
+import { join } from "pathe";
+
+import { buildMcpData } from "../ai/mcp/data.ts";
+import { AGENTS } from "../audit/agent.ts";
+import type { AgentKind } from "../audit/agent.ts";
+import type { BlumeProject } from "../core/project-graph.ts";
+import type { Diagnostic } from "../core/types.ts";
+import {
+  agentArgs,
+  parseVerdict,
+  readAgentOutput,
+  runAgentHeadless,
+  writeMcpConfig,
+} from "./agents.ts";
+import type { HeadlessRunner } from "./agents.ts";
+import { questionFinding, routeFindings } from "./findings.ts";
+import { judgePrompt, readerPrompt } from "./prompts.ts";
+import type { EvalQuestion, EvalsFile } from "./schema.ts";
+
+/** Reader runs search and read several pages; the judge grades one answer. */
+const DEFAULT_READER_TIMEOUT_MS = 180_000;
+const DEFAULT_JUDGE_TIMEOUT_MS = 60_000;
+
+export type QuestionStatus = "error" | "fail" | "pass" | "skip";
+
+/** One question's outcome, carrying everything the reports print. */
+export interface QuestionResult {
+  answer?: string;
+  costUsd?: number;
+  /** Why the run errored (timeout, agent failure, unparseable verdict). */
+  detail?: string;
+  durationMs: number;
+  expected: string[];
+  id: string;
+  missing: string[];
+  notes?: string;
+  question: string;
+  routes: string[];
+  score?: number;
+  status: QuestionStatus;
+}
+
+export interface EvalResult {
+  agent: AgentKind;
+  /** Total spend, when the agent CLI reports it (claude does, codex doesn't). */
+  costUsd?: number;
+  counts: Record<QuestionStatus, number>;
+  diagnostics: Diagnostic[];
+  durationMs: number;
+  results: QuestionResult[];
+}
+
+export type EvalProgress =
+  | {
+      kind: "question-end";
+      index: number;
+      result: QuestionResult;
+      total: number;
+    }
+  | { kind: "question-start"; id: string; index: number; total: number };
+
+export interface EvalRunOptions {
+  agent: AgentKind;
+  evals: EvalsFile;
+  /** Where the evals file lives, for findings with no usable route hint. */
+  evalsPath: string;
+  judgeTimeoutMs?: number;
+  onProgress?: (event: EvalProgress) => void;
+  project: BlumeProject;
+  /** The evals file's raw text, for line-anchoring findings. */
+  rawEvals: string;
+  readerTimeoutMs?: number;
+  /** The spawn function — injectable so tests never launch a real agent. */
+  run?: HeadlessRunner;
+}
+
+interface QuestionContext {
+  bin: string;
+  dir: string;
+  judgeTimeoutMs: number;
+  kind: AgentKind;
+  mcp: Awaited<ReturnType<typeof writeMcpConfig>>;
+  readerTimeoutMs: number;
+  run: HeadlessRunner;
+}
+
+const errored = (
+  question: EvalQuestion,
+  detail: string,
+  durationMs: number
+): QuestionResult => ({
+  detail,
+  durationMs,
+  expected: question.expected,
+  id: question.id,
+  missing: [],
+  question: question.question,
+  routes: question.routes,
+  status: "error",
+});
+
+/** Run one question: a fresh empty cwd, the reader, then the judge. */
+const runQuestion = async (
+  question: EvalQuestion,
+  index: number,
+  context: QuestionContext
+): Promise<QuestionResult> => {
+  const started = performance.now();
+  const elapsed = () => Math.round(performance.now() - started);
+
+  // An empty working directory per invocation is the fresh-eyes guardrail:
+  // even if a tool restriction slips, there is nothing here to read.
+  const workDir = join(context.dir, `work-${index}`);
+  await mkdir(workDir, { recursive: true });
+
+  const answerPath = join(workDir, "answer.txt");
+  const reader = await context.run(
+    context.bin,
+    agentArgs(context.kind, { lastMessagePath: answerPath, mcp: context.mcp }),
+    {
+      cwd: workDir,
+      prompt: readerPrompt(question),
+      timeoutMs: context.readerTimeoutMs,
+    }
+  );
+  const answer = await readAgentOutput(context.kind, reader, answerPath);
+  if (answer.isError) {
+    return {
+      ...errored(question, `reader ${answer.detail ?? "failed"}`, elapsed()),
+      costUsd: answer.costUsd,
+    };
+  }
+
+  const verdictPath = join(workDir, "verdict.txt");
+  const judge = await context.run(
+    context.bin,
+    agentArgs(context.kind, { lastMessagePath: verdictPath }),
+    {
+      cwd: workDir,
+      prompt: judgePrompt(question, answer.text),
+      timeoutMs: context.judgeTimeoutMs,
+    }
+  );
+  const graded = await readAgentOutput(context.kind, judge, verdictPath);
+  const costUsd =
+    answer.costUsd === undefined && graded.costUsd === undefined
+      ? undefined
+      : (answer.costUsd ?? 0) + (graded.costUsd ?? 0);
+  if (graded.isError) {
+    return {
+      ...errored(question, `judge ${graded.detail ?? "failed"}`, elapsed()),
+      answer: answer.text,
+      costUsd,
+    };
+  }
+
+  const verdict = parseVerdict(graded.text);
+  if (!verdict) {
+    return {
+      ...errored(question, "judge returned no parseable verdict", elapsed()),
+      answer: answer.text,
+      costUsd,
+    };
+  }
+
+  return {
+    answer: answer.text,
+    costUsd,
+    durationMs: elapsed(),
+    expected: question.expected,
+    id: question.id,
+    missing: verdict.missing,
+    notes: verdict.notes || undefined,
+    question: question.question,
+    routes: question.routes,
+    score: verdict.score,
+    status: verdict.pass ? "pass" : "fail",
+  };
+};
+
+/**
+ * Run every question through the reader/judge pair, sequentially: each
+ * question is already two agent sessions, and a serial run keeps progress
+ * output ordered and cost attribution obvious.
+ */
+export const runEval = async (options: EvalRunOptions): Promise<EvalResult> => {
+  const started = performance.now();
+  const kind = options.agent;
+  const run = options.run ?? runAgentHeadless;
+  const anchor = { path: options.evalsPath, raw: options.rawEvals };
+
+  const dir = await mkdtemp(join(tmpdir(), "blume-eval-"));
+  const snapshotPath = join(dir, "mcp-data.json");
+  await writeFile(
+    snapshotPath,
+    JSON.stringify(await buildMcpData(options.project))
+  );
+  const mcp = await writeMcpConfig(dir, snapshotPath);
+
+  const context: QuestionContext = {
+    bin: AGENTS[kind].bin,
+    dir,
+    judgeTimeoutMs: options.judgeTimeoutMs ?? DEFAULT_JUDGE_TIMEOUT_MS,
+    kind,
+    mcp,
+    readerTimeoutMs: options.readerTimeoutMs ?? DEFAULT_READER_TIMEOUT_MS,
+    run,
+  };
+
+  const diagnostics: Diagnostic[] = [];
+  const results: QuestionResult[] = [];
+  const { questions } = options.evals;
+
+  for (const [index, question] of questions.entries()) {
+    diagnostics.push(...routeFindings(question, options.project, anchor));
+
+    if (question.skip) {
+      results.push({
+        durationMs: 0,
+        expected: question.expected,
+        id: question.id,
+        missing: [],
+        question: question.question,
+        routes: question.routes,
+        status: "skip",
+      });
+      continue;
+    }
+
+    options.onProgress?.({
+      id: question.id,
+      index,
+      kind: "question-start",
+      total: questions.length,
+    });
+    // Sequential by design: each question is already two agent sessions, and
+    // a serial run keeps progress output ordered and cost attribution obvious.
+    // oxlint-disable-next-line no-await-in-loop
+    const result = await runQuestion(question, index, context);
+    results.push(result);
+    if (result.status === "fail" || result.status === "error") {
+      diagnostics.push(
+        questionFinding(
+          question,
+          {
+            detail: result.detail,
+            missing: result.missing,
+            status: result.status,
+          },
+          options.project,
+          anchor
+        )
+      );
+    }
+    options.onProgress?.({
+      index,
+      kind: "question-end",
+      result,
+      total: questions.length,
+    });
+  }
+
+  const counts: Record<QuestionStatus, number> = {
+    error: 0,
+    fail: 0,
+    pass: 0,
+    skip: 0,
+  };
+  for (const result of results) {
+    counts[result.status] += 1;
+  }
+  const costs = results.flatMap((result) =>
+    result.costUsd === undefined ? [] : [result.costUsd]
+  );
+
+  return {
+    agent: kind,
+    costUsd:
+      costs.length > 0
+        ? costs.reduce((total, cost) => total + cost, 0)
+        : undefined,
+    counts,
+    diagnostics,
+    durationMs: Math.round(performance.now() - started),
+    results,
+  };
+};