blume 1.1.4 → 1.2.0

package/src/eval/report.ts

@@ -0,0 +1,214 @@
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+
+import { join, relative } from "pathe";
+
+import { AGENTS } from "../audit/agent.ts";
+import { countBySeverity } from "../core/diagnostics.ts";
+import type { EvalResult, QuestionResult, QuestionStatus } from "./run.ts";
+
+const ESC = String.fromCodePoint(27);
+const COLORS = {
+  bold: `${ESC}[1m`,
+  cyan: `${ESC}[36m`,
+  dim: `${ESC}[2m`,
+  green: `${ESC}[32m`,
+  red: `${ESC}[31m`,
+  reset: `${ESC}[0m`,
+  yellow: `${ESC}[33m`,
+};
+
+const GLYPH: Record<QuestionStatus, string> = {
+  error: "!",
+  fail: "✖",
+  pass: "✔",
+  skip: "⊘",
+};
+
+const STATUS_COLOR: Record<QuestionStatus, string> = {
+  error: COLORS.yellow,
+  fail: COLORS.red,
+  pass: COLORS.green,
+  skip: COLORS.dim,
+};
+
+/** Longest id gets the room; everything shorter aligns to it. */
+const ID_PAD = 28;
+
+const seconds = (ms: number): string => `${(ms / 1000).toFixed(1)}s`;
+
+const money = (cost: number | undefined): string =>
+  cost === undefined ? "" : `$${cost.toFixed(2)}`;
+
+const duration = (ms: number): string => {
+  if (ms < 60_000) {
+    return seconds(ms);
+  }
+  const minutes = Math.floor(ms / 60_000);
+  const rest = Math.round((ms % 60_000) / 1000);
+  return `${minutes}m ${rest}s`;
+};
+
+/** One question's progress/report line: glyph, id, status, score, time, cost. */
+export const questionLine = (result: QuestionResult): string => {
+  const color = STATUS_COLOR[result.status];
+  const glyph = `${color}${GLYPH[result.status]}${COLORS.reset}`;
+  const id = result.id.padEnd(ID_PAD);
+  if (result.status === "skip") {
+    return `  ${glyph} ${id} ${COLORS.dim}skipped${COLORS.reset}`;
+  }
+  const score = result.score === undefined ? "" : result.score.toFixed(2);
+  const cells = [
+    `${color}${result.status}${COLORS.reset}`,
+    score,
+    `${COLORS.dim}${seconds(result.durationMs)}${COLORS.reset}`,
+    `${COLORS.dim}${money(result.costUsd)}${COLORS.reset}`,
+  ]
+    .filter((cell) => cell !== "")
+    .join("  ");
+  return `  ${glyph} ${id} ${cells}`;
+};
+
+/** Indented context under a question's line: missing facts, error detail. */
+export const questionDetails = (
+  result: QuestionResult,
+  verbose: boolean
+): string[] => {
+  const lines: string[] = [];
+  if (result.status === "fail") {
+    for (const fact of result.missing) {
+      lines.push(`      ${COLORS.dim}missing: ${fact}${COLORS.reset}`);
+    }
+  }
+  if (result.status === "error" && result.detail) {
+    lines.push(`      ${COLORS.dim}${result.detail}${COLORS.reset}`);
+  }
+  if (verbose && result.answer && result.status !== "pass") {
+    lines.push(
+      ...result.answer
+        .split("\n")
+        .map((line) => `      ${COLORS.dim}> ${line}${COLORS.reset}`)
+    );
+  }
+  return lines;
+};
+
+/** The one-line totals: `9 passed · 2 failed · 1 skipped · 1m 42s · $0.71`. */
+export const summaryLine = (result: EvalResult): string => {
+  const { counts } = result;
+  const parts = [
+    `${counts.pass} passed`,
+    counts.fail > 0 ? `${counts.fail} failed` : "",
+    counts.error > 0 ? `${counts.error} errored` : "",
+    counts.skip > 0 ? `${counts.skip} skipped` : "",
+    duration(result.durationMs),
+    money(result.costUsd),
+  ].filter((part) => part !== "");
+  return parts.join(" · ");
+};
+
+/** The header line the command prints before the first question runs. */
+export const headerLine = (total: number, agent: EvalResult["agent"]): string =>
+  `${COLORS.bold}blume eval${COLORS.reset}  ${total} question(s) · ${AGENTS[agent].name}`;
+
+/** The dim announce line while a question's agents run. */
+export const startLine = (id: string, index: number, total: number): string =>
+  `  ${COLORS.dim}▸ ${id} (${index + 1}/${total})${COLORS.reset}`;
+
+/** `fix:` pointers for failed questions, naming the file that resolves each. */
+export const fixLines = (result: EvalResult, root: string): string[] =>
+  result.diagnostics
+    .filter((diagnostic) => diagnostic.code !== "BLUME_EVAL_ROUTE_UNKNOWN")
+    .map((finding) => {
+      const site = finding.file
+        ? `${relative(root, finding.file)}${finding.line ? `:${finding.line}` : ""}`
+        : "";
+      return `  ${COLORS.cyan}fix:${COLORS.reset} ${site} ${COLORS.dim}${finding.message}${COLORS.reset}`;
+    });
+
+/** Dim warnings for route hints that no longer match a page. */
+export const warningLines = (result: EvalResult, root: string): string[] =>
+  result.diagnostics
+    .filter((diagnostic) => diagnostic.code === "BLUME_EVAL_ROUTE_UNKNOWN")
+    .map((finding) => {
+      const site = finding.file
+        ? ` ${relative(root, finding.file)}${finding.line ? `:${finding.line}` : ""}`
+        : "";
+      return `  ${COLORS.yellow}⚠${COLORS.reset}${site} ${COLORS.dim}${finding.message}${COLORS.reset}`;
+    });
+
+/** The human report, written to stderr by the command. */
+export const formatEvalReport = (
+  result: EvalResult,
+  root: string,
+  options: { verbose?: boolean } = {}
+): string => {
+  const lines: string[] = [headerLine(result.results.length, result.agent), ""];
+
+  for (const question of result.results) {
+    lines.push(
+      questionLine(question),
+      ...questionDetails(question, Boolean(options.verbose))
+    );
+  }
+  lines.push("");
+
+  // The finding tells the author which file fixes which failure.
+  const failures = fixLines(result, root);
+  lines.push(...failures);
+  if (failures.length > 0) {
+    lines.push("");
+  }
+
+  lines.push(`  ${summaryLine(result)}`, "");
+  return lines.join("\n");
+};
+
+/**
+ * The machine-readable report. The `diagnostics` + `summary` shape matches
+ * `blume validate --json` and `blume audit --json` exactly — anything parsing
+ * those keeps working — with the eval run's own results alongside.
+ */
+export const evalReportJson = (
+  result: EvalResult,
+  root: string,
+  threshold: number
+): string => {
+  const diagnostics = result.diagnostics.map((diagnostic) =>
+    diagnostic.file
+      ? { ...diagnostic, file: relative(root, diagnostic.file) }
+      : diagnostic
+  );
+  return `${JSON.stringify(
+    {
+      diagnostics,
+      eval: {
+        agent: result.agent,
+        costUsd: result.costUsd,
+        counts: result.counts,
+        durationMs: result.durationMs,
+        results: result.results,
+        threshold,
+      },
+      summary: countBySeverity(result.diagnostics),
+    },
+    null,
+    2
+  )}\n`;
+};
+
+/**
+ * Write the full JSON report where a `--fix` agent can read it — a file
+ * rather than inline prompt text, because a long run's answers would exceed
+ * the platform's argv limit.
+ */
+export const writeEvalReport = async (
+  result: EvalResult,
+  root: string,
+  threshold: number
+): Promise<string> => {
+  const dir = await mkdtemp(join(tmpdir(), "blume-eval-"));
+  const path = join(dir, "report.json");
+  await writeFile(path, evalReportJson(result, root, threshold));
+  return path;
+};

package/src/eval/run.ts

@@ -0,0 +1,290 @@
+import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+
+import { join } from "pathe";
+
+import { buildMcpData } from "../ai/mcp/data.ts";
+import { AGENTS } from "../audit/agent.ts";
+import type { AgentKind } from "../audit/agent.ts";
+import type { BlumeProject } from "../core/project-graph.ts";
+import type { Diagnostic } from "../core/types.ts";
+import {
+  agentArgs,
+  parseVerdict,
+  readAgentOutput,
+  runAgentHeadless,
+  writeMcpConfig,
+} from "./agents.ts";
+import type { HeadlessRunner } from "./agents.ts";
+import { questionFinding, routeFindings } from "./findings.ts";
+import { judgePrompt, readerPrompt } from "./prompts.ts";
+import type { EvalQuestion, EvalsFile } from "./schema.ts";
+
+/** Reader runs search and read several pages; the judge grades one answer. */
+const DEFAULT_READER_TIMEOUT_MS = 180_000;
+const DEFAULT_JUDGE_TIMEOUT_MS = 60_000;
+
+export type QuestionStatus = "error" | "fail" | "pass" | "skip";
+
+/** One question's outcome, carrying everything the reports print. */
+export interface QuestionResult {
+  answer?: string;
+  costUsd?: number;
+  /** Why the run errored (timeout, agent failure, unparseable verdict). */
+  detail?: string;
+  durationMs: number;
+  expected: string[];
+  id: string;
+  missing: string[];
+  notes?: string;
+  question: string;
+  routes: string[];
+  score?: number;
+  status: QuestionStatus;
+}
+
+export interface EvalResult {
+  agent: AgentKind;
+  /** Total spend, when the agent CLI reports it (claude does, codex doesn't). */
+  costUsd?: number;
+  counts: Record<QuestionStatus, number>;
+  diagnostics: Diagnostic[];
+  durationMs: number;
+  results: QuestionResult[];
+}
+
+export type EvalProgress =
+  | {
+      kind: "question-end";
+      index: number;
+      result: QuestionResult;
+      total: number;
+    }
+  | { kind: "question-start"; id: string; index: number; total: number };
+
+export interface EvalRunOptions {
+  agent: AgentKind;
+  evals: EvalsFile;
+  /** Where the evals file lives, for findings with no usable route hint. */
+  evalsPath: string;
+  judgeTimeoutMs?: number;
+  onProgress?: (event: EvalProgress) => void;
+  project: BlumeProject;
+  /** The evals file's raw text, for line-anchoring findings. */
+  rawEvals: string;
+  readerTimeoutMs?: number;
+  /** The spawn function — injectable so tests never launch a real agent. */
+  run?: HeadlessRunner;
+}
+
+interface QuestionContext {
+  bin: string;
+  dir: string;
+  judgeTimeoutMs: number;
+  kind: AgentKind;
+  mcp: Awaited<ReturnType<typeof writeMcpConfig>>;
+  readerTimeoutMs: number;
+  run: HeadlessRunner;
+}
+
+const errored = (
+  question: EvalQuestion,
+  detail: string,
+  durationMs: number
+): QuestionResult => ({
+  detail,
+  durationMs,
+  expected: question.expected,
+  id: question.id,
+  missing: [],
+  question: question.question,
+  routes: question.routes,
+  status: "error",
+});
+
+/** Run one question: a fresh empty cwd, the reader, then the judge. */
+const runQuestion = async (
+  question: EvalQuestion,
+  index: number,
+  context: QuestionContext
+): Promise<QuestionResult> => {
+  const started = performance.now();
+  const elapsed = () => Math.round(performance.now() - started);
+
+  // An empty working directory per invocation is the fresh-eyes guardrail:
+  // even if a tool restriction slips, there is nothing here to read.
+  const workDir = join(context.dir, `work-${index}`);
+  await mkdir(workDir, { recursive: true });
+
+  const answerPath = join(workDir, "answer.txt");
+  const reader = await context.run(
+    context.bin,
+    agentArgs(context.kind, { lastMessagePath: answerPath, mcp: context.mcp }),
+    {
+      cwd: workDir,
+      prompt: readerPrompt(question),
+      timeoutMs: context.readerTimeoutMs,
+    }
+  );
+  const answer = await readAgentOutput(context.kind, reader, answerPath);
+  if (answer.isError) {
+    return {
+      ...errored(question, `reader ${answer.detail ?? "failed"}`, elapsed()),
+      costUsd: answer.costUsd,
+    };
+  }
+
+  const verdictPath = join(workDir, "verdict.txt");
+  const judge = await context.run(
+    context.bin,
+    agentArgs(context.kind, { lastMessagePath: verdictPath }),
+    {
+      cwd: workDir,
+      prompt: judgePrompt(question, answer.text),
+      timeoutMs: context.judgeTimeoutMs,
+    }
+  );
+  const graded = await readAgentOutput(context.kind, judge, verdictPath);
+  const costUsd =
+    answer.costUsd === undefined && graded.costUsd === undefined
+      ? undefined
+      : (answer.costUsd ?? 0) + (graded.costUsd ?? 0);
+  if (graded.isError) {
+    return {
+      ...errored(question, `judge ${graded.detail ?? "failed"}`, elapsed()),
+      answer: answer.text,
+      costUsd,
+    };
+  }
+
+  const verdict = parseVerdict(graded.text);
+  if (!verdict) {
+    return {
+      ...errored(question, "judge returned no parseable verdict", elapsed()),
+      answer: answer.text,
+      costUsd,
+    };
+  }
+
+  return {
+    answer: answer.text,
+    costUsd,
+    durationMs: elapsed(),
+    expected: question.expected,
+    id: question.id,
+    missing: verdict.missing,
+    notes: verdict.notes || undefined,
+    question: question.question,
+    routes: question.routes,
+    score: verdict.score,
+    status: verdict.pass ? "pass" : "fail",
+  };
+};
+
+/**
+ * Run every question through the reader/judge pair, sequentially: each
+ * question is already two agent sessions, and a serial run keeps progress
+ * output ordered and cost attribution obvious.
+ */
+export const runEval = async (options: EvalRunOptions): Promise<EvalResult> => {
+  const started = performance.now();
+  const kind = options.agent;
+  const run = options.run ?? runAgentHeadless;
+  const anchor = { path: options.evalsPath, raw: options.rawEvals };
+
+  const dir = await mkdtemp(join(tmpdir(), "blume-eval-"));
+  const snapshotPath = join(dir, "mcp-data.json");
+  await writeFile(
+    snapshotPath,
+    JSON.stringify(await buildMcpData(options.project))
+  );
+  const mcp = await writeMcpConfig(dir, snapshotPath);
+
+  const context: QuestionContext = {
+    bin: AGENTS[kind].bin,
+    dir,
+    judgeTimeoutMs: options.judgeTimeoutMs ?? DEFAULT_JUDGE_TIMEOUT_MS,
+    kind,
+    mcp,
+    readerTimeoutMs: options.readerTimeoutMs ?? DEFAULT_READER_TIMEOUT_MS,
+    run,
+  };
+
+  const diagnostics: Diagnostic[] = [];
+  const results: QuestionResult[] = [];
+  const { questions } = options.evals;
+
+  for (const [index, question] of questions.entries()) {
+    diagnostics.push(...routeFindings(question, options.project, anchor));
+
+    if (question.skip) {
+      results.push({
+        durationMs: 0,
+        expected: question.expected,
+        id: question.id,
+        missing: [],
+        question: question.question,
+        routes: question.routes,
+        status: "skip",
+      });
+      continue;
+    }
+
+    options.onProgress?.({
+      id: question.id,
+      index,
+      kind: "question-start",
+      total: questions.length,
+    });
+    // Sequential by design: each question is already two agent sessions, and
+    // a serial run keeps progress output ordered and cost attribution obvious.
+    // oxlint-disable-next-line no-await-in-loop
+    const result = await runQuestion(question, index, context);
+    results.push(result);
+    if (result.status === "fail" || result.status === "error") {
+      diagnostics.push(
+        questionFinding(
+          question,
+          {
+            detail: result.detail,
+            missing: result.missing,
+            status: result.status,
+          },
+          options.project,
+          anchor
+        )
+      );
+    }
+    options.onProgress?.({
+      index,
+      kind: "question-end",
+      result,
+      total: questions.length,
+    });
+  }
+
+  const counts: Record<QuestionStatus, number> = {
+    error: 0,
+    fail: 0,
+    pass: 0,
+    skip: 0,
+  };
+  for (const result of results) {
+    counts[result.status] += 1;
+  }
+  const costs = results.flatMap((result) =>
+    result.costUsd === undefined ? [] : [result.costUsd]
+  );
+
+  return {
+    agent: kind,
+    costUsd:
+      costs.length > 0
+        ? costs.reduce((total, cost) => total + cost, 0)
+        : undefined,
+    counts,
+    diagnostics,
+    durationMs: Math.round(performance.now() - started),
+    results,
+  };
+};

package/src/eval/schema.ts

@@ -0,0 +1,124 @@
+import { readFile } from "node:fs/promises";
+
+import { load } from "js-yaml";
+import { z } from "zod";
+
+/** Question ids are kebab-case slugs so they read well in reports and CI logs. */
+const ID_PATTERN = /^[a-z0-9][a-z0-9-]*$/u;
+
+const questionSchema = z.strictObject({
+  expected: z
+    .array(z.string().min(1))
+    .min(1, "expected must list at least one fact"),
+  id: z
+    .string()
+    .regex(ID_PATTERN, "id must be a kebab-case slug (a-z, 0-9, dashes)"),
+  question: z.string().min(1),
+  routes: z
+    .union([z.string(), z.array(z.string())])
+    .default([])
+    .transform((value) => (typeof value === "string" ? [value] : value)),
+  severity: z.enum(["error", "warning"]).default("error"),
+  skip: z.boolean().default(false),
+});
+
+/** One author-written eval: a question plus the facts a passing answer states. */
+export type EvalQuestion = z.infer<typeof questionSchema>;
+
+const fullSchema = z.strictObject({
+  questions: z.array(questionSchema).min(1),
+  version: z.literal(1).default(1),
+});
+
+/**
+ * The evals file schema. A bare top-level list of questions is accepted as
+ * shorthand — `loadEvalsFile` wraps it before validating, rather than a
+ * `z.union`, so schema errors name the offending field instead of collapsing
+ * into an opaque "invalid union" issue.
+ */
+export const evalsFileSchema = fullSchema.superRefine((value, context) => {
+  const seen = new Set<string>();
+  for (const question of value.questions) {
+    if (seen.has(question.id)) {
+      context.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `duplicate question id "${question.id}"`,
+        path: ["questions"],
+      });
+    }
+    seen.add(question.id);
+  }
+});
+
+export type EvalsFile = z.infer<typeof evalsFileSchema>;
+
+/** A problem loading or validating the evals file, with the path it names. */
+export class EvalsFileError extends Error {
+  readonly path: string;
+
+  constructor(path: string, message: string) {
+    super(message);
+    this.name = "EvalsFileError";
+    this.path = path;
+  }
+}
+
+const describeIssues = (error: z.ZodError): string =>
+  error.issues
+    .map((issue) => {
+      const at = issue.path.length > 0 ? ` at ${issue.path.join(".")}` : "";
+      return `${issue.message}${at}`;
+    })
+    .join("; ");
+
+/**
+ * Read and validate an evals file. Returns the parsed questions plus the raw
+ * text, kept so findings can anchor to the line a question is defined on.
+ */
+export const loadEvalsFile = async (
+  path: string
+): Promise<{ evals: EvalsFile; raw: string }> => {
+  let raw: string;
+  try {
+    raw = await readFile(path, "utf-8");
+  } catch {
+    throw new EvalsFileError(
+      path,
+      `No evals file found at ${path}. Run \`blume eval init\` to draft one.`
+    );
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = load(raw);
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new EvalsFileError(path, `Invalid YAML in ${path}: ${detail}`);
+  }
+
+  // The bare-list shorthand: a top-level sequence of questions.
+  const candidate = Array.isArray(parsed) ? { questions: parsed } : parsed;
+  const result = evalsFileSchema.safeParse(candidate);
+  if (!result.success) {
+    throw new EvalsFileError(
+      path,
+      `Invalid evals file at ${path}: ${describeIssues(result.error)}`
+    );
+  }
+  return { evals: result.data, raw };
+};
+
+/**
+ * The 1-based line where a question's `id:` entry appears in the raw evals
+ * file, so a finding with no route hint can still point somewhere editable.
+ */
+export const locateQuestion = (raw: string, id: string): number | undefined => {
+  const pattern = new RegExp(`^\\s*-?\\s*id:\\s*["']?${id}["']?\\s*$`, "u");
+  const lines = raw.split("\n");
+  for (const [index, line] of lines.entries()) {
+    if (pattern.test(line)) {
+      return index + 1;
+    }
+  }
+  return undefined;
+};

package/src/openapi/references.ts

@@ -38,6 +38,12 @@ export interface ReferenceSource {
    */
   basePath: string;
   label: string;
+  /** Whether generated pages are included in llms.txt/llms-full.txt. */
+  includeInLlms: boolean;
+  /** Whether generated pages are included in site search. */
+  includeInSearch: boolean;
+  /** Whether generated pages emit noindex metadata and stay out of the sitemap. */
+  noindex: boolean;
   /** Local path or `http(s)` URL, verbatim from config. */
   spec: string;
   /** Per-block Scalar theme name override, if any (Scalar renderer only). */
@@ -78,10 +84,22 @@ type Block = ResolvedConfig["openapi"] | ResolvedConfig["asyncapi"];
 /** A spec is a single source (`spec` shorthand prepended to any `sources`). */
 const sourcesOf = (
   block: Block
-): { label?: string; route?: string; spec: string }[] => {
+): {
+  includeInLlms: boolean;
+  includeInSearch: boolean;
+  label?: string;
+  noindex: boolean;
+  route?: string;
+  spec: string;
+}[] => {
   const sources = [...block.sources];
   if (block.spec) {
-    sources.unshift({ spec: block.spec });
+    sources.unshift({
+      includeInLlms: true,
+      includeInSearch: true,
+      noindex: false,
+      spec: block.spec,
+    });
   }
   return sources;
 };
@@ -118,8 +136,11 @@ const referencesFor = (
     return {
       basePath,
       display,
+      includeInLlms: source.includeInLlms,
+      includeInSearch: source.includeInSearch,
       kind,
       label,
+      noindex: source.noindex,
       renderer,
       route,
       scalar: block.scalar,