blume 1.6.5 → 1.6.6

package/dist/cli/chunk-0ewz4trd.js

@@ -0,0 +1,679 @@
+#!/usr/bin/env node
+import {
+  agentArgs,
+  duration,
+  money,
+  parseVerdict,
+  readAgentOutput,
+  runAgentHeadless,
+  seconds,
+  writeMcpConfig
+} from "./chunk-s5e5jt53.js";
+import {
+  AGENTS,
+  launchAgent
+} from "./chunk-8gnpdsn1.js";
+import {
+  reportInternalError
+} from "./chunk-btfr9yvw.js";
+import {
+  buildMcpData
+} from "./chunk-n4qjabmt.js";
+import"./chunk-5yvt556e.js";
+import"./chunk-3k0kzs6d.js";
+import"./chunk-vxv4x1n8.js";
+import {
+  YAML_SCHEMA,
+  scanProject
+} from "./chunk-etsqspj6.js";
+import"./chunk-vv3f8mb6.js";
+import"./chunk-27gtm2ym.js";
+import"./chunk-4xyggvgf.js";
+import"./chunk-6kzzpsx8.js";
+import {
+  BlumeError,
+  countBySeverity,
+  flushStdout,
+  logger
+} from "./chunk-ey89bjj1.js";
+import {
+  commandMeta
+} from "./chunk-2aj8ddew.js";
+
+// src/cli/commands/eval.ts
+import { existsSync } from "node:fs";
+import { defineCommand } from "citty";
+import { join as join3 } from "pathe";
+
+// src/eval/prompts.ts
+var readerPrompt = (question) => `You are evaluating whether a product's documentation can answer a user's question.
+
+Answer the question below using ONLY the connected documentation tools (search_docs, get_page, list_pages, get_navigation). Rules:
+- Do not use prior knowledge about the product. Do not guess.
+- Do not read files, run commands, or access the network.
+- Search first, then read the most relevant pages with get_page.
+- If the documentation does not contain the answer, say exactly what information is missing instead of inventing one.
+
+Question: ${question.question}
+
+Reply with a concise answer containing the specific facts the documentation provides. Plain text only.`;
+var judgePrompt = (question, answer) => {
+  const facts = question.expected.map((fact) => `- ${fact}`).join(`
+`);
+  return `You are grading an answer against expected facts. Do not use any tools.
+
+Question: ${question.question}
+
+Expected facts — each must be present in substance (paraphrase is fine, contradiction is not):
+${facts}
+
+Answer to grade:
+"""
+${answer}
+"""
+
+An answer that states the documentation lacks the information FAILS.
+
+Reply with ONLY this JSON object on a single line, no markdown fences:
+{"pass": true|false, "score": 0.0-1.0, "missing": ["expected facts absent or contradicted"], "notes": "one sentence"}`;
+};
+var evalFixPrompt = (reportPath) => `Fix the documentation gaps found by \`blume eval\` in this project.
+
+The full report is at ${reportPath}. It is JSON: each entry in \`eval.results\` with status "fail" is one question the documentation could not answer. Each carries the \`question\`, the \`expected\` facts, the judge's \`missing\` facts, and the reader agent's \`answer\` (what the docs currently convey). The matching \`diagnostics\` entry names the source \`file\` of the page that should answer it.
+
+Work through every failed question:
+1. Read the page named in the finding (or choose the best page when none is named).
+2. Edit the documentation so it states the missing facts explicitly. Add prose, not filler; keep the page's voice.
+3. Never delete questions from the evals file or weaken expected facts.
+
+When you are done, run \`blume eval\` to verify, and repeat until every question passes.`;
+var initPrompt = (evalsPath) => `Draft a starter evals file for \`blume eval\` in this documentation project.
+
+Read the documentation source pages in this project and write ${evalsPath} with about 10 high-value questions a real user would ask — installation, configuration, deployment, and the project's headline features. For each question, list the expected facts a correct answer must state, grounded in what the documentation actually promises (never invent facts the docs don't state).
+
+The file format is YAML:
+
+questions:
+  - id: kebab-case-slug
+    question: One user question?
+    expected:
+      - a fact the answer must contain
+      - another required fact
+    routes:
+      - /route/of/the/page/that/answers/it
+
+Rules:
+- Every \`expected\` fact must be verifiable in the docs today.
+- Prefer questions whose answers live on one page; set \`routes\` to that page.
+- Keep ids unique and questions short.
+
+When you are done, print the file and suggest running \`blume eval\` to try it.`;
+
+// src/eval/report.ts
+import { mkdtemp, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { colors } from "consola/utils";
+import { join, relative } from "pathe";
+var GLYPH = {
+  error: "!",
+  fail: "✖",
+  pass: "✔",
+  skip: "⊘"
+};
+var STATUS_COLOR = {
+  error: colors.yellow,
+  fail: colors.red,
+  pass: colors.green,
+  skip: colors.dim
+};
+var ID_PAD = 28;
+var questionLine = (result) => {
+  const color = STATUS_COLOR[result.status];
+  const glyph = color(GLYPH[result.status]);
+  const id = result.id.padEnd(ID_PAD);
+  if (result.status === "skip") {
+    return `  ${glyph} ${id} ${colors.dim("skipped")}`;
+  }
+  const score = result.score === undefined ? "" : result.score.toFixed(2);
+  const cost = money(result.costUsd);
+  const cells = [
+    color(result.status),
+    score,
+    colors.dim(seconds(result.durationMs)),
+    cost === "" ? "" : colors.dim(cost)
+  ].filter((cell) => cell !== "").join("  ");
+  return `  ${glyph} ${id} ${cells}`;
+};
+var questionDetails = (result, verbose) => {
+  const lines = [];
+  if (result.status === "fail") {
+    for (const fact of result.missing) {
+      lines.push(`      ${colors.dim(`missing: ${fact}`)}`);
+    }
+  }
+  if (result.status === "error" && result.detail) {
+    lines.push(`      ${colors.dim(result.detail)}`);
+  }
+  if (verbose && result.answer && result.status !== "pass") {
+    lines.push(...result.answer.split(`
+`).map((line) => `      ${colors.dim(`> ${line}`)}`));
+  }
+  return lines;
+};
+var summaryLine = (result) => {
+  const { counts } = result;
+  const parts = [
+    `${counts.pass} passed`,
+    counts.fail > 0 ? `${counts.fail} failed` : "",
+    counts.error > 0 ? `${counts.error} errored` : "",
+    counts.skip > 0 ? `${counts.skip} skipped` : "",
+    duration(result.durationMs),
+    money(result.costUsd)
+  ].filter((part) => part !== "");
+  return parts.join(" · ");
+};
+var headerLine = (total, agent) => `${colors.bold("blume eval")}  ${total} question(s) · ${AGENTS[agent].name}`;
+var startLine = (id, index, total) => `  ${colors.dim(`▸ ${id} (${index + 1}/${total})`)}`;
+var fixLines = (result, root) => result.diagnostics.filter((diagnostic) => diagnostic.code !== "BLUME_EVAL_ROUTE_UNKNOWN").map((finding) => {
+  const site = finding.file ? `${relative(root, finding.file)}${finding.line ? `:${finding.line}` : ""}` : "";
+  return `  ${colors.cyan("fix:")} ${site} ${colors.dim(finding.message)}`;
+});
+var warningLines = (result, root) => result.diagnostics.filter((diagnostic) => diagnostic.code === "BLUME_EVAL_ROUTE_UNKNOWN").map((finding) => {
+  const site = finding.file ? ` ${relative(root, finding.file)}${finding.line ? `:${finding.line}` : ""}` : "";
+  return `  ${colors.yellow("⚠")}${site} ${colors.dim(finding.message)}`;
+});
+var evalReportJson = (result, root, threshold) => {
+  const diagnostics = result.diagnostics.map((diagnostic) => diagnostic.file ? { ...diagnostic, file: relative(root, diagnostic.file) } : diagnostic);
+  return `${JSON.stringify({
+    diagnostics,
+    eval: {
+      agent: result.agent,
+      costUsd: result.costUsd,
+      counts: result.counts,
+      durationMs: result.durationMs,
+      results: result.results,
+      threshold
+    },
+    summary: countBySeverity(result.diagnostics)
+  }, null, 2)}
+`;
+};
+var writeEvalReport = async (result, root, threshold) => {
+  const dir = await mkdtemp(join(tmpdir(), "blume-eval-"));
+  const path = join(dir, "report.json");
+  await writeFile(path, evalReportJson(result, root, threshold));
+  return path;
+};
+
+// src/eval/run.ts
+import { mkdir, mkdtemp as mkdtemp2, writeFile as writeFile2 } from "node:fs/promises";
+import { tmpdir as tmpdir2 } from "node:os";
+import { join as join2 } from "pathe";
+
+// src/eval/schema.ts
+import { readFile } from "node:fs/promises";
+import { load } from "js-yaml";
+import { z } from "zod";
+var ID_PATTERN = /^[a-z0-9][a-z0-9-]*$/u;
+var questionSchema = z.strictObject({
+  expected: z.array(z.string().min(1)).min(1, "expected must list at least one fact"),
+  id: z.string().regex(ID_PATTERN, "id must be a kebab-case slug (a-z, 0-9, dashes)"),
+  question: z.string().min(1),
+  routes: z.union([z.string(), z.array(z.string())]).default([]).transform((value) => Array.isArray(value) ? value : [value]),
+  severity: z.enum(["error", "warning"]).default("error"),
+  skip: z.boolean().default(false)
+});
+var fullSchema = z.strictObject({
+  questions: z.array(questionSchema).min(1),
+  version: z.literal(1).default(1)
+});
+var evalsFileSchema = fullSchema.superRefine((value, context) => {
+  const seen = new Set;
+  for (const question of value.questions) {
+    if (seen.has(question.id)) {
+      context.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `duplicate question id "${question.id}"`,
+        path: ["questions"]
+      });
+    }
+    seen.add(question.id);
+  }
+});
+
+class EvalsFileError extends Error {
+  path;
+  constructor(path, message) {
+    super(message);
+    this.name = "EvalsFileError";
+    this.path = path;
+  }
+}
+var describeIssues = (error) => error.issues.map((issue) => {
+  const at = issue.path.length > 0 ? ` at ${issue.path.join(".")}` : "";
+  return `${issue.message}${at}`;
+}).join("; ");
+var loadEvalsFile = async (path) => {
+  let raw;
+  try {
+    raw = await readFile(path, "utf-8");
+  } catch {
+    throw new EvalsFileError(path, `No evals file found at ${path}. Run \`blume eval init\` to draft one.`);
+  }
+  let parsed;
+  try {
+    parsed = load(raw, { schema: YAML_SCHEMA });
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new EvalsFileError(path, `Invalid YAML in ${path}: ${detail}`);
+  }
+  const candidate = Array.isArray(parsed) ? { questions: parsed } : parsed;
+  const result = evalsFileSchema.safeParse(candidate);
+  if (!result.success) {
+    throw new EvalsFileError(path, `Invalid evals file at ${path}: ${describeIssues(result.error)}`);
+  }
+  return { evals: result.data, raw };
+};
+var locateQuestion = (raw, id) => {
+  const pattern = new RegExp(`^\\s*-?\\s*id:\\s*["']?${id}["']?\\s*$`, "u");
+  const lines = raw.split(`
+`);
+  for (const [index, line] of lines.entries()) {
+    if (pattern.test(line)) {
+      return index + 1;
+    }
+  }
+  return;
+};
+
+// src/eval/findings.ts
+var DOCS_URL = "https://useblume.dev/docs/reference/eval";
+var hintedRoute = (question, project) => {
+  for (const hint of question.routes) {
+    const route = project.manifest.routes.find((candidate) => candidate.path === hint);
+    if (route) {
+      return route;
+    }
+  }
+};
+var routeFindings = (question, project, anchor) => {
+  const known = new Set(project.manifest.routes.map((route) => route.path));
+  return question.routes.filter((hint) => !known.has(hint)).map((hint) => ({
+    code: "BLUME_EVAL_ROUTE_UNKNOWN",
+    docsUrl: DOCS_URL,
+    file: anchor.path,
+    line: locateQuestion(anchor.raw, question.id),
+    message: `Question "${question.id}" hints at route "${hint}", which matches no page.`,
+    severity: "warning",
+    suggestion: "Update the question's `routes` to the page's current route, or remove the hint."
+  }));
+};
+var questionFinding = (question, outcome, project, anchor) => {
+  const route = hintedRoute(question, project);
+  const site = route ? { file: route.sourcePath, url: route.path } : { file: anchor.path, line: locateQuestion(anchor.raw, question.id) };
+  if (outcome.status === "error") {
+    return {
+      code: "BLUME_EVAL_QUESTION_ERROR",
+      docsUrl: DOCS_URL,
+      message: `Eval run failed for "${question.question}"${outcome.detail ? ` — ${outcome.detail}` : ""}.`,
+      severity: question.severity,
+      suggestion: "Rerun `blume eval`; if it persists, check the agent CLI installation and the failure detail.",
+      ...site
+    };
+  }
+  const missing = outcome.missing.length > 0 ? ` — missing: ${outcome.missing.join("; ")}` : "";
+  return {
+    code: "BLUME_EVAL_QUESTION_FAILED",
+    docsUrl: DOCS_URL,
+    message: `Docs could not answer: "${question.question}"${missing}`,
+    severity: question.severity,
+    suggestion: "State the missing facts on this page, then rerun `blume eval`.",
+    ...site
+  };
+};
+
+// src/eval/run.ts
+var DEFAULT_READER_TIMEOUT_MS = 180000;
+var DEFAULT_JUDGE_TIMEOUT_MS = 60000;
+var errored = (question, detail, durationMs) => ({
+  detail,
+  durationMs,
+  expected: question.expected,
+  id: question.id,
+  missing: [],
+  question: question.question,
+  routes: question.routes,
+  status: "error"
+});
+var runQuestion = async (question, index, context) => {
+  const started = performance.now();
+  const elapsed = () => Math.round(performance.now() - started);
+  const workDir = join2(context.dir, `work-${index}`);
+  await mkdir(workDir, { recursive: true });
+  const answerPath = join2(workDir, "answer.txt");
+  const reader = await context.run(context.bin, agentArgs(context.kind, { lastMessagePath: answerPath, mcp: context.mcp }), {
+    cwd: workDir,
+    prompt: readerPrompt(question),
+    timeoutMs: context.readerTimeoutMs
+  });
+  const answer = await readAgentOutput(context.kind, reader, answerPath);
+  if (answer.isError) {
+    return {
+      ...errored(question, `reader ${answer.detail ?? "failed"}`, elapsed()),
+      costUsd: answer.costUsd
+    };
+  }
+  const verdictPath = join2(workDir, "verdict.txt");
+  const judge = await context.run(context.bin, agentArgs(context.kind, { lastMessagePath: verdictPath }), {
+    cwd: workDir,
+    prompt: judgePrompt(question, answer.text),
+    timeoutMs: context.judgeTimeoutMs
+  });
+  const graded = await readAgentOutput(context.kind, judge, verdictPath);
+  const costUsd = answer.costUsd === undefined && graded.costUsd === undefined ? undefined : (answer.costUsd ?? 0) + (graded.costUsd ?? 0);
+  if (graded.isError) {
+    return {
+      ...errored(question, `judge ${graded.detail ?? "failed"}`, elapsed()),
+      answer: answer.text,
+      costUsd
+    };
+  }
+  const verdict = parseVerdict(graded.text);
+  if (!verdict) {
+    return {
+      ...errored(question, "judge returned no parseable verdict", elapsed()),
+      answer: answer.text,
+      costUsd
+    };
+  }
+  return {
+    answer: answer.text,
+    costUsd,
+    durationMs: elapsed(),
+    expected: question.expected,
+    id: question.id,
+    missing: verdict.missing,
+    notes: verdict.notes || undefined,
+    question: question.question,
+    routes: question.routes,
+    score: verdict.score,
+    status: verdict.pass ? "pass" : "fail"
+  };
+};
+var runEval = async (options) => {
+  const started = performance.now();
+  const kind = options.agent;
+  const run = options.run ?? runAgentHeadless;
+  const anchor = { path: options.evalsPath, raw: options.rawEvals };
+  const dir = await mkdtemp2(join2(tmpdir2(), "blume-eval-"));
+  const snapshotPath = join2(dir, "mcp-data.json");
+  await writeFile2(snapshotPath, JSON.stringify(await buildMcpData(options.project)));
+  const mcp = await writeMcpConfig(dir, snapshotPath);
+  const context = {
+    bin: AGENTS[kind].bin,
+    dir,
+    judgeTimeoutMs: options.judgeTimeoutMs ?? DEFAULT_JUDGE_TIMEOUT_MS,
+    kind,
+    mcp,
+    readerTimeoutMs: options.readerTimeoutMs ?? DEFAULT_READER_TIMEOUT_MS,
+    run
+  };
+  const diagnostics = [];
+  const results = [];
+  const { questions } = options.evals;
+  for (const [index, question] of questions.entries()) {
+    diagnostics.push(...routeFindings(question, options.project, anchor));
+    if (question.skip) {
+      results.push({
+        durationMs: 0,
+        expected: question.expected,
+        id: question.id,
+        missing: [],
+        question: question.question,
+        routes: question.routes,
+        status: "skip"
+      });
+      continue;
+    }
+    options.onProgress?.({
+      id: question.id,
+      index,
+      kind: "question-start",
+      total: questions.length
+    });
+    const result = await runQuestion(question, index, context);
+    results.push(result);
+    if (result.status === "fail" || result.status === "error") {
+      diagnostics.push(questionFinding(question, {
+        detail: result.detail,
+        missing: result.missing,
+        status: result.status
+      }, options.project, anchor));
+    }
+    options.onProgress?.({
+      index,
+      kind: "question-end",
+      result,
+      total: questions.length
+    });
+  }
+  const counts = {
+    error: 0,
+    fail: 0,
+    pass: 0,
+    skip: 0
+  };
+  for (const result of results) {
+    counts[result.status] += 1;
+  }
+  const costs = results.flatMap((result) => result.costUsd === undefined ? [] : [result.costUsd]);
+  return {
+    agent: kind,
+    costUsd: costs.length > 0 ? costs.reduce((total, cost) => total + cost, 0) : undefined,
+    counts,
+    diagnostics,
+    durationMs: Math.round(performance.now() - started),
+    results
+  };
+};
+
+// src/cli/commands/eval.ts
+var DEFAULT_FILE = "evals.yaml";
+var DEFAULT_TIMEOUT_S = 180;
+var isAgentKind = (value) => (value in AGENTS);
+var notInstalled = (agent) => {
+  const cli = AGENTS[agent];
+  logger.error(`${cli.name} (\`${cli.bin}\`) was not found on PATH. Install it with \`${cli.install}\`.`);
+  return process.exit(1);
+};
+var launchAgentCode = async (agent, prompt) => {
+  try {
+    return await launchAgent(AGENTS[agent].bin, prompt);
+  } catch (error) {
+    if (error?.code !== "ENOENT") {
+      throw error;
+    }
+    return notInstalled(agent);
+  }
+};
+var passFraction = (result) => {
+  const ran = result.results.length - result.counts.skip;
+  return ran === 0 ? 1 : result.counts.pass / ran;
+};
+var parseFlags = (args) => {
+  if (!isAgentKind(args.agent)) {
+    logger.error(`Invalid --agent "${args.agent}" (use claude | codex).`);
+    process.exit(1);
+  }
+  if (args.action !== undefined && args.action !== "init") {
+    logger.error(`Unknown action "${args.action}" (did you mean "init"?).`);
+    process.exit(1);
+  }
+  if (args.json && args.fix) {
+    logger.error("--json and --fix are mutually exclusive.");
+    process.exit(1);
+  }
+  const threshold = args.threshold === undefined ? 1 : Number(args.threshold);
+  if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
+    logger.error(`Invalid --threshold "${args.threshold}" (use 0..1).`);
+    process.exit(1);
+  }
+  const timeoutS = args.timeout === undefined ? DEFAULT_TIMEOUT_S : Number(args.timeout);
+  if (!Number.isInteger(timeoutS) || timeoutS <= 0) {
+    logger.error(`Invalid --timeout "${args.timeout}" (whole seconds).`);
+    process.exit(1);
+  }
+  return { agent: args.agent, threshold, timeoutS };
+};
+var runFixHandoff = async (agent, result, root, threshold) => {
+  const count = result.counts.fail + result.counts.error;
+  if (count === 0) {
+    return;
+  }
+  const cli = AGENTS[agent];
+  const report = await writeEvalReport(result, root, threshold);
+  process.stderr.write(`  Handing ${count} failed question${count === 1 ? "" : "s"} to ${cli.name}…
+
+`);
+  const code = await launchAgentCode(agent, evalFixPrompt(report));
+  if (code !== 0) {
+    process.exit(code);
+  }
+};
+var runInit = async (agent, file) => {
+  const path = join3(process.cwd(), file);
+  if (existsSync(path)) {
+    logger.error(`${file} already exists — edit it directly, or pass --file to draft elsewhere.`);
+    process.exit(1);
+  }
+  const code = await launchAgentCode(agent, initPrompt(file));
+  if (code !== 0) {
+    process.exit(code);
+  }
+};
+var evalCommand = defineCommand({
+  args: {
+    action: {
+      description: 'Optional action: "init" drafts a starter evals file.',
+      required: false,
+      type: "positional"
+    },
+    agent: {
+      default: "claude",
+      description: "Agent CLI that reads and grades the docs: claude | codex.",
+      type: "string"
+    },
+    file: {
+      default: DEFAULT_FILE,
+      description: "The evals file to run.",
+      type: "string"
+    },
+    fix: {
+      description: "After a failing run, hand the report to the agent to fix the docs interactively.",
+      type: "boolean"
+    },
+    json: {
+      description: "Emit the report as JSON on stdout (for CI/editors).",
+      type: "boolean"
+    },
+    threshold: {
+      description: "Minimum passing fraction (0..1) before the run exits non-zero. Defaults to 1.",
+      type: "string"
+    },
+    timeout: {
+      description: `Reader time limit per question, in seconds. Defaults to ${DEFAULT_TIMEOUT_S}.`,
+      type: "string"
+    },
+    verbose: {
+      description: "Include the reader's full answer under each failure.",
+      type: "boolean"
+    }
+  },
+  meta: commandMeta.eval,
+  async run({ args }) {
+    const root = process.cwd();
+    const { agent, threshold, timeoutS } = parseFlags(args);
+    if (args.action === "init") {
+      await runInit(agent, args.file);
+      return;
+    }
+    let result;
+    try {
+      const project = await scanProject(root, { mode: "build" });
+      const evalsPath = join3(root, args.file);
+      const { evals, raw } = await loadEvalsFile(evalsPath);
+      process.stderr.write(`${headerLine(evals.questions.length, agent)}
+
+`);
+      result = await runEval({
+        agent,
+        evals,
+        evalsPath,
+        onProgress: (event) => {
+          if (event.kind === "question-start") {
+            process.stderr.write(`${startLine(event.id, event.index, event.total)}
+`);
+            return;
+          }
+          const lines = [
+            questionLine(event.result),
+            ...questionDetails(event.result, Boolean(args.verbose))
+          ];
+          process.stderr.write(`${lines.join(`
+`)}
+`);
+        },
+        project,
+        rawEvals: raw,
+        readerTimeoutMs: timeoutS * 1000
+      });
+    } catch (error) {
+      if (error instanceof EvalsFileError) {
+        logger.error(error.message);
+        process.exit(1);
+      }
+      if (error instanceof BlumeError) {
+        logger.error(error.diagnostic.message);
+        process.exit(1);
+      }
+      if (error?.code === "ENOENT") {
+        notInstalled(agent);
+      }
+      reportInternalError(error);
+      process.exit(1);
+    }
+    const tail = [
+      "",
+      ...warningLines(result, root),
+      ...fixLines(result, root),
+      "",
+      `  ${summaryLine(result)}`,
+      ""
+    ];
+    process.stderr.write(tail.join(`
+`));
+    const failed = passFraction(result) < threshold;
+    if (args.fix) {
+      await runFixHandoff(agent, result, root, threshold);
+      return;
+    }
+    if (args.json) {
+      process.stdout.write(evalReportJson(result, root, threshold));
+      if (failed) {
+        await flushStdout();
+        process.exit(1);
+      }
+      return;
+    }
+    if (failed) {
+      process.exit(1);
+    }
+  }
+});
+export {
+  evalCommand,
+  passFraction
+};
+
+//# debugId=D2F08D9122CA47D464756E2164756E21
+//# sourceMappingURL=chunk-0ewz4trd.js.map