@hawkeyexl/manni 4.3.0 → 4.4.0

package/dist/chunk-77HX34KD.js

@@ -0,0 +1,1287 @@
+#!/usr/bin/env node
+import {
+  EVALS_KEY,
+  EvalWriter,
+  META_PROVENANCE_KEY,
+  SCRIPTGEN_SYSTEM_PROMPT,
+  appendPageEvals,
+  assertProviderSelection,
+  buildScriptgenUser,
+  constructProvider,
+  entryObject,
+  hasEditableEval,
+  loadReviews,
+  makeGenerateScripts,
+  makeJudge,
+  makeProvider,
+  manifestEvalList,
+  recordReview,
+  resolveProviderIdentity,
+  scriptLocationFor,
+  selectProvider,
+  updateConfigEval,
+  updateManifestEval,
+  updatePageEval
+} from "./chunk-PHF3D7ER.js";
+import {
+  DEFAULT_CONFIG_FILENAME,
+  assertRegisteredGraders,
+  discoverPages,
+  documentSet,
+  leadingFrontmatterFormat,
+  loadConfig,
+  loadExternalReader,
+  loadRunConfig,
+  looksLikeOverflow,
+  partLabel,
+  readPage,
+  resolvePage,
+  resolvePages,
+  runConfigOptions,
+  sha256,
+  splitBody,
+  withExternalMetadata
+} from "./chunk-NWZ2SLYL.js";
+import {
+  isTurnBudgetSkip
+} from "./chunk-LJ2RDH3F.js";
+import {
+  SUMMARY_FORMATS,
+  parseFormat
+} from "./chunk-ZXRAE55H.js";
+import {
+  DocevalsError
+} from "./chunk-SE4JN5XN.js";
+import {
+  SEVERITIES
+} from "./chunk-77AINBYM.js";
+import {
+  mergeMetaProvenance
+} from "./chunk-K3D6FPV6.js";
+import {
+  palette
+} from "./chunk-RNVYGWIC.js";
+import {
+  warn
+} from "./chunk-Q35YIDBV.js";
+import {
+  errorMessage
+} from "./chunk-G3T47CFV.js";
+
+// src/docevals/commands/generate.ts
+async function runGenerate(paths, options = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  const config = loadRunConfig(runConfigOptions(paths, options), cwd);
+  const flags = { provider: options.provider, model: options.model, local: options.local };
+  assertProviderSelection(selectProvider(config, flags));
+  const pages = await withExternalMetadata(
+    discoverPages(config, documentSet(paths, options, "read"), cwd),
+    config,
+    cwd
+  );
+  const plans = resolvePages(pages, config);
+  assertRegisteredGraders(plans, config);
+  const targets = [];
+  const seenConfigEvals = /* @__PURE__ */ new Set();
+  for (const plan of plans) {
+    if (plan.skip || plan.problems.some((p) => p.level === "error")) continue;
+    for (const ev of plan.evals) {
+      if (ev.skip || ev.grader !== "command") continue;
+      const missing = !ev.command;
+      const stale = ev.command != null && ev.generatedAssertionHash != null && ev.assertion != null && ev.generatedAssertionHash !== sha256(ev.assertion);
+      if ((missing || stale) && ev.assertion) {
+        if (ev.source === "config") {
+          if (seenConfigEvals.has(ev.name)) continue;
+          seenConfigEvals.add(ev.name);
+        }
+        targets.push({ plan, eval: ev });
+      }
+    }
+  }
+  if (targets.length === 0) return { generatedPaths: [], targets: 0, refusals: [] };
+  const provider = options.providerInstance ?? await makeProvider(config, flags);
+  const generate = makeGenerateScripts({ provider, root: cwd });
+  const { generatedPaths, refusals } = await generate(targets, config, {});
+  return { generatedPaths, targets: targets.length, refusals: refusals ?? [] };
+}
+
+// src/docevals/commands/fill.ts
+import { writeFileSync as writeFileSync2 } from "node:fs";
+import { resolve } from "node:path";
+
+// src/docevals/fill/cache.ts
+import { mkdirSync, readFileSync, writeFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+
+// src/docevals/fill/prompt.ts
+import { Ajv2020 } from "ajv/dist/2020.js";
+var FILL_PROMPT_VERSION = 4;
+var FILL_SYSTEM_PROMPT = [
+  "You propose evals for documentation pages. An eval is a plain-language",
+  "assertion about a page that an LLM judge will grade against the page",
+  "content on every future run \u2014 a durable quality contract, not a one-off",
+  "observation.",
+  "",
+  "Rules for every proposal:",
+  "- Assert durable, page-specific properties a maintainer would want",
+  "  guarded (structure, coverage, accuracy anchors) \u2014 never incidental",
+  "  phrasing, styling, or facts likely to change by design.",
+  "- The assertion must be judgeable from the page content alone.",
+  "- Ids are short kebab-case identifiers, unique on the page.",
+  "- Provide examples.pass and examples.fail: one sentence each describing",
+  "  a state of the page that satisfies or violates the assertion.",
+  "- Report an honest confidence between 0 and 1 that the assertion is",
+  "  correct, checkable, and worth guarding. Do not inflate it.",
+  "- Do not duplicate or rephrase the page's existing evals.",
+  "- Do NOT restate the page. An assertion derived from the page's own",
+  "  headings, or lifted from a style guide, passes by construction and keeps",
+  "  passing however the page rots. Anchor on what a reader would notice if it",
+  "  broke.",
+  "- Prefer the cheapest check that can express the assertion. A verifiable",
+  "  fact \u2014 a string that must appear, a file that must exist \u2014 belongs in a",
+  "  deterministic eval; propose a judged assertion only for what genuinely",
+  "  needs reading comprehension.",
+  "- A check lifted from a style rule is secondary: propose it at",
+  "  severity warning, never as a page's only guard.",
+  "- Do not soften an assertion until it always passes. An eval that cannot",
+  "  fail is a vanity metric \u2014 if you cannot state a version that would catch",
+  "  a real regression, propose nothing for it.",
+  "- Propose at most the requested number, fewer when the page offers",
+  "  little worth guarding, and nothing at all when nothing qualifies."
+].join("\n");
+var PROPOSAL_SCHEMA = {
+  type: "object",
+  required: ["evals"],
+  properties: {
+    evals: {
+      type: "array",
+      items: {
+        type: "object",
+        required: ["id", "assertion", "confidence", "examples"],
+        properties: {
+          id: { type: "string", pattern: "^[a-z0-9][a-z0-9-]*$" },
+          assertion: { type: "string", minLength: 1 },
+          type: { enum: ["capability", "regression"] },
+          evidence: { type: "string" },
+          examples: {
+            type: "object",
+            required: ["pass", "fail"],
+            properties: {
+              pass: { type: "string" },
+              fail: { type: "string" }
+            },
+            additionalProperties: false
+          },
+          severity: { enum: SEVERITIES },
+          confidence: { type: "number", minimum: 0, maximum: 1 },
+          rationale: { type: "string" }
+        },
+        additionalProperties: false
+      }
+    }
+  },
+  additionalProperties: false
+};
+var ajv = new Ajv2020({ allErrors: true });
+var validateProposal = ajv.compile(
+  PROPOSAL_SCHEMA
+);
+function isValidProposal(value) {
+  return validateProposal(value);
+}
+function buildFillUser(file, body, existing, maxEvals, part) {
+  const existingLines = existing.length === 0 ? ["(none)"] : existing.map(
+    (e) => e.assertion ? `- ${e.id}: ${e.assertion}` : `- ${e.id}`
+  );
+  return [
+    "# Page path",
+    file,
+    "",
+    "# Existing evals (do not duplicate these)",
+    ...existingLines,
+    "",
+    "# Maximum proposals",
+    String(maxEvals),
+    "",
+    part === void 0 ? "# Page content" : `# Page content (${partLabel(part.index, part.total)})`,
+    "",
+    // No cap. A page longer than one call is split and proposed against in
+    // parts; truncating instead meant proposing evals for a page whose second
+    // half the model never saw, with nothing in the output to say so.
+    body
+  ].join("\n");
+}
+
+// src/docevals/fill/cache.ts
+function fillCacheKey(provider, model, temperature, maxEvals, body, existingNames, chunkChars) {
+  return sha256(
+    [
+      provider,
+      model,
+      `fill-v${FILL_PROMPT_VERSION}`,
+      `t${temperature}`,
+      `n${maxEvals}`,
+      // The chunk budget decides how the page was split, and so which
+      // proposals came back. Without it, two runs at different budgets share
+      // a key and the second silently replays the first — and halve-and-retry
+      // makes that a real collision, not a hypothetical one.
+      `chunk${chunkChars}`,
+      sha256(body),
+      existingNames.join(",")
+    ].join("|")
+  );
+}
+var FillCache = class {
+  constructor(dir, enabled = true) {
+    this.dir = dir;
+    this.enabled = enabled;
+  }
+  dir;
+  enabled;
+  /** Cache-write failures warn once per run, not once per page. */
+  warned = false;
+  get(key) {
+    if (!this.enabled) return void 0;
+    const path = join(this.dir, `${key}.json`);
+    if (!existsSync(path)) return void 0;
+    try {
+      const proposal = JSON.parse(readFileSync(path, "utf8"));
+      if (!isValidProposal(proposal)) return void 0;
+      return proposal;
+    } catch {
+      return void 0;
+    }
+  }
+  set(key, proposal) {
+    if (!this.enabled) return;
+    try {
+      mkdirSync(this.dir, { recursive: true });
+      writeFileSync(
+        join(this.dir, `${key}.json`),
+        JSON.stringify(proposal, null, 2)
+      );
+    } catch (e) {
+      if (!this.warned) {
+        this.warned = true;
+        warn(
+          `could not write the fill cache at ${this.dir} (${errorMessage(e)}). Continuing without caching.`
+        );
+      }
+    }
+  }
+};
+
+// src/docevals/commands/fill.ts
+function toEntry(p) {
+  return {
+    id: p.id,
+    assertion: p.assertion,
+    type: p.type ?? "regression",
+    grader: "ai",
+    evidence: p.evidence,
+    examples: p.examples,
+    severity: p.severity
+  };
+}
+async function runFill(paths, options = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  let config = loadRunConfig(runConfigOptions(paths, options), cwd);
+  const flags = { provider: options.provider, model: options.model, local: options.local };
+  assertProviderSelection(selectProvider(config, flags));
+  const load = async () => resolvePages(
+    await withExternalMetadata(
+      discoverPages(config, documentSet(paths, options, "fill"), cwd),
+      config,
+      cwd
+    ),
+    config
+  );
+  let plans = await load();
+  const writerFor = () => EvalWriter.for(config, cwd, paths);
+  let writer = writerFor();
+  if (options.confirm !== void 0 && options.dryRun !== true) {
+    const flagged = [];
+    for (const plan of plans) {
+      if (plan.skip || plan.problems.some((p) => p.level === "error")) continue;
+      const home = await writer.homeFor(
+        plan.page.file,
+        plan.page.frontmatter.data,
+        EVALS_KEY
+      );
+      if (home.kind === "page" && home.prefersExternal) {
+        flagged.push({ label: plan.page.file, key: EVALS_KEY, home: home.proposed });
+      }
+    }
+    const applied = await writer.offer(flagged, {
+      confirm: options.confirm,
+      ...options.onNotice === void 0 ? {} : { onNotice: options.onNotice },
+      ...options.onRelocated === void 0 ? {} : { onRelocated: options.onRelocated }
+    });
+    if (applied) {
+      config = loadRunConfig(runConfigOptions(paths, options), cwd);
+      plans = await load();
+      writer = writerFor();
+    }
+  }
+  const threshold = options.confidence ?? config.fill.confidenceThreshold;
+  const maxTurns = options.maxTurns ?? config.fill.maxTurns;
+  const temperature = config.fill.temperature;
+  const maxEvals = config.fill.maxEvalsPerPage;
+  const chunkChars = options.chunkChars ?? config.fill.chunkChars;
+  const cache = new FillCache(
+    resolve(cwd, config.fill.cacheDir),
+    !options.noCache
+  );
+  const injected = options.providerInstance;
+  let resolving;
+  const resolveOnce = () => resolving ??= resolveProviderIdentity(config, flags);
+  const getIdentity = async () => injected ? { provider: injected.provider(), model: injected.modelName() } : resolveOnce();
+  let provider = injected;
+  const getProvider = async () => provider ??= constructProvider(config, await resolveOnce());
+  let turns = 0;
+  const results = [];
+  for (const plan of plans) {
+    results.push(await fillOne(plan));
+  }
+  for (const line of writer.warnings(options.dryRun === true)) warn(line);
+  return {
+    results,
+    threshold,
+    turns,
+    exitCode: results.some((r) => r.status === "error") ? 1 : 0
+  };
+  async function fillOne(plan) {
+    const base = {
+      file: plan.page.file,
+      status: "nothing-proposed",
+      written: [],
+      belowThreshold: [],
+      capped: [],
+      duplicates: [],
+      cached: false
+    };
+    if (plan.skip) return { ...base, status: "skipped" };
+    const problem = plan.problems.find((p) => p.level === "error");
+    if (problem) return { ...base, status: "error", error: problem.message };
+    const format = leadingFrontmatterFormat(plan.page.content);
+    if (format === "toml" || format === "json") {
+      return {
+        ...base,
+        status: "error",
+        error: `only YAML frontmatter can be filled (found ${format} frontmatter)`
+      };
+    }
+    const existing = plan.evals.map((e) => ({
+      id: e.name,
+      assertion: e.assertion
+    }));
+    const existingNames = existing.map((e) => e.id).sort();
+    const identity = await getIdentity();
+    const keyFor = (budget) => fillCacheKey(
+      identity.provider,
+      identity.model,
+      temperature,
+      maxEvals,
+      plan.page.body,
+      existingNames,
+      budget
+    );
+    const key = keyFor(chunkChars);
+    let raw = cache.get(key);
+    const cached = raw !== void 0;
+    if (!raw) {
+      if (maxTurns !== null && turns >= maxTurns) {
+        return { ...base, status: "skipped-budget" };
+      }
+      let budget = chunkChars;
+      let partsRead = 0;
+      let merged2;
+      let failure;
+      for (let attempt = 0; attempt < 2 && merged2 === void 0; attempt++) {
+        const chunks = splitBody(plan.page.body, budget);
+        const collected = /* @__PURE__ */ new Map();
+        let overflowed = false;
+        let outOfTurns = false;
+        partsRead = 0;
+        for (const [i, chunk] of chunks.entries()) {
+          if (maxTurns !== null && turns >= maxTurns) {
+            outOfTurns = true;
+            break;
+          }
+          turns += 1;
+          partsRead += 1;
+          let response;
+          try {
+            response = await (await getProvider()).completeJSON({
+              system: FILL_SYSTEM_PROMPT,
+              user: buildFillUser(
+                plan.page.file,
+                chunk,
+                existing,
+                maxEvals,
+                chunks.length > 1 ? { index: i, total: chunks.length } : void 0
+              ),
+              schema: PROPOSAL_SCHEMA,
+              temperature
+            });
+          } catch (e) {
+            const message = errorMessage(e);
+            failure = message;
+            if (looksLikeOverflow(message)) overflowed = true;
+            break;
+          }
+          if (!isValidProposal(response.json)) {
+            failure = "provider returned a proposal that does not match the schema";
+            break;
+          }
+          for (const proposal of response.json.evals) {
+            const held = collected.get(proposal.id);
+            if (held === void 0 || proposal.confidence > held.confidence) {
+              collected.set(proposal.id, proposal);
+            }
+          }
+          if (i === chunks.length - 1) merged2 = collected;
+        }
+        if (outOfTurns) {
+          return {
+            ...base,
+            status: "skipped-budget",
+            ...chunks.length > 1 ? {
+              error: `--max-turns reached after ${String(partsRead)} of ${String(chunks.length)} part(s) of this page`
+            } : {}
+          };
+        }
+        if (merged2 === void 0 && overflowed && attempt === 0) {
+          budget = Math.max(1, Math.floor(budget / 2));
+          continue;
+        }
+        if (merged2 === void 0) break;
+      }
+      if (merged2 === void 0) {
+        return {
+          ...base,
+          status: "error",
+          error: failure ?? "the provider returned no proposal"
+        };
+      }
+      raw = { evals: [...merged2.values()] };
+      cache.set(keyFor(budget), raw);
+    }
+    const seen = new Set(existingNames);
+    const duplicates = [];
+    const fresh = [];
+    for (const p of raw.evals) {
+      if (seen.has(p.id)) {
+        duplicates.push(p.id);
+        continue;
+      }
+      seen.add(p.id);
+      fresh.push(p);
+    }
+    const capped = fresh.slice(maxEvals);
+    const belowThreshold = [];
+    const written = [];
+    for (const p of fresh.slice(0, maxEvals)) {
+      if (p.confidence >= threshold) written.push(p);
+      else belowThreshold.push(p);
+    }
+    const result = {
+      ...base,
+      written,
+      belowThreshold,
+      capped,
+      duplicates,
+      cached
+    };
+    if (written.length === 0) return result;
+    const label = plan.page.file;
+    const data = plan.page.frontmatter.data;
+    let evalsHome;
+    let metaHome;
+    try {
+      evalsHome = await writer.homeFor(label, data, EVALS_KEY);
+      metaHome = await writer.homeFor(label, data, META_PROVENANCE_KEY);
+    } catch (e) {
+      return { ...result, status: "error", error: errorMessage(e), written: [] };
+    }
+    if (evalsHome.kind === "url") throw writer.urlRefusal(evalsHome, EVALS_KEY);
+    if (evalsHome.kind === "no-entry") {
+      return { ...result, status: "error", error: evalsHome.message, written: [] };
+    }
+    const merged = metaHome.kind === "url" || metaHome.kind === "no-entry" ? void 0 : mergeMetaProvenance(
+      data[META_PROVENANCE_KEY],
+      identity.model,
+      "evals",
+      written.map((p) => ({ name: p.id, confidence: p.confidence }))
+    );
+    const metaManifest = metaHome.kind === "manifest" ? metaHome : void 0;
+    let metaProvenance;
+    if (metaHome.kind === "url") {
+      metaProvenance = { written: false, skipReason: "manifest-owned", manifest: metaHome.file };
+    } else if (metaHome.kind === "no-entry") {
+      metaProvenance = { written: false, skipReason: "unwritable" };
+    } else if (merged === void 0) {
+      metaProvenance = { written: false, skipReason: "schema-mismatch" };
+    } else if (metaManifest !== void 0) {
+      metaProvenance = {
+        written: true,
+        entry: { ...merged.entry, evals: merged.names },
+        destination: metaManifest.file
+      };
+    } else {
+      metaProvenance = { written: true, entry: { ...merged.entry, evals: merged.names } };
+    }
+    const manifest = evalsHome.kind === "manifest" ? evalsHome : void 0;
+    const pageEntries = manifest === void 0 ? written.map(toEntry) : [];
+    const pageMeta = metaProvenance.written && metaManifest === void 0 ? merged?.list : void 0;
+    try {
+      if (pageEntries.length > 0 || pageMeta !== void 0) {
+        const updated = appendPageEvals(
+          plan.page.content,
+          label,
+          pageEntries,
+          pageMeta
+        );
+        if (!options.dryRun && updated !== plan.page.content) {
+          writeFileSync2(plan.page.absPath, updated);
+        }
+      }
+      if (!options.dryRun && manifest !== void 0) {
+        const held = manifestEvalList(
+          await writer.readManifest(manifest, EVALS_KEY),
+          manifest.file,
+          manifest.entry
+        );
+        await writer.writeManifest(manifest, EVALS_KEY, [
+          ...held,
+          ...written.map((p) => entryObject(toEntry(p)))
+        ]);
+      }
+      if (!options.dryRun && metaManifest !== void 0 && merged !== void 0) {
+        await writer.writeManifest(metaManifest, META_PROVENANCE_KEY, merged.list);
+      }
+    } catch (e) {
+      return {
+        ...result,
+        status: "error",
+        error: errorMessage(e),
+        written: []
+      };
+    }
+    if (evalsHome.kind === "page" && evalsHome.prefersExternal) {
+      writer.stayedOnPage(label, EVALS_KEY, evalsHome.proposed);
+    }
+    return {
+      ...result,
+      status: options.dryRun ? "proposed" : "filled",
+      wroteTo: manifest === void 0 ? "page" : manifest.file,
+      metaProvenance
+    };
+  }
+}
+var STATUS_LABELS = {
+  filled: "filled",
+  proposed: "proposed",
+  "nothing-proposed": "no-op",
+  skipped: "skipped",
+  "skipped-budget": "skipped",
+  error: "error"
+};
+function renderFill(report, format, opts = {}) {
+  parseFormat(format, SUMMARY_FORMATS, "format");
+  if (format === "json") return JSON.stringify(report, null, 2);
+  const pc = palette(opts.color === true);
+  const lines = [];
+  const names = (evals) => evals.map((p) => `${p.id} ${p.confidence.toFixed(2)}`).join(", ");
+  for (const r of report.results) {
+    const label = STATUS_LABELS[r.status].padEnd(8);
+    const cachedTag = r.cached ? " [cached]" : "";
+    switch (r.status) {
+      case "filled":
+        lines.push(
+          `${pc.green(label)} ${r.file}  +${r.written.length} evals (${names(r.written)})${cachedTag}`
+        );
+        break;
+      case "proposed":
+        lines.push(
+          `${pc.cyan(label)} ${r.file}  +${r.written.length} evals (${names(r.written)})${cachedTag} \u2014 dry run, not written`
+        );
+        break;
+      case "nothing-proposed":
+        lines.push(`${pc.dim(label)} ${r.file}  (nothing new proposed)${cachedTag}`);
+        break;
+      case "skipped":
+        lines.push(`${pc.dim(label)} ${r.file}  (evals.skip)`);
+        break;
+      case "skipped-budget":
+        lines.push(`${pc.yellow(label)} ${r.file}  (turn budget exhausted)`);
+        break;
+      case "error":
+        lines.push(`${pc.red(label)} ${r.file}: ${r.error ?? "unknown error"}`);
+        break;
+    }
+    if (r.wroteTo !== void 0 && r.wroteTo !== "page") {
+      lines.push(`    ${pc.cyan("evals")} \u2192 ${r.wroteTo}`);
+    }
+    const record = r.metaProvenance;
+    if (record?.written === true) {
+      const destination = record.destination === void 0 ? "" : ` \u2192 ${record.destination}`;
+      lines.push(
+        `    ${pc.cyan("meta-provenance")}  ${record.entry["generated-by"]}: ${record.entry.evals.join(", ")}${destination}`
+      );
+    } else if (record?.skipReason === "schema-mismatch") {
+      lines.push(
+        pc.dim("    meta-provenance not written: this page's schemas do not allow it")
+      );
+    } else if (record?.skipReason === "manifest-owned") {
+      lines.push(
+        pc.dim(
+          `    meta-provenance not written: owned by manifest ${record.manifest}, which is fetched and cannot be written`
+        )
+      );
+    } else if (record?.skipReason === "unwritable") {
+      lines.push(
+        pc.dim("    meta-provenance not written: its manifest has no entry for this page")
+      );
+    }
+    if (r.belowThreshold.length > 0) {
+      lines.push(
+        pc.dim(
+          `         below ${report.threshold}: ${names(r.belowThreshold)}`
+        )
+      );
+    }
+    if (r.capped.length > 0) {
+      lines.push(
+        pc.dim(`         over per-page cap: ${names(r.capped)}`)
+      );
+    }
+    if (r.duplicates.length > 0) {
+      lines.push(pc.dim(`         duplicates: ${r.duplicates.join(", ")}`));
+    }
+  }
+  lines.push("");
+  lines.push(
+    `Threshold: ${report.threshold} \xB7 inference calls: ${report.turns}`
+  );
+  return lines.join("\n");
+}
+
+// src/docevals/commands/promote.ts
+import { mkdirSync as mkdirSync2, readFileSync as readFileSync2, writeFileSync as writeFileSync3 } from "node:fs";
+import { dirname, relative } from "node:path";
+var PROMOTE_SCHEMA = {
+  type: "object",
+  required: ["promotable", "rationale"],
+  properties: {
+    promotable: {
+      type: "boolean",
+      description: "True only when the assertion can be fully checked by a deterministic script with no semantic judgment."
+    },
+    rationale: { type: "string" },
+    code: {
+      type: "string",
+      description: "The check script source, required when promotable is true."
+    }
+  },
+  additionalProperties: false
+};
+var PROMOTE_SYSTEM = [
+  "You review documentation eval assertions and decide whether each can be",
+  "verified by a deterministic script instead of LLM judgment. Promote only",
+  "assertions that are fully checkable with string/regex/structural logic \u2014",
+  "anything requiring semantic interpretation stays with the LLM judge.",
+  "",
+  SCRIPTGEN_SYSTEM_PROMPT,
+  "",
+  'Respond with JSON: { "promotable": bool, "rationale": "...", "code": "..." }',
+  "(omit code when not promotable)."
+].join("\n");
+async function assess(provider, target) {
+  try {
+    const response = await provider.completeJSON({
+      system: PROMOTE_SYSTEM,
+      user: buildScriptgenUser(
+        target.eval.assertion ?? "",
+        target.eval.name,
+        target.plan.page.file,
+        target.plan.page.body
+      ),
+      schema: PROMOTE_SCHEMA,
+      temperature: 0
+    });
+    const json = response.json;
+    return {
+      promotable: json.promotable === true,
+      rationale: typeof json.rationale === "string" ? json.rationale : "",
+      code: typeof json.code === "string" ? json.code : void 0
+    };
+  } catch (e) {
+    return {
+      promotable: false,
+      rationale: `assessment failed: ${errorMessage(e)}`
+    };
+  }
+}
+async function runPromote(paths, options = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  const config = loadRunConfig(runConfigOptions(paths, options), cwd);
+  const flags = { provider: options.provider, model: options.model, local: options.local };
+  assertProviderSelection(selectProvider(config, flags));
+  const pages = await withExternalMetadata(
+    discoverPages(config, documentSet(paths, options, "read"), cwd),
+    config,
+    cwd
+  );
+  const plans = resolvePages(pages, config);
+  assertRegisteredGraders(plans, config);
+  let provider = options.providerInstance;
+  const getProvider = async () => provider ??= await makeProvider(config, flags);
+  const proposals = [];
+  const seenConfigEvals = /* @__PURE__ */ new Set();
+  let writer;
+  const getWriter = () => writer ??= EvalWriter.for(config, cwd, paths);
+  for (const plan of plans) {
+    if (plan.skip || plan.problems.some((p) => p.level === "error")) continue;
+    for (const ev of plan.evals) {
+      if (ev.skip || ev.grader !== "ai" || !ev.assertion) continue;
+      if (ev.source === "config") {
+        if (seenConfigEvals.has(ev.name)) continue;
+        seenConfigEvals.add(ev.name);
+      }
+      const target = { plan, eval: ev };
+      const assessment = await assess(await getProvider(), target);
+      const proposal = {
+        file: plan.page.file,
+        evalName: ev.name,
+        source: ev.source,
+        promotable: assessment.promotable,
+        rationale: assessment.rationale,
+        applied: false
+      };
+      if (assessment.promotable && assessment.code && options.write) {
+        const location = scriptLocationFor(target, config, cwd);
+        mkdirSync2(dirname(location.scriptAbsPath), { recursive: true });
+        writeFileSync3(location.scriptAbsPath, assessment.code);
+        proposal.scriptPath = relative(cwd, location.scriptAbsPath).replace(
+          /\\/g,
+          "/"
+        );
+        const updates = {
+          grader: "command",
+          command: location.command,
+          // The kebab key `applyUpdates` actually reads. This was the
+          // pre-1.0 `generated: { assertionHash }` wrapper, which the update
+          // type does not consume, so promoted evals were written with no
+          // hash and never went stale when their assertion changed
+          // (renamed by ADR 01009).
+          "generated-assertion-hash": sha256(ev.assertion)
+        };
+        const home = ev.source === "page" ? await getWriter().homeFor(
+          plan.page.file,
+          plan.page.frontmatter.data,
+          EVALS_KEY
+        ) : void 0;
+        if (home?.kind === "url") {
+          throw getWriter().urlRefusal(home, EVALS_KEY);
+        }
+        if (home?.kind === "no-entry") {
+          proposal.error = home.message;
+        } else if (home?.kind === "manifest") {
+          const at = getWriter();
+          const list = manifestEvalList(
+            await at.readManifest(home, EVALS_KEY),
+            home.file,
+            home.entry
+          );
+          const next = updateManifestEval(list, ev.name, updates);
+          if (next === void 0) {
+            proposal.error = `${home.file}: eval "${ev.name}" not found in the entry for ${home.entry}`;
+          } else {
+            await at.writeManifest(home, EVALS_KEY, next);
+            proposal.applied = true;
+          }
+        } else if (ev.source === "page" && hasEditableEval(plan.page.content, ev.name)) {
+          const updated = updatePageEval(
+            readFileSync2(plan.page.absPath, "utf8"),
+            plan.page.file,
+            ev.name,
+            updates
+          );
+          writeFileSync3(plan.page.absPath, updated);
+          proposal.applied = true;
+        } else if (ev.source === "config") {
+          const updated = updateConfigEval(
+            readFileSync2(config.configPath, "utf8"),
+            config.configPath,
+            ev.name,
+            updates
+          );
+          writeFileSync3(config.configPath, updated);
+          proposal.applied = true;
+        }
+      }
+      proposals.push(proposal);
+    }
+  }
+  return proposals;
+}
+
+// src/docevals/commands/review.ts
+import { existsSync as existsSync2 } from "node:fs";
+import { resolve as resolve2, relative as relative2 } from "node:path";
+function listReviews(cwd = process.cwd()) {
+  return loadReviews(cwd);
+}
+function renderReviews(reviews, opts = {}) {
+  if (reviews.length === 0) return "No recorded reviews.";
+  const pc = palette(opts.color === true);
+  return reviews.map((r) => {
+    const verdict = r.verdict === "pass" ? pc.green(r.verdict) : pc.red(r.verdict);
+    const meta = [r.reviewer, r.date].filter(Boolean).join(" ");
+    return `${verdict} ${r.file} ${pc.bold(r.evalName)}${meta ? pc.dim(` (${meta})`) : ""}${r.note ? pc.dim(` \u2014 ${r.note}`) : ""}`;
+  }).join("\n");
+}
+function runReview(file, evalName, verdict, options = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  if (verdict !== "pass" && verdict !== "fail") {
+    throw new DocevalsError(`Verdict must be "pass" or "fail", got "${verdict}"`);
+  }
+  const absPath = resolve2(cwd, file);
+  if (!existsSync2(absPath)) {
+    throw new DocevalsError(`Page not found: ${absPath}`);
+  }
+  const page = readPage(absPath, cwd);
+  const entry = {
+    file: relative2(cwd, absPath).replace(/\\/g, "/"),
+    evalName,
+    contentHash: sha256(page.body),
+    verdict,
+    reviewer: options.reviewer,
+    date: (/* @__PURE__ */ new Date()).toISOString().slice(0, 10),
+    note: options.note
+  };
+  recordReview(cwd, entry);
+  return entry;
+}
+
+// src/docevals/commands/calibrate.ts
+import { existsSync as existsSync3, mkdirSync as mkdirSync3, readFileSync as readFileSync3, writeFileSync as writeFileSync4 } from "node:fs";
+import { join as join2, resolve as resolve3 } from "node:path";
+import fg from "fast-glob";
+import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
+var AGREEMENT_THRESHOLD = 0.7;
+var GOLDEN_DIR = join2(".manni/docevals", "golden");
+var SEEDED_GOLDEN_FILE = "from-reviews.yaml";
+function normalizeGoldenCase(raw) {
+  return {
+    file: raw.file,
+    eval: raw.eval,
+    expected: raw.expected,
+    ...typeof raw.rationale === "string" ? { rationale: raw.rationale } : {},
+    reviewed: raw.reviewed === true,
+    ...typeof raw["content-hash"] === "string" ? { contentHash: raw["content-hash"] } : {},
+    ...raw.source === "review" || raw.source === "manual" ? { source: raw.source } : {},
+    ...typeof raw["reviewed-by"] === "string" ? { reviewedBy: raw["reviewed-by"] } : {}
+  };
+}
+function serializeGoldenCase(c) {
+  return {
+    file: c.file,
+    eval: c.eval,
+    expected: c.expected,
+    ...c.rationale === void 0 ? {} : { rationale: c.rationale },
+    reviewed: c.reviewed,
+    ...c.contentHash === void 0 ? {} : { "content-hash": c.contentHash },
+    ...c.source === void 0 ? {} : { source: c.source },
+    ...c.reviewedBy === void 0 ? {} : { "reviewed-by": c.reviewedBy }
+  };
+}
+function loadGoldenCases(dir) {
+  if (!existsSync3(dir)) {
+    throw new DocevalsError(
+      `Golden set directory not found: ${dir} \u2014 seed it from recorded reviews with \`manni docevals calibrate --seed\`, or hand-author 20-50 verified cases`
+    );
+  }
+  const files = fg.sync("*.{yaml,yml}", { cwd: dir, absolute: true });
+  const cases = [];
+  for (const file of files) {
+    const raw = parseYaml(readFileSync3(file, "utf8"));
+    if (!Array.isArray(raw)) continue;
+    for (const item of raw) {
+      const c = item;
+      if (typeof c.file === "string" && typeof c.eval === "string" && (c.expected === "pass" || c.expected === "fail")) {
+        cases.push(normalizeGoldenCase(c));
+      } else {
+        throw new DocevalsError(
+          `Invalid golden case in ${file}: needs file, eval, expected: pass|fail`
+        );
+      }
+    }
+  }
+  if (cases.length === 0) {
+    throw new DocevalsError(`No golden cases found in ${dir}`);
+  }
+  return cases;
+}
+function seedGoldenCases(options = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  const dir = resolve3(cwd, options.golden ?? GOLDEN_DIR);
+  const path = join2(dir, SEEDED_GOLDEN_FILE);
+  const priorRaw = existsSync3(path) ? parseYaml(readFileSync3(path, "utf8")) : null;
+  const existing = Array.isArray(priorRaw) ? priorRaw.filter((i) => typeof i === "object" && i !== null).map(normalizeGoldenCase) : [];
+  const byKey = new Map(existing.map((c) => [`${c.file}\0${c.eval}`, c]));
+  let added = 0;
+  let updated = 0;
+  for (const review of loadReviews(cwd)) {
+    const key = `${review.file}\0${review.evalName}`;
+    const prior = byKey.get(key);
+    const confirmedStillApplies = prior?.reviewed === true && prior.expected === review.verdict;
+    const next = {
+      file: review.file,
+      eval: review.evalName,
+      expected: review.verdict,
+      // A review's note wins when it has one; otherwise keep whatever the
+      // human wrote into the file. Rebuilding the entry from the review alone
+      // erased the reasoning behind a case the same human had just marked
+      // `reviewed: true` -- and it is the field renderCalibration prints on
+      // every disagreement.
+      ...review.note !== void 0 ? { rationale: review.note } : prior?.rationale !== void 0 ? { rationale: prior.rationale } : {},
+      reviewed: confirmedStillApplies,
+      contentHash: review.contentHash,
+      source: "review",
+      ...confirmedStillApplies && prior.reviewedBy !== void 0 ? { reviewedBy: prior.reviewedBy } : {}
+    };
+    byKey.set(key, next);
+    if (!prior) added += 1;
+    else if (JSON.stringify(serializeGoldenCase(prior)) !== JSON.stringify(serializeGoldenCase(next))) {
+      updated += 1;
+    }
+  }
+  const cases = [...byKey.values()];
+  if (cases.length > 0) {
+    mkdirSync3(dir, { recursive: true });
+    writeFileSync4(path, stringifyYaml(cases.map(serializeGoldenCase)));
+  }
+  return {
+    path,
+    added,
+    updated,
+    total: cases.length,
+    unreviewed: cases.filter((c) => !c.reviewed).length
+  };
+}
+async function runCalibrate(options = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  const config = loadConfig(options.config, cwd);
+  const flags = { provider: options.provider, model: options.model, local: options.local };
+  assertProviderSelection(selectProvider(config, flags));
+  const goldenDir = resolve3(cwd, options.golden ?? GOLDEN_DIR);
+  const cases = loadGoldenCases(goldenDir);
+  const results = [];
+  const targets = [];
+  const targetIndex = [];
+  const external = await loadExternalReader(
+    config,
+    cwd,
+    cases.map((c) => resolve3(cwd, c.file)).filter((p) => existsSync3(p))
+  );
+  for (const goldenCase of cases) {
+    const absPath = resolve3(cwd, goldenCase.file);
+    if (!existsSync3(absPath)) {
+      results.push({ ...goldenCase, error: "page not found" });
+      continue;
+    }
+    const read = readPage(absPath, cwd);
+    const page = external === null ? read : await external.forPage(read);
+    const plan = resolvePage(page, config);
+    const ev = plan.evals.find(
+      (e) => e.name === goldenCase.eval && e.grader === "ai"
+    );
+    if (!ev) {
+      results.push({
+        ...goldenCase,
+        error: `ai-graded eval "${goldenCase.eval}" not resolvable on this page`
+      });
+      continue;
+    }
+    const stale = goldenCase.contentHash !== void 0 && goldenCase.contentHash !== sha256(plan.page.body);
+    targetIndex.push(results.length);
+    results.push({ ...goldenCase, ...stale ? { stale } : {} });
+    targets.push({ plan, eval: ev });
+  }
+  if (targets.length > 0) {
+    const judge = options.judge ?? makeJudge({
+      provider: await makeProvider(config, flags),
+      root: cwd
+    });
+    const judged = await judge(targets, config, {
+      // The flags reach the judge too, so a golden case whose eval names its
+      // own provider or model is overridden by them as it is under `run`.
+      ...flags,
+      runs: options.runs,
+      noCache: options.noCache,
+      maxTurns: options.maxTurns
+    });
+    for (const [i, at] of targetIndex.entries()) {
+      const slot = results[at];
+      if (slot === void 0) {
+        throw new DocevalsError(
+          `calibrate: no golden case at index ${at} for judged target ${i}`
+        );
+      }
+      const consensus = judged[i]?.consensus;
+      if (!consensus) {
+        const reason = judged[i]?.skipReason;
+        const fromBudget = isTurnBudgetSkip(reason);
+        results[at] = {
+          ...slot,
+          ...fromBudget ? { budgetSkipped: true } : {},
+          error: fromBudget ? `not judged: ${reason}` : "judge returned no consensus"
+        };
+        continue;
+      }
+      const verdict = consensus.verdict === "pass" ? "pass" : "fail";
+      results[at] = {
+        ...slot,
+        judged: verdict,
+        agrees: verdict === slot.expected
+      };
+    }
+  }
+  const judgedCases = results.filter((r) => r.judged != null);
+  const agreements = judgedCases.filter((r) => r.agrees).length;
+  const expectedPass = judgedCases.filter((r) => r.expected === "pass");
+  const falsePositives = expectedPass.filter((r) => r.judged === "fail").length;
+  const expectedFailCases = judgedCases.filter((r) => r.expected === "fail");
+  const falseNegatives = expectedFailCases.filter(
+    (r) => r.judged === "pass"
+  ).length;
+  const agreementRate = judgedCases.length > 0 ? agreements / judgedCases.length : 0;
+  const falsePositiveRate = expectedPass.length > 0 ? falsePositives / expectedPass.length : 0;
+  const budgetSkipped = results.filter((r) => r.budgetSkipped === true).length;
+  const unjudged = results.length - judgedCases.length;
+  return {
+    cases: results,
+    budgetSkipped,
+    unjudged,
+    unreviewed: judgedCases.filter((r) => !r.reviewed).length,
+    stale: judgedCases.filter((r) => r.stale === true).length,
+    total: results.length,
+    agreements,
+    agreementRate,
+    falsePositives,
+    falsePositiveRate,
+    falseNegatives,
+    meetsThreshold: agreementRate >= AGREEMENT_THRESHOLD,
+    expectedPass: expectedPass.length,
+    expectedFail: expectedFailCases.length,
+    balanced: expectedPass.length > 0 && expectedFailCases.length > 0,
+    fpAlert: falsePositiveRate > config.judge.falsePositiveAlert
+  };
+}
+function renderCalibration(report, opts = {}) {
+  const pc = palette(opts.color === true);
+  const lines = [];
+  for (const c of report.cases) {
+    if (c.error) {
+      lines.push(`${pc.red("error")} ${c.file} ${c.eval}: ${c.error}`);
+      continue;
+    }
+    const tag = c.agrees ? pc.green("agree") : pc.red("DISAGREE");
+    const flags = [
+      c.reviewed ? "" : pc.yellow(" [unreviewed]"),
+      c.stale ? pc.yellow(" [stale]") : ""
+    ].join("");
+    lines.push(
+      `${tag} ${c.file} ${pc.bold(c.eval)}${flags}: judge=${c.judged ?? "(no verdict)"} human=${c.expected}` + (!c.agrees && c.rationale ? pc.dim(` \u2014 human: ${c.rationale}`) : "")
+    );
+  }
+  lines.push("");
+  const judged = report.cases.filter((c) => c.judged != null).length;
+  lines.push(
+    `Agreement: ${report.agreements}/${judged} judged (${(report.agreementRate * 100).toFixed(0)}%) \u2014 threshold ${(AGREEMENT_THRESHOLD * 100).toFixed(0)}%` + (judged === report.total ? "" : ` \u2014 ${report.total - judged} of ${report.total} case(s) not judged`)
+  );
+  lines.push(
+    `False positives: ${report.falsePositives} (${(report.falsePositiveRate * 100).toFixed(0)}% of human-passes), false negatives: ${report.falseNegatives}`
+  );
+  if (report.unjudged > 0) {
+    lines.push(
+      pc.red(
+        `
+${report.unjudged} of ${report.total} case(s) never reached a verdict, so the rate above is over a sample. A calibration measured on part of the set does not certify the judge.`
+      )
+    );
+  }
+  if (judged > 0 && !report.balanced) {
+    const missing = report.expectedFail === 0 ? "fail" : "pass";
+    const always = missing === "fail" ? "pass" : "fail";
+    lines.push(
+      pc.red(
+        `
+No \`expected: ${missing}\` cases among the ${judged} judged. A judge that answered "${always}" every time would score 100% here, so this run cannot certify anything. Add at least one case the judge should ${missing === "fail" ? "reject" : "accept"}.`
+      )
+    );
+  }
+  if (judged > 0 && !report.meetsThreshold) {
+    lines.push(
+      pc.red(
+        "\nAgreement is below threshold. Refine the eval criteria first \u2014 make assertions more specific \u2014 before changing the grading mechanism."
+      )
+    );
+  }
+  if (report.budgetSkipped > 0) {
+    lines.push(
+      pc.red(
+        `
+${report.budgetSkipped} case(s) were never judged: the turn budget ran out. A rate measured over the rest is not a calibration \u2014 raise --max-turns and re-run.`
+      )
+    );
+  }
+  if (report.fpAlert) {
+    lines.push(
+      pc.yellow(
+        "\nFalse-positive rate exceeds judge.falsePositiveAlert \u2014 the judge is flagging content humans accept. Consider tightening assertions or examples."
+      )
+    );
+  }
+  if (report.unreviewed > 0) {
+    lines.push(
+      pc.yellow(
+        `
+${report.unreviewed} of ${report.total} case(s) are unreviewed \u2014 they count toward the rate above, but no human has confirmed they belong in the golden set. Read them and set \`reviewed: true\`.`
+      )
+    );
+  }
+  if (report.stale > 0) {
+    lines.push(
+      pc.yellow(
+        `
+${report.stale} case(s) are stale: the page changed since the verdict was recorded, so the case describes a document that no longer exists. Re-verify and update \`content-hash\`.`
+      )
+    );
+  }
+  return lines.join("\n");
+}
+
+// src/docevals/commands/init.ts
+import { existsSync as existsSync4, writeFileSync as writeFileSync5 } from "node:fs";
+import { resolve as resolve4 } from "node:path";
+var STARTER_CONFIG = `# manni.config.yaml \u2014 shared configuration for the manni family of tools.
+# Each tool reads its own top-level key; manni docevals reads "docevals:".
+# Docs: https://hawkeyexl.github.io/manni/docevals/
+
+# The documents every manni tool reads, declared once. A bare
+# \`manni docevals run\` evaluates every collection listed here; pass
+# --collection <name> to pick one, or name paths on the command line instead.
+# node_modules and .git are never read.
+collections:
+  - name: site
+    paths:
+      - "docs/**/*.{md,mdx}"
+
+# The inference provider, declared once for every manni tool that sends
+# content to a model: manni docevals and manni meta fill. auto detects what
+# this machine can use: an Anthropic key, then an OpenAI key, then the Claude
+# CLI, then a local model. Name one to pin it. A tool's own provider key, or
+# --provider, still wins for that tool.
+providers:
+  provider: auto # auto | anthropic | openai | claude-cli | llama-cpp
+  # model: <id>  # needs a named provider; unset takes that provider's default
+  # anthropic: { apiKeyEnv: ANTHROPIC_API_KEY }
+  # openai: { baseUrl: http://localhost:11434/v1, apiKeyEnv: OPENAI_API_KEY }
+  # claude-cli: { command: claude } # uses local CLI auth, no API key
+  # llama-cpp: { modelsDir: .manni/models, thoughtTokens: 0 } # modelsDir is relative to this file
+
+docevals:
+  defaults:
+    # Suite applied to pages without an eval-suite frontmatter key. Naming the
+    # suite defined at the bottom of this file is what makes a fresh corpus
+    # check anything at all: with "null", a page carrying no eval frontmatter
+    # resolves zero evals, and a run over zero evals is a usage error rather
+    # than a green build (ADR 01041).
+    suite: default
+    failFast: false
+    concurrency: 4
+
+  judge:
+    ensembleRuns: 3 # 3 isolated runs per eval; agreement is signal
+    # NOTE: defaults.suite attaches an ai eval to every discovered page, so a
+    # keyed run on a large corpus issues ensembleRuns x pages requests. Set
+    # maxTurns once you know what a full pass costs you \u2014 deliberately, and
+    # high enough to cover the corpus. A budget set *below* what a full pass
+    # needs stops early, reports the remaining pages as skipped, and still exits
+    # 0 (ADR 01019): partial coverage that reads as success. Start with
+    # --deterministic-only, which needs no provider at all.
+    temperature: 0
+    zones:
+      autoPass: 0.8 # unanimous pass + mean confidence >= 0.8
+      autoFail: 0.8
+    falsePositiveAlert: 0.15
+    cacheDir: .manni/docevals/cache
+
+  execution:
+    # Every grant holds unless this list narrows it. frontmatter-commands lets
+    # command evals declared in page frontmatter run. Write [] if you do not
+    # trust whoever can edit this corpus's pages.
+    allow: [frontmatter-commands]
+
+  scripts:
+    dir: "{docDir}/manni-docevals" # generated check scripts live beside the docs
+    configDir: manni-docevals-scripts
+
+  evals:
+    no-future-promises:
+      type: regression
+      assertion: The page makes no claims about unreleased or future functionality.
+      grader: ai
+      evidence: All prose sections
+      examples:
+        pass: Describes only shipped behavior.
+        fail: Says "coming soon" or references an unreleased version.
+    no-todo-markers:
+      assertion: The page carries no TODO, TBD or FIXME markers.
+      grader: tool:regex
+      options:
+        pattern: "\\\\b(TODO|TBD|FIXME)\\\\b"
+        match: not-contains
+      severity: error
+
+  suites:
+    default:
+      target-pass-rate: 1.0 # regression suites target ~100%
+      evals: [no-future-promises, no-todo-markers]
+`;
+function runInit(cwd = process.cwd()) {
+  const path = resolve4(cwd, DEFAULT_CONFIG_FILENAME);
+  if (existsSync4(path)) {
+    throw new DocevalsError(`${DEFAULT_CONFIG_FILENAME} already exists`);
+  }
+  writeFileSync5(path, STARTER_CONFIG);
+  return path;
+}
+
+export {
+  runGenerate,
+  FILL_PROMPT_VERSION,
+  FILL_SYSTEM_PROMPT,
+  PROPOSAL_SCHEMA,
+  isValidProposal,
+  buildFillUser,
+  fillCacheKey,
+  FillCache,
+  runFill,
+  renderFill,
+  runPromote,
+  listReviews,
+  renderReviews,
+  runReview,
+  GOLDEN_DIR,
+  SEEDED_GOLDEN_FILE,
+  loadGoldenCases,
+  seedGoldenCases,
+  runCalibrate,
+  renderCalibration,
+  runInit
+};
+//# sourceMappingURL=chunk-77HX34KD.js.map