@hawkeyexl/manni 4.3.0 → 4.4.0

package/dist/chunk-PHF3D7ER.js

@@ -0,0 +1,1042 @@
+#!/usr/bin/env node
+import {
+  EVIDENCE_SCHEMA,
+  EVIDENCE_SYSTEM_PROMPT,
+  JUDGE_SYSTEM_PROMPT,
+  VerdictCache,
+  buildEvidenceUser,
+  buildUserContent,
+  cacheKey,
+  frontmatterSchema,
+  judgeCacheBody,
+  leadingFrontmatterFormat,
+  loadRunConfig,
+  readTarget,
+  renderEvidence,
+  runConfigOptions,
+  runEvals,
+  selfPreferenceOf,
+  sha256,
+  splitBody,
+  urlManifestRefusal
+} from "./chunk-NWZ2SLYL.js";
+import {
+  turnBudgetSkipReason
+} from "./chunk-LJ2RDH3F.js";
+import {
+  isExecutionGrant,
+  unknownGrantsMessage
+} from "./chunk-YNHZHSSM.js";
+import {
+  DocevalsError
+} from "./chunk-SE4JN5XN.js";
+import {
+  externalWriteWarnings,
+  offerExternalHomes
+} from "./chunk-K3D6FPV6.js";
+import {
+  LOCATION_KEYWORD,
+  Validator,
+  keyHome,
+  readManifestOrEmpty,
+  readManifestValue,
+  spliceManifestValue
+} from "./chunk-RLR4TKUZ.js";
+import {
+  writeFileAtomic
+} from "./chunk-JVTNMVYU.js";
+import {
+  assertKnownProvider,
+  assertLocalFlag,
+  assertModelHasProvider,
+  localNotice,
+  noticeOnce,
+  providerSpecFor,
+  resolveIdentity,
+  selectProvider,
+  warn
+} from "./chunk-Q35YIDBV.js";
+import {
+  errorMessage
+} from "./chunk-G3T47CFV.js";
+
+// src/docevals/judge/judge.ts
+import {
+  computeConsensus,
+  runEnsemble,
+  zoneFor
+} from "@hawkeyexl/inference";
+
+// src/docevals/judge/verdict-schema.json
+var verdict_schema_default = {
+  $schema: "https://json-schema.org/draft/2020-12/schema",
+  $id: "manni:docevals:verdict",
+  title: "manni docevals judge verdict",
+  type: "object",
+  required: ["claim", "observed", "match", "confidence", "reasoning"],
+  properties: {
+    claim: {
+      type: "string",
+      description: "The specific documented assertion under evaluation."
+    },
+    observed: {
+      type: "string",
+      description: "What was actually observed in the page, quoting evidence where possible."
+    },
+    match: {
+      enum: ["pass", "fail", "partial"],
+      description: "pass only if the assertion is fully satisfied; partial for partial compliance."
+    },
+    confidence: {
+      type: "number",
+      minimum: 0,
+      maximum: 1,
+      description: "Self-reported confidence in this verdict."
+    },
+    reasoning: {
+      type: "string",
+      description: "Why this conclusion was reached."
+    }
+  },
+  additionalProperties: false
+};
+
+// src/docevals/core/reviews.ts
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
+var REVIEWS_PATH = join(".manni/docevals", "reviews.yaml");
+function loadReviews(root) {
+  const path = join(root, REVIEWS_PATH);
+  if (!existsSync(path)) return [];
+  const raw = parseYaml(readFileSync(path, "utf8"));
+  if (!Array.isArray(raw)) return [];
+  return raw;
+}
+function saveReviews(root, reviews) {
+  const path = join(root, REVIEWS_PATH);
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, stringifyYaml(reviews));
+}
+function recordReview(root, entry) {
+  const reviews = loadReviews(root).filter(
+    (r) => !(r.file === entry.file && r.evalName === entry.evalName)
+  );
+  reviews.push(entry);
+  saveReviews(root, reviews);
+}
+function findReview(reviews, file, evalName, body) {
+  const entry = reviews.find(
+    (r) => r.file === file && r.evalName === evalName
+  );
+  if (!entry) return void 0;
+  return entry.contentHash === sha256(body) ? entry : void 0;
+}
+
+// src/docevals/judge/provider.ts
+import {
+  makeProvider as makeInferenceProvider
+} from "@hawkeyexl/inference";
+var CONFIG_KEY = "docevals.provider";
+var toDocevalsError = (message) => new DocevalsError(message);
+function selectProvider2(config, flags = {}, ev = {}) {
+  assertLocalFlag(flags, toDocevalsError);
+  return selectProvider(flags, [
+    {
+      ...ev.provider !== void 0 ? { provider: ev.provider } : {},
+      ...ev.model !== void 0 ? { model: ev.model } : {},
+      ...ev.origin !== void 0 ? { origin: ev.origin } : {}
+    },
+    {
+      ...config.provider !== null ? { provider: config.provider } : {},
+      ...config.model !== null ? { model: config.model } : {},
+      origin: CONFIG_KEY
+    },
+    {
+      ...config.providers.provider !== void 0 ? { provider: config.providers.provider } : {},
+      ...config.providers.model !== void 0 ? { model: config.providers.model } : {},
+      origin: "providers.provider"
+    }
+  ]);
+}
+function announceSelection(selection) {
+  const message = localNotice(selection);
+  if (message !== void 0) noticeOnce(message);
+}
+function assertProviderSelection(selection) {
+  assertKnownProvider(selection.provider, toDocevalsError);
+  assertModelHasProvider(selection.provider, selection.model, CONFIG_KEY, toDocevalsError);
+}
+function providerSpecFor2(config, selection) {
+  const spec = providerSpecFor(config.providers, selection);
+  switch (selection.provider) {
+    case "anthropic":
+      return { ...spec, anthropic: { toolName: "record_verdict" } };
+    case "openai":
+      return { ...spec, openai: { schemaName: "verdict" } };
+    default:
+      return spec;
+  }
+}
+async function resolveProviderIdentity(config, flags = {}, ev = {}) {
+  const selection = selectProvider2(config, flags, ev);
+  const { provider, model } = selection;
+  assertKnownProvider(provider, toDocevalsError);
+  assertModelHasProvider(provider, model, CONFIG_KEY, toDocevalsError);
+  announceSelection(selection);
+  return resolveIdentity(providerSpecFor2(config, { provider, model: model ?? null }), toDocevalsError);
+}
+function constructProvider(config, identity) {
+  try {
+    return makeInferenceProvider(providerSpecFor2(config, identity));
+  } catch (e) {
+    throw new DocevalsError(errorMessage(e));
+  }
+}
+async function makeProvider(config, flags = {}, ev = {}) {
+  return constructProvider(config, await resolveProviderIdentity(config, flags, ev));
+}
+
+// src/docevals/judge/judge.ts
+import { resolve as resolvePath } from "node:path";
+var verdictSchema = verdict_schema_default;
+function makeJudge(deps) {
+  return async (targets, config, options) => {
+    const { provider, root } = deps;
+    const overridden = /* @__PURE__ */ new Map();
+    const buildProvider = deps.providerFor ?? ((ev) => makeProvider(config, options, ev));
+    const keyOf = (s) => `${s.provider}:${s.model ?? ""}`;
+    const defaultKey = keyOf(selectProvider2(config, options));
+    const providerFor = (ev, file) => {
+      const selection = selectProvider2(config, options, {
+        ...ev.provider !== void 0 ? { provider: ev.provider } : {},
+        ...ev.model !== void 0 ? { model: ev.model } : {},
+        origin: `eval "${ev.name}" in ${file}`
+      });
+      announceSelection(selection);
+      const key = keyOf(selection);
+      if (key === defaultKey) return Promise.resolve(provider);
+      let p = overridden.get(key);
+      if (p === void 0) {
+        p = (async () => {
+          assertProviderSelection(selection);
+          return buildProvider(ev);
+        })();
+        overridden.set(key, p);
+      }
+      return p;
+    };
+    const runsFor = (ev) => options.runs ?? ev.runs ?? config.judge.ensembleRuns;
+    const temperature = config.judge.temperature;
+    const chunkChars = options.chunkChars ?? config.judge.chunkChars;
+    const cache = new VerdictCache(
+      resolvePath(root, config.judge.cacheDir),
+      options.noCache !== true,
+      "manni-docevals"
+    );
+    const reviews = loadReviews(root);
+    const maxTurns = options.maxTurns ?? config.judge.maxTurns;
+    let turnsSpent = 0;
+    const results = [];
+    const configured = config.judge.concurrency;
+    const concurrency = Number.isInteger(configured) && configured >= 1 ? configured : config.defaults.concurrency;
+    let index = 0;
+    const judgeTarget = async (target) => {
+      const { plan, eval: ev } = target;
+      const start = Date.now();
+      const runsPerEval = runsFor(ev);
+      let judgeProvider;
+      try {
+        judgeProvider = await providerFor(ev, plan.page.file);
+      } catch (e) {
+        if (!(e instanceof DocevalsError)) throw e;
+        return {
+          evalName: ev.name,
+          type: ev.type,
+          grader: ev.grader,
+          file: plan.page.file,
+          outcome: "error",
+          skipReason: e.message,
+          durationMs: Date.now() - start
+        };
+      }
+      const preference = selfPreferenceOf(plan, ev, judgeProvider.modelName());
+      if (preference) warn(`${plan.page.file}: ${preference.message}`);
+      const selected = readTarget(ev.target, plan);
+      if (!selected.ok) {
+        return {
+          evalName: ev.name,
+          type: ev.type,
+          grader: ev.grader,
+          file: plan.page.file,
+          outcome: "error",
+          skipReason: selected.reason,
+          durationMs: Date.now() - start
+        };
+      }
+      const chunks = splitBody(selected.text, chunkChars);
+      let judged = selected.text;
+      let judgedLabel = selected.label;
+      const key = cacheKey(
+        judgeProvider.provider(),
+        judgeProvider.modelName(),
+        runsPerEval,
+        temperature,
+        judgeCacheBody(chunkChars, selected.text),
+        ev
+      );
+      const cached = cache.get(key) !== void 0;
+      if (chunks.length > 1 && !cached) {
+        const gathered = [];
+        for (const [i, chunk] of chunks.entries()) {
+          if (maxTurns != null && turnsSpent >= maxTurns) {
+            return {
+              evalName: ev.name,
+              type: ev.type,
+              grader: ev.grader,
+              file: plan.page.file,
+              outcome: "skipped",
+              skipReason: `${turnBudgetSkipReason(maxTurns)} (after ${String(i)} of ${String(chunks.length)} parts)`,
+              durationMs: Date.now() - start
+            };
+          }
+          turnsSpent += 1;
+          try {
+            const res = await judgeProvider.completeJSON({
+              system: EVIDENCE_SYSTEM_PROMPT,
+              user: buildEvidenceUser(ev, chunk, {
+                index: i,
+                total: chunks.length
+              }),
+              schema: EVIDENCE_SCHEMA,
+              temperature
+            });
+            const json = res.json;
+            gathered.push({
+              supports: json.supports ?? [],
+              contradicts: json.contradicts ?? []
+            });
+          } catch (e) {
+            return {
+              evalName: ev.name,
+              type: ev.type,
+              grader: ev.grader,
+              file: plan.page.file,
+              outcome: "error",
+              skipReason: `gathering evidence from part ${String(i + 1)} of ${String(chunks.length)} failed: ${errorMessage(e)}`,
+              durationMs: Date.now() - start
+            };
+          }
+        }
+        judged = renderEvidence(gathered, chunks.length);
+        judgedLabel = `${selected.label}, ${String(chunks.length)} parts`;
+      }
+      if (maxTurns != null && !cached) {
+        if (turnsSpent + runsPerEval > maxTurns) {
+          return {
+            evalName: ev.name,
+            type: ev.type,
+            grader: ev.grader,
+            file: plan.page.file,
+            outcome: "skipped",
+            skipReason: turnBudgetSkipReason(maxTurns),
+            durationMs: 0
+          };
+        }
+        turnsSpent += runsPerEval;
+      }
+      const runs = await runEnsemble({
+        provider: judgeProvider,
+        system: JUDGE_SYSTEM_PROMPT,
+        user: buildUserContent(ev, judged, judgedLabel),
+        runs: runsPerEval,
+        temperature,
+        schema: verdictSchema,
+        cache,
+        cacheKey: key,
+        label: "manni-docevals"
+      });
+      const consensusBase = computeConsensus(runs);
+      const zone = zoneFor(consensusBase, config.judge.zones);
+      const consensus = { ...consensusBase, zone };
+      let outcome = zone === "auto-pass" ? "pass" : zone === "auto-fail" ? "fail" : "needs-review";
+      let via;
+      if (outcome === "needs-review") {
+        const review = findReview(reviews, plan.page.file, ev.name, plan.page.body);
+        if (review) {
+          outcome = review.verdict;
+          via = "human-review";
+        }
+      }
+      return {
+        evalName: ev.name,
+        type: ev.type,
+        grader: ev.grader,
+        file: plan.page.file,
+        outcome,
+        consensus,
+        via,
+        ...preference ? { selfPreference: { axis: preference.axis, model: preference.model } } : {},
+        durationMs: Date.now() - start
+      };
+    };
+    const workers = Array.from(
+      { length: Math.min(concurrency, targets.length) },
+      async () => {
+        while (index < targets.length) {
+          const i = index++;
+          const target = targets[i];
+          if (target === void 0) break;
+          results[i] = await judgeTarget(target);
+        }
+      }
+    );
+    await Promise.all(workers);
+    return results;
+  };
+}
+
+// src/docevals/graders/scriptgen.ts
+import { mkdirSync as mkdirSync2, readFileSync as readFileSync2, writeFileSync as writeFileSync2 } from "node:fs";
+import { basename, dirname as dirname2, extname, join as join2, relative, resolve as resolve2 } from "node:path";
+
+// src/docevals/core/frontmatter-edit.ts
+import { parseDocument, Document, YAMLSeq, isMap, isScalar } from "yaml";
+var NAMESPACE = "docevals";
+var BOM = "\uFEFF";
+function splitYamlFrontmatter(content, path) {
+  const bom = content.startsWith(BOM) ? BOM : "";
+  const body = bom ? content.slice(1) : content;
+  const openMatch = /^---(\r?\n)/.exec(body);
+  if (!openMatch) {
+    throw new DocevalsError(
+      `${path}: no YAML frontmatter block to edit (only YAML frontmatter is editable)`
+    );
+  }
+  const eol = openMatch[1] === "\r\n" ? "\r\n" : "\n";
+  const [openLine = "", ...rest] = body.split(/(?<=\n)/);
+  let offset = openLine.length;
+  for (const line of rest) {
+    const stripped = line.replace(/\r?\n$/, "");
+    if (stripped === "---" || stripped === "...") {
+      const blockEnd = offset;
+      return {
+        open: bom + openLine,
+        block: body.slice(openLine.length, blockEnd),
+        suffix: body.slice(blockEnd),
+        eol
+      };
+    }
+    offset += line.length;
+  }
+  throw new DocevalsError(`${path}: unterminated frontmatter block`);
+}
+function applyUpdates(doc, node, updates) {
+  if (updates.grader !== void 0) node.set("grader", updates.grader);
+  if (updates.command !== void 0) {
+    const seq = doc.createNode(updates.command);
+    seq.flow = true;
+    node.set("command", seq);
+  }
+  const hash = updates["generated-assertion-hash"];
+  if (hash !== void 0) node.set("generated-assertion-hash", hash);
+}
+function evalSeq(doc) {
+  const node = doc.get("evals", true);
+  return node instanceof YAMLSeq ? node : void 0;
+}
+function findEvalNode(doc, evalName) {
+  const evals = evalSeq(doc);
+  if (!evals) return void 0;
+  for (const item of evals.items) {
+    if (isMap(item)) {
+      const name = item.get("id") ?? item.get("use");
+      if (name === evalName) return item;
+    }
+  }
+  return void 0;
+}
+function updatePageEval(content, path, evalName, updates) {
+  const { open, block, suffix, eol } = splitYamlFrontmatter(content, path);
+  const doc = parseDocument(block);
+  if (doc.errors.length > 0) {
+    throw new DocevalsError(
+      `${path}: cannot edit frontmatter \u2014 ${doc.errors[0]?.message ?? "parse error"}`
+    );
+  }
+  const node = findEvalNode(doc, evalName);
+  if (!node) {
+    throw new DocevalsError(
+      `${path}: eval "${evalName}" not found in frontmatter`
+    );
+  }
+  applyUpdates(doc, node, updates);
+  let newBlock = doc.toString();
+  if (eol === "\r\n") newBlock = newBlock.replace(/(?<!\r)\n/g, "\r\n");
+  return open + newBlock + suffix;
+}
+function updateConfigEval(configText, configPath, evalName, updates) {
+  const doc = parseDocument(configText);
+  if (doc.errors.length > 0) {
+    throw new DocevalsError(
+      `${configPath}: cannot edit config \u2014 ${doc.errors[0]?.message ?? "parse error"}`
+    );
+  }
+  const node = doc.getIn([NAMESPACE, "evals", evalName]);
+  if (!isMap(node)) {
+    throw new DocevalsError(
+      `${configPath}: eval "${evalName}" not found in config`
+    );
+  }
+  applyUpdates(doc, node, updates);
+  return doc.toString();
+}
+function entryObject(entry) {
+  const obj = { id: entry.id, assertion: entry.assertion };
+  if (entry.type !== void 0) obj.type = entry.type;
+  if (entry.grader !== void 0) obj.grader = entry.grader;
+  if (entry.evidence !== void 0) obj.evidence = entry.evidence;
+  if (entry.examples !== void 0) obj.examples = entry.examples;
+  if (entry.severity !== void 0) obj.severity = entry.severity;
+  return obj;
+}
+var META_PROVENANCE_KEY = "meta-provenance";
+function assertNoCollision(seq, name, path) {
+  for (const item of seq.items) {
+    const existing = isMap(item) ? item.get("id") ?? item.get("use") : isScalar(item) ? item.value : void 0;
+    if (existing === name) {
+      throw new DocevalsError(
+        `${path}: eval "${name}" already exists in frontmatter`
+      );
+    }
+  }
+}
+function appendPageEvals(content, path, entries, metaProvenance) {
+  const format = leadingFrontmatterFormat(content);
+  if (format === "toml" || format === "json") {
+    throw new DocevalsError(
+      `${path}: only YAML frontmatter can be edited (found ${format} frontmatter)`
+    );
+  }
+  const bom = content.startsWith(BOM) ? BOM : "";
+  const stripped = bom ? content.slice(1) : content;
+  const eol = stripped.includes("\r\n") ? "\r\n" : "\n";
+  if (format === void 0) {
+    const doc2 = new Document(
+      entries.length === 0 ? {} : { evals: entries.map(entryObject) }
+    );
+    if (metaProvenance) doc2.set(META_PROVENANCE_KEY, doc2.createNode(metaProvenance));
+    let block2 = doc2.toString();
+    if (eol === "\r\n") block2 = block2.replace(/(?<!\r)\n/g, "\r\n");
+    return `${bom}---${eol}${block2}---${eol}${stripped}`;
+  }
+  const { open, block, suffix, eol: blockEol } = splitYamlFrontmatter(content, path);
+  const doc = parseDocument(block);
+  if (doc.errors.length > 0) {
+    throw new DocevalsError(
+      `${path}: cannot edit frontmatter \u2014 ${doc.errors[0]?.message ?? "parse error"}`
+    );
+  }
+  if (entries.length === 0) {
+    if (metaProvenance) doc.set(META_PROVENANCE_KEY, doc.createNode(metaProvenance));
+    let onlyEntry = doc.toString();
+    if (blockEol === "\r\n") onlyEntry = onlyEntry.replace(/(?<!\r)\n/g, "\r\n");
+    return open + onlyEntry + suffix;
+  }
+  let seq = evalSeq(doc);
+  if (!seq) {
+    const node = doc.get("evals", true);
+    if (node !== void 0) {
+      throw new DocevalsError(
+        `${path}: the evals key in frontmatter is not a list \u2014 expand the string shorthand into a list before appending`
+      );
+    }
+    seq = doc.createNode([]);
+    doc.set("evals", seq);
+  }
+  for (const entry of entries) {
+    assertNoCollision(seq, entry.id, path);
+    seq.add(doc.createNode(entryObject(entry)));
+  }
+  if (metaProvenance) doc.set(META_PROVENANCE_KEY, doc.createNode(metaProvenance));
+  let newBlock = doc.toString();
+  if (blockEol === "\r\n") newBlock = newBlock.replace(/(?<!\r)\n/g, "\r\n");
+  return open + newBlock + suffix;
+}
+function hasEditableEval(content, evalName) {
+  try {
+    const { block } = splitYamlFrontmatter(content, "");
+    const doc = parseDocument(block);
+    return findEvalNode(doc, evalName) !== void 0;
+  } catch {
+    return false;
+  }
+}
+
+// src/docevals/core/write-location.ts
+import { resolve } from "node:path";
+var EVALS_KEY = "evals";
+var META_PROVENANCE_KEY2 = "meta-provenance";
+function prefersExternal(key) {
+  const properties = frontmatterSchema.properties;
+  if (typeof properties !== "object" || properties === null) return false;
+  const property = properties[key];
+  if (typeof property !== "object" || property === null) return false;
+  return property[LOCATION_KEYWORD] === "external";
+}
+function noEntryMessage(label, join3, file, key) {
+  return `${label} carries no ${join3}, which ${file} joins on, so its ${key} has no entry there.`;
+}
+var EvalWriter = class _EvalWriter {
+  constructor(ctx, configSource) {
+    this.ctx = ctx;
+    this.configSource = configSource;
+  }
+  ctx;
+  configSource;
+  landed = [];
+  /**
+   * Build the writer for a run. `targets` are the positional paths as typed,
+   * which is where a page in no collection gets its proposed home from; a run
+   * that took its paths from the collections passes none.
+   *
+   * The context is built from the config docevals already loaded rather than
+   * by loading it again through meta. A `manni.config.yaml` carrying only a
+   * `docevals:` key is a single-tool file to the metadata tool, which refuses
+   * it; docevals reads it every run, and where its own evals go cannot depend
+   * on whether a sibling can read the same file. The collections are the
+   * declared ones, since a page named by path is still a member of whatever
+   * contains it. `config: null` leaves the page's schema set to the page,
+   * which is enough: the keys this writer homes are named outright in the
+   * relocation it plans.
+   */
+  static for(config, cwd, targets) {
+    const noConfig = config.configSource === null;
+    const collections = config.collections;
+    return new _EvalWriter(
+      {
+        cwd,
+        // Page labels are relative to the run's discovery root, so a manifest
+        // path this writer reports is spelled against the same directory
+        // `core/external.ts` reports a merged value's source against.
+        base: cwd,
+        noConfig,
+        config: null,
+        ...noConfig ? {} : { configDir: config.configDir, configPath: config.configPath },
+        collections,
+        declaredCollections: collections,
+        targets,
+        validator: new Validator()
+      },
+      config.configSource ?? cwd
+    );
+  }
+  /** Where `key` of the page labelled `label` goes. `data` is its metadata. */
+  async homeFor(label, data, key) {
+    const home = await keyHome(this.ctx, label, data, key);
+    if (home.kind === "url") {
+      return { kind: "url", collection: home.collection, file: home.file };
+    }
+    if (home.kind === "manifest") {
+      if (home.entry === void 0) {
+        return {
+          kind: "no-entry",
+          message: noEntryMessage(label, home.join, home.file, key)
+        };
+      }
+      return {
+        kind: "manifest",
+        file: home.file,
+        absPath: home.absPath,
+        join: home.join,
+        entry: home.entry,
+        perPage: home.perPage
+      };
+    }
+    return { kind: "page", prefersExternal: prefersExternal(key), proposed: home.home };
+  }
+  /**
+   * The refusal for an eval key a URL manifest owns, in chunk 1's wording. A
+   * run reaches it only where the load-time refusal could not: a manifest
+   * declared for a collection whose pages this run never merged.
+   */
+  urlRefusal(home, key) {
+    return new DocevalsError(urlManifestRefusal(this.configSource, home.collection, key));
+  }
+  /** Record that `key` stayed on the page, for the W1 and W2 lines. */
+  stayedOnPage(label, key, proposed) {
+    this.landed.push({ label, key, home: proposed });
+  }
+  /** The W1 and W2 lines for everything that stayed on a page. Without the prefix. */
+  warnings(dryRun) {
+    return this.landed.length === 0 ? [] : externalWriteWarnings(this.landed, dryRun);
+  }
+  /**
+   * Ask P1 once per collection the flagged writes would be homed in. Returns
+   * true when a relocation was applied, which leaves the caller's config,
+   * pages and this writer stale: rebuild all three before writing.
+   */
+  async offer(writes, hooks) {
+    if (writes.length === 0) return false;
+    const applied = await offerExternalHomes(this.ctx, writes, hooks);
+    return applied.size > 0;
+  }
+  /** The value the manifest holds for this page's entry, or `undefined`. */
+  async readManifest(home, key) {
+    return readManifestValue(await this.text(home.absPath, home.perPage), {
+      entry: home.entry,
+      key,
+      join: home.join
+    });
+  }
+  /** Write `value` into this page's entry, changing no other byte of the file. */
+  async writeManifest(home, key, value) {
+    const text = await this.text(home.absPath, home.perPage);
+    let spliced;
+    try {
+      spliced = spliceManifestValue(text, {
+        entry: home.entry,
+        key,
+        value,
+        join: home.join,
+        file: home.file
+      }).text;
+    } catch (e) {
+      throw new DocevalsError(errorMessage(e));
+    }
+    if (spliced !== text) {
+      await writeFileAtomic(home.absPath, spliced, { createParents: true });
+    }
+  }
+  /**
+   * The manifest's bytes. A per-page manifest that is not on disk yet is a
+   * page with nothing recorded, held as empty text and created by the write,
+   * exactly as `manni meta fill` holds one. A missing concrete manifest is
+   * refused, as it always was.
+   */
+  async text(absPath, perPage) {
+    try {
+      return await readManifestOrEmpty(absPath, perPage);
+    } catch (e) {
+      throw new DocevalsError(
+        `${resolve(absPath)} could not be read: ${errorMessage(e)}`
+      );
+    }
+  }
+};
+function manifestEvalList(value, file, entry) {
+  if (value === void 0 || value === null) return [];
+  if (Array.isArray(value)) return [...value];
+  throw new DocevalsError(
+    `${file}: the evals key of ${entry} is not a list \u2014 expand the string shorthand into a list before appending`
+  );
+}
+function updateManifestEval(list, name, updates) {
+  const index = list.findIndex((item) => named(item) === name);
+  const held = index === -1 ? void 0 : list[index];
+  if (held === void 0) return void 0;
+  const set = { ...held };
+  if (updates.grader !== void 0) set.grader = updates.grader;
+  if (updates.command !== void 0) set.command = updates.command;
+  const hash = updates["generated-assertion-hash"];
+  if (hash !== void 0) set["generated-assertion-hash"] = hash;
+  const next = [...list];
+  next[index] = set;
+  return next;
+}
+function named(item) {
+  if (typeof item !== "object" || item === null || Array.isArray(item)) return void 0;
+  const entry = item;
+  const id = entry.id ?? entry.use;
+  return typeof id === "string" ? id : void 0;
+}
+
+// src/docevals/graders/scriptgen.ts
+var SCRIPTGEN_SYSTEM_PROMPT = [
+  "You write small, self-contained Node.js check scripts that verify a",
+  "deterministic assertion about a documentation page.",
+  "",
+  "Contract for every script:",
+  "- ES module (.mjs) using only Node.js built-in modules. No dependencies.",
+  "- The page's absolute path arrives as process.argv[2] (also MANNI_DOCEVALS_FILE).",
+  "- Exit 0 when the assertion holds, 1 when it fails, 2 on operational error.",
+  "- On failure, print a short human-readable reason to stderr.",
+  "- Deterministic: no network access, no spawning processes, no randomness.",
+  "- The page content may start with a YAML frontmatter block fenced by ---",
+  "  lines; account for it when checking body content.",
+  "- Prefer simple, reviewable logic (regex/string checks) over cleverness.",
+  'Respond with JSON: { "code": "<the full script source>" }.'
+].join("\n");
+var SCRIPT_SCHEMA = {
+  type: "object",
+  required: ["code"],
+  properties: {
+    code: { type: "string", minLength: 1 },
+    notes: { type: "string" }
+  },
+  additionalProperties: false
+};
+function buildScriptgenUser(assertion, evalName, file, body) {
+  return [
+    `# Assertion to check`,
+    assertion,
+    "",
+    `# Eval name`,
+    evalName,
+    "",
+    `# Page path`,
+    file,
+    "",
+    "# Current page content (for grounding \u2014 the script must express the",
+    "# assertion generally, not hardcode this exact content)",
+    "",
+    body
+  ].join("\n");
+}
+function scriptLocationFor(target, config, root) {
+  const ev = target.eval;
+  const page = target.plan.page;
+  if (ev.source === "page") {
+    const pageDir = dirname2(page.absPath);
+    const pattern = config.scripts.dir;
+    const dir2 = pattern.includes("{docDir}") ? pattern.replaceAll("{docDir}", pageDir) : resolve2(root, pattern);
+    const base = basename(page.absPath, extname(page.absPath));
+    const scriptAbsPath2 = join2(dir2, `${base}.${ev.name}.mjs`);
+    const rel2 = relative(pageDir, scriptAbsPath2).replace(/\\/g, "/");
+    return { scriptAbsPath: scriptAbsPath2, command: ["node", rel2, "{file}"] };
+  }
+  const dir = resolve2(config.configDir, config.scripts.configDir);
+  const scriptAbsPath = join2(dir, `${ev.name}.mjs`);
+  const rel = relative(config.configDir, scriptAbsPath).replace(/\\/g, "/");
+  return { scriptAbsPath, command: ["node", rel, "{file}"] };
+}
+function header(assertion, evalName) {
+  return [
+    "// manni docevals generated check",
+    `// Eval: ${evalName}`,
+    `// Assertion: ${assertion.replace(/\s+/g, " ").trim()}`,
+    "// Exit 0 = pass, 1 = fail, 2 = operational error.",
+    ""
+  ].join("\n");
+}
+function makeGenerateScripts(deps) {
+  return async (targets, config, _options) => {
+    const generatedPaths = [];
+    const refusals = [];
+    const doneConfigEvals = /* @__PURE__ */ new Set();
+    let provider;
+    let writer;
+    const getWriter = () => writer ??= EvalWriter.for(config, deps.root, []);
+    for (const target of targets) {
+      const ev = target.eval;
+      if (!ev.assertion) continue;
+      if (ev.source === "config" && doneConfigEvals.has(ev.name)) continue;
+      const page = target.plan.page;
+      let home;
+      if (ev.source === "page") {
+        home = await getWriter().homeFor(
+          page.file,
+          page.frontmatter.data,
+          EVALS_KEY
+        );
+        if (home.kind === "url") {
+          throw getWriter().urlRefusal(home, EVALS_KEY);
+        }
+        if (home.kind === "no-entry") {
+          refusals.push({ file: page.file, evalName: ev.name, message: home.message });
+          continue;
+        }
+      }
+      if (provider === void 0) {
+        try {
+          provider = typeof deps.provider === "function" ? await deps.provider() : deps.provider;
+        } catch (e) {
+          if (!(e instanceof DocevalsError)) throw e;
+          return { generatedPaths, refusals, unavailable: e.message };
+        }
+      }
+      const location = scriptLocationFor(target, config, deps.root);
+      let code;
+      try {
+        const response = await provider.completeJSON({
+          system: SCRIPTGEN_SYSTEM_PROMPT,
+          user: buildScriptgenUser(
+            ev.assertion,
+            ev.name,
+            target.plan.page.file,
+            target.plan.page.body
+          ),
+          schema: SCRIPT_SCHEMA,
+          temperature: 0
+        });
+        const json = response.json;
+        if (typeof json.code !== "string" || json.code.length === 0) {
+          continue;
+        }
+        code = json.code;
+      } catch {
+        continue;
+      }
+      mkdirSync2(dirname2(location.scriptAbsPath), { recursive: true });
+      writeFileSync2(
+        location.scriptAbsPath,
+        // Exactly one final newline, whatever the model sent: the script is
+        // committed and reviewed, and a missing one shows in every diff.
+        `${header(ev.assertion, ev.name)}${code.trim()}
+`
+      );
+      generatedPaths.push(
+        relative(deps.root, location.scriptAbsPath).replace(/\\/g, "/")
+      );
+      const updates = {
+        command: location.command,
+        "generated-assertion-hash": sha256(ev.assertion)
+      };
+      if (ev.source === "page" && home?.kind === "manifest") {
+        const at = getWriter();
+        const list = manifestEvalList(
+          await at.readManifest(home, EVALS_KEY),
+          home.file,
+          home.entry
+        );
+        const next = updateManifestEval(list, ev.name, updates);
+        if (next === void 0) {
+          throw new DocevalsError(
+            `${home.file}: eval "${ev.name}" not found in the entry for ${home.entry}`
+          );
+        }
+        await at.writeManifest(home, EVALS_KEY, next);
+      } else if (ev.source === "page") {
+        const abs = page.absPath;
+        const updated = updatePageEval(
+          readFileSync2(abs, "utf8"),
+          page.file,
+          ev.name,
+          updates
+        );
+        writeFileSync2(abs, updated);
+      } else {
+        const updated = updateConfigEval(
+          readFileSync2(config.configPath, "utf8"),
+          config.configPath,
+          ev.name,
+          updates
+        );
+        writeFileSync2(config.configPath, updated);
+        doneConfigEvals.add(ev.name);
+      }
+      if (ev.source === "config") {
+        for (const t of targets) {
+          if (t.eval.name === ev.name && t.eval.source === "config") {
+            t.eval.command = scriptLocationFor(t, config, deps.root).command;
+            t.eval.generatedAssertionHash = updates["generated-assertion-hash"];
+          }
+        }
+      } else {
+        target.eval.command = location.command;
+        target.eval.generatedAssertionHash = updates["generated-assertion-hash"];
+      }
+    }
+    return { generatedPaths, refusals };
+  };
+}
+
+// src/docevals/commands/run.ts
+function asGrants(values) {
+  if (values === void 0) return void 0;
+  const unknown = values.filter((v) => !isExecutionGrant(v));
+  if (unknown.length > 0) {
+    throw new DocevalsError(unknownGrantsMessage(unknown));
+  }
+  return values;
+}
+async function runRun(paths, options = {}, engineOverrides = {}) {
+  const cwd = options.cwd ?? process.cwd();
+  const judgeOptions = {
+    provider: options.provider,
+    model: options.model,
+    local: options.local,
+    runs: options.runs,
+    chunkChars: options.chunkChars,
+    noCache: options.cache === false,
+    maxTurns: options.maxTurns ?? null
+  };
+  const config = loadRunConfig(runConfigOptions(paths, options), cwd);
+  assertProviderSelection(selectProvider2(config, judgeOptions));
+  let judge;
+  let generateScripts;
+  let resolving;
+  const provider = () => resolving ??= makeProvider(config, judgeOptions);
+  const wantsJudge = !("judge" in engineOverrides);
+  const wantsGeneration = !("generateScripts" in engineOverrides);
+  if (wantsJudge || wantsGeneration) {
+    if (!options.deterministicOnly) {
+      try {
+        const resolved = await provider();
+        if (wantsJudge) judge = makeJudge({ provider: resolved, root: cwd });
+        if (wantsGeneration && options.generate !== false) {
+          generateScripts = makeGenerateScripts({ provider: resolved, root: cwd });
+        }
+      } catch (e) {
+        if (options.aiOnly || !(e instanceof DocevalsError)) throw e;
+        warn(`provider unavailable \u2014 ${e.message}. Running deterministic evals only.`);
+      }
+    } else if (wantsGeneration && options.generate !== false) {
+      generateScripts = makeGenerateScripts({ provider, root: cwd });
+    }
+  }
+  return runEvals({
+    judge,
+    generateScripts,
+    config,
+    configPath: options.config,
+    noConfig: options.noConfig,
+    paths,
+    collection: options.collection,
+    exclude: options.exclude,
+    cwd: options.cwd,
+    deterministicOnly: options.deterministicOnly,
+    aiOnly: options.aiOnly,
+    allowExecution: asGrants(options.allowExecution),
+    execution: options.execution,
+    generate: options.generate,
+    failOnReview: options.failOnReview,
+    evalNames: options.evalNames,
+    suite: options.suite,
+    since: options.since,
+    newerThan: options.newerThan,
+    baseline: options.baseline,
+    writeBaseline: options.writeBaseline,
+    toolVersion: options.toolVersion,
+    judgeOptions,
+    ...engineOverrides
+  });
+}
+
+export {
+  loadReviews,
+  recordReview,
+  selectProvider2 as selectProvider,
+  assertProviderSelection,
+  providerSpecFor2 as providerSpecFor,
+  resolveProviderIdentity,
+  constructProvider,
+  makeProvider,
+  makeJudge,
+  updatePageEval,
+  updateConfigEval,
+  entryObject,
+  appendPageEvals,
+  hasEditableEval,
+  EVALS_KEY,
+  META_PROVENANCE_KEY2 as META_PROVENANCE_KEY,
+  EvalWriter,
+  manifestEvalList,
+  updateManifestEval,
+  SCRIPTGEN_SYSTEM_PROMPT,
+  buildScriptgenUser,
+  scriptLocationFor,
+  makeGenerateScripts,
+  runRun
+};
+//# sourceMappingURL=chunk-PHF3D7ER.js.map